1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.kuali.maven.mojo.s3;
17
18 import java.io.ByteArrayInputStream;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.text.SimpleDateFormat;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.TimeZone;
29
30 import org.apache.commons.beanutils.BeanUtils;
31 import org.apache.commons.io.IOUtils;
32 import org.apache.commons.lang.StringUtils;
33 import org.apache.maven.plugin.MojoExecutionException;
34 import org.apache.maven.plugin.MojoFailureException;
35 import org.apache.maven.plugin.descriptor.PluginDescriptor;
36 import org.apache.maven.project.MavenProject;
37 import org.kuali.common.threads.ElementHandler;
38 import org.kuali.common.threads.ExecutionStatistics;
39 import org.kuali.common.threads.ThreadInvoker;
40 import org.kuali.maven.common.UrlBuilder;
41
42 import com.amazonaws.AmazonServiceException;
43 import com.amazonaws.auth.AWSCredentials;
44 import com.amazonaws.services.s3.AmazonS3Client;
45 import com.amazonaws.services.s3.model.CannedAccessControlList;
46 import com.amazonaws.services.s3.model.CopyObjectRequest;
47 import com.amazonaws.services.s3.model.ListObjectsRequest;
48 import com.amazonaws.services.s3.model.ObjectListing;
49 import com.amazonaws.services.s3.model.ObjectMetadata;
50 import com.amazonaws.services.s3.model.PutObjectRequest;
51 import com.amazonaws.services.s3.model.S3Object;
52 import com.amazonaws.services.s3.model.S3ObjectSummary;
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83 public class UpdateOriginBucketMojo extends S3Mojo {
84 UrlBuilder builder = new UrlBuilder();
85 SimpleFormatter formatter = new SimpleFormatter();
86 ThreadInvoker invoker = new ThreadInvoker();
87
88 private static final String S3_INDEX_METADATA_KEY = "maven-cloudfront-plugin-index";
89 private static final String S3_INDEX_CONTENT_TYPE = "text/html";
90 CloudFrontHtmlGenerator generator;
91 S3DataConverter converter;
92
93
94
95
96
97
98
99 private List<String> orgPomGavs;
100
101
102
103
104
105
106 private int threads;
107
108
109
110
111
112
113
114
115
116
117
118
119
120 private String organizationGroupId;
121
122
123
124
125
126
127
128 private String cacheControl;
129
130
131
132
133
134
135 private boolean copyDefaultObjectWithDelimiter;
136
137
138
139
140
141
142
143
144 private boolean copyDefaultObjectWithoutDelimiter;
145
146
147
148
149
150
151 private String css;
152
153
154
155
156
157
158
159 private String fileImage;
160
161
162
163
164
165
166
167 private String directoryImage;
168
169
170
171
172
173
174 private String timezone;
175
176
177
178
179
180
181 private String dateFormat;
182
183
184
185
186
187
188
189
190 private String defaultObjectKey;
191
192
193
194
195
196
197 private String browseKey;
198
199
200
201
202
203
204
205 private CannedAccessControlList acl;
206
207
208
209
210
211
212
213
214
215
216 protected List<String> getPrefixes(ObjectListing listing, String prefix, String delimiter) {
217 List<String> commonPrefixes = listing.getCommonPrefixes();
218 List<String> pathPrefixes = getPathsToRoot(delimiter, prefix);
219 List<String> prefixes = new ArrayList<String>();
220 prefixes.addAll(commonPrefixes);
221 prefixes.addAll(pathPrefixes);
222 Collections.sort(prefixes);
223 return prefixes;
224 }
225
226
227
228
229
230
231
232
233 protected ObjectListing getObjectListing(S3BucketContext context, ListObjectsRequest request) {
234 String prefix = getPrefix();
235 String bucket = context.getBucket();
236 String delimiter = context.getDelimiter();
237 Integer maxKeys = context.getMaxKeys();
238
239
240
241 long start = System.currentTimeMillis();
242 ObjectListing listing = context.getClient().listObjects(request);
243 long millis = System.currentTimeMillis() - start;
244 getLog().info(getListMsg(listing.getCommonPrefixes().size(), millis));
245
246
247 if (listing.isTruncated()) {
248 throw new AmazonServiceException("The listing for " + bucket + delimiter + prefix + " exceeded " + maxKeys);
249 }
250 return listing;
251 }
252
253
254
255
256
257
258
259
260 protected List<ListObjectsContext> getListObjectsContexts(S3BucketContext bucketContext, List<String> prefixes) {
261 List<ListObjectsContext> contexts = new ArrayList<ListObjectsContext>();
262 for (String prefix : prefixes) {
263 ListObjectsRequest request = getListObjectsRequest(bucketContext, prefix);
264 ListObjectsContext context = new ListObjectsContext();
265 context.setRequest(request);
266 context.setBucketContext(bucketContext);
267 contexts.add(context);
268 }
269 return contexts;
270 }
271
272
273
274
275
276
277
278
279 protected ListObjectsRequest getListObjectsRequest(S3BucketContext context, String prefix) {
280 String bucket = context.getBucket();
281 String delimiter = context.getDelimiter();
282 Integer maxKeys = context.getMaxKeys();
283 if (prefix.equals(delimiter)) {
284 prefix = null;
285 }
286 ListObjectsRequest request = new ListObjectsRequest(bucket, prefix, null, delimiter, maxKeys);
287 return request;
288 }
289
290
291
292
293
294
295
296
297 protected String getListMsg(int subDirectoryCount, long millis) {
298 StringBuilder sb = new StringBuilder();
299 sb.append("S3 Directories: " + subDirectoryCount);
300 sb.append(" Listing Request: " + formatter.getTime(millis));
301 return sb.toString();
302 }
303
304 @Override
305 public void executeMojo() throws MojoExecutionException, MojoFailureException {
306 try {
307
308 long start = System.currentTimeMillis();
309 getLog().info("Updating S3 bucket - " + getBucket());
310
311
312 updateMojoState();
313
314
315 S3BucketContext context = getS3BucketContext();
316
317
318 generator = new CloudFrontHtmlGenerator(context);
319 converter = new S3DataConverter(context);
320 converter.setBrowseKey(getBrowseKey());
321
322
323 String prefix = getPrefix();
324 getLog().info("Re-indexing content for - " + prefix);
325
326
327 ListObjectsRequest request = getListObjectsRequest(context, prefix);
328
329
330 ObjectListing projectDirectory = getObjectListing(context, request);
331
332
333
334 List<String> prefixes = getPrefixes(projectDirectory, prefix, context.getDelimiter());
335
336
337 List<ObjectListing> listings = getObjectListings(context, prefixes, threads);
338
339
340 listings.add(projectDirectory);
341
342
343 List<S3PrefixContext> contexts = getS3PrefixContexts(context, listings);
344
345
346 List<UpdateDirectoryContext> udcs = getUpdateDirContexts(contexts);
347
348
349 ElementHandler<UpdateDirectoryContext> handler = new UpdateDirectoryContextHandler(this);
350
351
352 ExecutionStatistics stats = invoker.invokeThreads(threads, handler, udcs);
353
354
355 long millis = stats.getExecutionTime();
356 long count = stats.getIterationCount();
357 getLog().info("Updated " + count + " bucket keys. Time: " + formatter.getTime(millis));
358 getLog().info("S3 Bucket update complete - " + getBucket());
359 getLog().info("Total time: " + formatter.getTime(System.currentTimeMillis() - start));
360 } catch (Exception e) {
361 throw new MojoExecutionException("Unexpected error: ", e);
362 }
363 }
364
365
366
367
368
369
370
371
372
373 protected List<ObjectListing> getObjectListings(S3BucketContext context, List<String> prefixes, int maxThreads) {
374 List<ListObjectsContext> contexts = getListObjectsContexts(context, prefixes);
375 ListObjectsContextHandler elementHandler = new ListObjectsContextHandler();
376 ExecutionStatistics stats = invoker.invokeThreads(maxThreads, elementHandler, contexts);
377 long millis = stats.getExecutionTime();
378 long count = stats.getIterationCount();
379 getLog().info("Acquired listings for " + count + " prefixes. Time: " + formatter.getTime(millis));
380 return elementHandler.getObjectListings();
381 }
382
383
384
385
386
387
388
389
390 protected List<UpdateDirectoryContext> getUpdateDirContexts(List<S3PrefixContext> contexts) {
391 List<UpdateDirectoryContext> list = new ArrayList<UpdateDirectoryContext>();
392 for (S3PrefixContext context : contexts) {
393
394
395 if (context.isRoot()) {
396 UpdateDirectoryContext udc = new UpdateDirectoryContext();
397 udc.setContext(context);
398 list.add(udc);
399 continue;
400 }
401
402
403 String delimiter = context.getBucketContext().getDelimiter();
404 String trimmedPrefix = converter.getTrimmedPrefix(context.getPrefix(), delimiter);
405
406 UpdateDirectoryContext udc1 = new UpdateDirectoryContext();
407 udc1.setContext(context);
408 udc1.setCopyIfExists(isCopyDefaultObjectWithDelimiter());
409 udc1.setCopyToKey(context.getPrefix());
410
411 UpdateDirectoryContext udc2 = new UpdateDirectoryContext();
412 udc2.setContext(context);
413 udc2.setCopyIfExists(isCopyDefaultObjectWithoutDelimiter());
414 udc2.setCopyToKey(trimmedPrefix);
415
416 list.add(udc1);
417 list.add(udc2);
418
419 }
420 return list;
421 }
422
423
424
425
426
427
428
429
430
431 protected List<String> getPathsToRoot(String delimiter, String startingPrefix) {
432 List<String> list = new ArrayList<String>();
433
434 list.add(delimiter);
435
436 String[] prefixes = StringUtils.splitByWholeSeparator(startingPrefix, delimiter);
437 String newPrefix = "";
438 for (int i = 0; i < prefixes.length - 2; i++) {
439 newPrefix += prefixes[i] + delimiter;
440 list.add(newPrefix);
441 }
442 return list;
443 }
444
445
446
447
448
449
450
451
452 protected String getDefaultPrefix(MavenProject project, String orgGroupId) {
453 List<MavenProject> orgPoms = builder.getMavenProjects(orgPomGavs);
454 return builder.getSitePath(project, orgPoms, orgGroupId);
455 }
456
457
458
459
460 protected void updatePrefix() {
461 String s = getPrefix();
462 if (StringUtils.isEmpty(s)) {
463 s = getDefaultPrefix(getProject(), getOrganizationGroupId());
464 }
465 if (!s.endsWith(getDelimiter())) {
466 s = s + getDelimiter();
467 }
468 setPrefix(s);
469 }
470
471
472
473
474 protected void updateMojoState() throws MojoExecutionException {
475 updateCredentials();
476 validateCredentials();
477 updatePrefix();
478 }
479
480
481
482
483 protected S3BucketContext getS3BucketContext() throws MojoExecutionException {
484 AWSCredentials credentials = getCredentials();
485 AmazonS3Client client = new AmazonS3Client(credentials);
486 S3BucketContext context = new S3BucketContext();
487 try {
488 BeanUtils.copyProperties(context, this);
489 } catch (Exception e) {
490 throw new MojoExecutionException("Error copying properties", e);
491 }
492 context.setClient(client);
493 context.setLastModifiedDateFormatter(getLastModifiedDateFormatter());
494 context.setAbout(getAbout());
495 return context;
496 }
497
498
499
500
501
502
503 protected PutObjectRequest getPutIndexObjectRequest(String html, String key) {
504 InputStream in = new ByteArrayInputStream(html.getBytes());
505 ObjectMetadata om = new ObjectMetadata();
506 om.setCacheControl(getCacheControl());
507 String contentType = S3_INDEX_CONTENT_TYPE;
508 om.setContentType(contentType);
509 om.setContentLength(html.length());
510 om.addUserMetadata(S3_INDEX_METADATA_KEY, "true");
511 PutObjectRequest request = new PutObjectRequest(getBucket(), key, in, om);
512 request.setCannedAcl(getAcl());
513 return request;
514 }
515
516
517
518
519 protected SimpleDateFormat getLastModifiedDateFormatter() {
520 SimpleDateFormat sdf = new SimpleDateFormat(getDateFormat());
521 sdf.setTimeZone(TimeZone.getTimeZone(getTimezone()));
522 return sdf;
523 }
524
525
526
527
528 protected boolean isEmpty(Collection<?> c) {
529 return c == null || c.size() == 0;
530 }
531
532
533
534
535 protected String getAbout() {
536 String date = getLastModifiedDateFormatter().format(new Date());
537 PluginDescriptor descriptor = (PluginDescriptor) this.getPluginContext().get("pluginDescriptor");
538 if (descriptor == null) {
539
540 return "Listing generated by the maven-cloudfront-plugin on " + date;
541 } else {
542 String name = descriptor.getArtifactId();
543 String version = descriptor.getVersion();
544 return "Listing generated by the " + name + " v" + version + " on " + date;
545 }
546 }
547
548 public void updateDirectory(UpdateDirectoryContext context) throws IOException {
549 updateDirectory(context.getContext(), context.isCopyIfExists(), context.getCopyToKey());
550 }
551
552
553
554
555
556
557 protected void updateDirectory(S3PrefixContext context, boolean isCopyIfExists, String copyToKey)
558 throws IOException {
559 S3BucketContext bucketContext = context.getBucketContext();
560 AmazonS3Client client = context.getBucketContext().getClient();
561 String bucket = bucketContext.getBucket();
562
563 boolean containsDefaultObject = isExistingObject(context.getObjectListing(), context.getDefaultObjectKey());
564 if (containsDefaultObject && isCopyIfExists) {
565
566 String sourceKey = context.getDefaultObjectKey();
567 String destKey = copyToKey;
568 CopyObjectRequest request = getCopyObjectRequest(bucket, sourceKey, destKey);
569 getLog().debug("Copy: " + sourceKey + " to " + destKey);
570 client.copyObject(request);
571 } else {
572
573 PutObjectRequest request = getPutIndexObjectRequest(context.getHtml(), copyToKey);
574 getLog().debug("Put: " + copyToKey);
575 client.putObject(request);
576 }
577 }
578
579
580
581
582 protected void updateDirectory(final S3PrefixContext context) throws IOException {
583 String trimmedPrefix = converter.getTrimmedPrefix(context.getPrefix(), context.getBucketContext()
584 .getDelimiter());
585
586
587 updateDirectory(context, isCopyDefaultObjectWithDelimiter(), context.getPrefix());
588
589
590 updateDirectory(context, isCopyDefaultObjectWithoutDelimiter(), trimmedPrefix);
591
592
593
594
595 }
596
597
598
599
600
601 public void updateRoot(UpdateDirectoryContext udc) throws IOException {
602 updateRoot(udc.getContext());
603 }
604
605
606
607
608
609 protected void updateRoot(S3PrefixContext context) throws IOException {
610 AmazonS3Client client = context.getBucketContext().getClient();
611
612
613 PutObjectRequest request1 = getPutIndexObjectRequest(context.getHtml(), context.getBrowseHtmlKey());
614 StringBuilder sb = new StringBuilder();
615 sb.append(context.getBrowseHtmlKey());
616 client.putObject(request1);
617
618 boolean isCreateOrUpdateDefaultObject = isCreateOrUpdateDefaultObject(context);
619 if (!isCreateOrUpdateDefaultObject) {
620 getLog().debug("Put: " + sb.toString());
621 return;
622 }
623
624
625 PutObjectRequest request2 = getPutIndexObjectRequest(context.getHtml(), context.getDefaultObjectKey());
626 getLog().debug("Put: " + sb.toString() + ", " + context.getDefaultObjectKey());
627 client.putObject(request2);
628 }
629
630
631
632
633 protected List<S3PrefixContext> getS3PrefixContexts(S3BucketContext context, List<ObjectListing> listings) {
634 List<S3PrefixContext> contexts = new ArrayList<S3PrefixContext>();
635 for (ObjectListing listing : listings) {
636 S3PrefixContext prefixContext = getS3PrefixContext(context, listing);
637 contexts.add(prefixContext);
638 }
639 return contexts;
640 }
641
642
643
644
645 protected S3PrefixContext getS3PrefixContext(S3BucketContext context, ObjectListing objectListing) {
646 String prefix = objectListing.getPrefix();
647 String delimiter = context.getDelimiter();
648 List<String[]> data = converter.getData(objectListing, prefix, delimiter);
649 String html = generator.getHtml(data, prefix, delimiter);
650 String defaultObjectKey = StringUtils.isEmpty(prefix) ? getDefaultObjectKey() : prefix + getDefaultObjectKey();
651 String browseHtmlKey = StringUtils.isEmpty(prefix) ? getBrowseKey() : prefix + getBrowseKey();
652
653 boolean isRoot = StringUtils.isEmpty(prefix);
654
655 S3PrefixContext prefixContext = new S3PrefixContext();
656 prefixContext.setObjectListing(objectListing);
657 prefixContext.setHtml(html);
658 prefixContext.setRoot(isRoot);
659 prefixContext.setDefaultObjectKey(defaultObjectKey);
660 prefixContext.setPrefix(prefix);
661 prefixContext.setBucketContext(context);
662 prefixContext.setBrowseHtmlKey(browseHtmlKey);
663 return prefixContext;
664 }
665
666
667
668
669 protected boolean isExistingObject(final ObjectListing objectListing, final String key) {
670 List<S3ObjectSummary> summaries = objectListing.getObjectSummaries();
671 for (S3ObjectSummary summary : summaries) {
672 if (key.equals(summary.getKey())) {
673 return true;
674 }
675 }
676 return false;
677 }
678
679
680
681
682
683
684 protected boolean isCreateOrUpdateDefaultObject(final S3PrefixContext context) {
685 if (!isExistingObject(context.getObjectListing(), context.getDefaultObjectKey())) {
686
687 return true;
688 }
689 S3BucketContext s3Context = context.getBucketContext();
690
691
692 S3Object s3Object = s3Context.getClient().getObject(s3Context.getBucket(), context.getDefaultObjectKey());
693 boolean isOurDefaultObject = isOurObject(s3Object);
694 IOUtils.closeQuietly(s3Object.getObjectContent());
695 if (isOurDefaultObject) {
696 return true;
697 } else {
698 return false;
699 }
700 }
701
702
703
704
705
706 protected boolean isOurObject(final S3Object s3Object) {
707 ObjectMetadata metadata = s3Object.getObjectMetadata();
708 Map<String, String> userMetadata = metadata.getUserMetadata();
709 String value = userMetadata.get(S3_INDEX_METADATA_KEY);
710 boolean isOurObject = "true".equals(value);
711 return isOurObject;
712 }
713
714
715
716
717 protected CopyObjectRequest getCopyObjectRequest(final String bucket, final String sourceKey, final String destKey) {
718 CopyObjectRequest request = new CopyObjectRequest(bucket, sourceKey, bucket, destKey);
719 request.setCannedAccessControlList(getAcl());
720 return request;
721 }
722
723 public String getTimezone() {
724 return timezone;
725 }
726
727 public void setTimezone(final String timezone) {
728 this.timezone = timezone;
729 }
730
731 public String getDateFormat() {
732 return dateFormat;
733 }
734
735 public void setDateFormat(final String dateFormat) {
736 this.dateFormat = dateFormat;
737 }
738
739 public String getDefaultObjectKey() {
740 return defaultObjectKey;
741 }
742
743 public void setDefaultObjectKey(final String defaultCloudFrontObject) {
744 this.defaultObjectKey = defaultCloudFrontObject;
745 }
746
747 public String getFileImage() {
748 return fileImage;
749 }
750
751 public void setFileImage(final String fileImage) {
752 this.fileImage = fileImage;
753 }
754
755 public String getDirectoryImage() {
756 return directoryImage;
757 }
758
759 public void setDirectoryImage(final String directoryImage) {
760 this.directoryImage = directoryImage;
761 }
762
763 public String getCss() {
764 return css;
765 }
766
767 public void setCss(final String css) {
768 this.css = css;
769 }
770
771 public boolean isCopyDefaultObjectWithDelimiter() {
772 return copyDefaultObjectWithDelimiter;
773 }
774
775 public void setCopyDefaultObjectWithDelimiter(final boolean copyDefaultObjectWithDelimiter) {
776 this.copyDefaultObjectWithDelimiter = copyDefaultObjectWithDelimiter;
777 }
778
779 public boolean isCopyDefaultObjectWithoutDelimiter() {
780 return copyDefaultObjectWithoutDelimiter;
781 }
782
783 public void setCopyDefaultObjectWithoutDelimiter(final boolean copyDefaultObjectWithoutDelimiter) {
784 this.copyDefaultObjectWithoutDelimiter = copyDefaultObjectWithoutDelimiter;
785 }
786
787 public String getCacheControl() {
788 return cacheControl;
789 }
790
791 public void setCacheControl(final String cacheControl) {
792 this.cacheControl = cacheControl;
793 }
794
795 public String getBrowseKey() {
796 return browseKey;
797 }
798
799 public void setBrowseKey(final String browseHtml) {
800 this.browseKey = browseHtml;
801 }
802
803
804
805
806 public String getOrganizationGroupId() {
807 return organizationGroupId;
808 }
809
810
811
812
813
814 public void setOrganizationGroupId(final String organizationGroupId) {
815 this.organizationGroupId = organizationGroupId;
816 }
817
818 public int getThreads() {
819 return threads;
820 }
821
822 public void setThreads(int threadCount) {
823 this.threads = threadCount;
824 }
825
826 public CannedAccessControlList getAcl() {
827 return acl;
828 }
829
830 public void setAcl(CannedAccessControlList acl) {
831 this.acl = acl;
832 }
833
834 public List<String> getOrgPomGavs() {
835 return orgPomGavs;
836 }
837
838 public void setOrgPomGavs(List<String> orgPomGavs) {
839 this.orgPomGavs = orgPomGavs;
840 }
841 }