1 package org.kuali.common.devops.jenkins.upgrade;
2
3 import static com.google.common.base.Optional.absent;
4 import static com.google.common.base.Stopwatch.createStarted;
5 import static com.google.common.collect.Iterables.concat;
6 import static com.google.common.collect.Lists.newArrayList;
7 import static java.lang.String.format;
8 import static org.apache.commons.lang.StringUtils.removeStart;
9 import static org.kuali.common.devops.archive.sweep.Functions.allParentsAsListSingle;
10 import static org.kuali.common.devops.archive.sweep.Functions.hostnameToKey;
11 import static org.kuali.common.devops.jenkins.upgrade.S3.sumFileObjects;
12 import static org.kuali.common.util.FormatUtils.getTime;
13 import static org.kuali.common.util.log.Loggers.newLogger;
14
15 import java.nio.file.Path;
16 import java.util.List;
17 import java.util.Set;
18
19 import org.kuali.common.aws.s3.S3Service;
20 import org.kuali.common.aws.s3.model.ListObjectsRequest;
21 import org.kuali.common.aws.s3.model.ObjectListing;
22 import org.kuali.common.aws.s3.model.ObjectSummary;
23 import org.kuali.common.core.build.ValidatingBuilder;
24 import org.slf4j.Logger;
25
26 import com.google.common.base.Optional;
27 import com.google.common.base.Stopwatch;
28 import com.google.common.base.Supplier;
29 import com.google.common.collect.ImmutableSet;
30
31 public class S3Scanner implements Supplier<Set<ObjectSummary>> {
32
33 private static final Logger logger = newLogger();
34
35 private final S3Service s3;
36 private final String bucket;
37 private final String hostname;
38 private final Path basedir;
39
40 @Override
41 public Set<ObjectSummary> get() {
42 Stopwatch sw = createStarted();
43 logger.info(format("scanning -> s3://%s/%s%s", bucket, hostnameToKey().apply(hostname), basedir));
44 String prefix = getBucketScanPrefix();
45 List<ObjectSummary> right = scanRight(prefix);
46 List<ObjectSummary> left = scanLeft();
47 Set<ObjectSummary> set = ImmutableSet.copyOf(concat(right, left));
48 logger.info(format("disk usage -> %s", sumFileObjects(set)));
49 logger.info(format("elapsed -> %s", getTime(sw)));
50 return set;
51 }
52
53 private List<ObjectSummary> scanRight(String prefix) {
54 return s3.getCompleteList(bucket, prefix);
55 }
56
57 private List<ObjectSummary> scanLeft() {
58 String prefix = getBucketScanPrefix();
59 List<String> dirs = allParentsAsListSingle().apply(prefix);
60 List<ObjectSummary> list = newArrayList();
61 for (String dir : dirs) {
62 String s3Key = removeStart(dir, "/") + "/";
63 Optional<ObjectSummary> summary = getObjectSummary(s3Key);
64 if (summary.isPresent()) {
65 list.add(summary.get());
66 }
67 }
68 return list;
69 }
70
71 private Optional<ObjectSummary> getObjectSummary(String key) {
72 ListObjectsRequest request = ListObjectsRequest.builder(bucket).withMax(1).withPrefix(key).build();
73 ObjectListing ol = s3.getObjectListing(request);
74 List<ObjectSummary> list = ol.getSummaries();
75 if (list.isEmpty()) {
76 return absent();
77 }
78 ObjectSummary summary = list.iterator().next();
79 if (summary.getKey().equals(key)) {
80 return Optional.of(summary);
81 } else {
82 return absent();
83 }
84 }
85
86 private String getBucketScanPrefix() {
87 return getHostnamePrefix() + basedir.toString();
88 }
89
90 private String getHostnamePrefix() {
91 return hostnameToKey().apply(hostname);
92 }
93
94 private S3Scanner(Builder builder) {
95 this.s3 = builder.s3;
96 this.bucket = builder.bucket;
97 this.hostname = builder.hostname;
98 this.basedir = builder.basedir;
99 }
100
101 public static Builder builder() {
102 return new Builder();
103 }
104
105 public static class Builder extends ValidatingBuilder<S3Scanner> {
106
107 private S3Service s3;
108 private String bucket;
109 private String hostname;
110 private Path basedir;
111
112 public Builder withS3(S3Service s3) {
113 this.s3 = s3;
114 return this;
115 }
116
117 public Builder withBucket(String bucket) {
118 this.bucket = bucket;
119 return this;
120 }
121
122 public Builder withHostname(String hostname) {
123 this.hostname = hostname;
124 return this;
125 }
126
127 public Builder withBasedir(Path basedir) {
128 this.basedir = basedir;
129 return this;
130 }
131
132 @Override
133 public S3Scanner build() {
134 return validate(new S3Scanner(this));
135 }
136 }
137
138 public S3Service getS3() {
139 return s3;
140 }
141
142 public String getBucket() {
143 return bucket;
144 }
145
146 public String getHostname() {
147 return hostname;
148 }
149
150 public Path getBasedir() {
151 return basedir;
152 }
153
154 }