001package org.kuali.common.devops.jenkins.upgrade;
002
003import static com.google.common.base.Optional.absent;
004import static com.google.common.base.Stopwatch.createStarted;
005import static com.google.common.collect.Iterables.concat;
006import static com.google.common.collect.Lists.newArrayList;
007import static java.lang.String.format;
008import static org.apache.commons.lang.StringUtils.removeStart;
009import static org.kuali.common.devops.archive.sweep.Functions.allParentsAsListSingle;
010import static org.kuali.common.devops.archive.sweep.Functions.hostnameToKey;
011import static org.kuali.common.devops.jenkins.upgrade.S3.sumFileObjects;
012import static org.kuali.common.util.FormatUtils.getTime;
013import static org.kuali.common.util.log.Loggers.newLogger;
014
015import java.nio.file.Path;
016import java.util.List;
017import java.util.Set;
018
019import org.kuali.common.aws.s3.S3Service;
020import org.kuali.common.aws.s3.model.ListObjectsRequest;
021import org.kuali.common.aws.s3.model.ObjectListing;
022import org.kuali.common.aws.s3.model.ObjectSummary;
023import org.kuali.common.core.build.ValidatingBuilder;
024import org.slf4j.Logger;
025
026import com.google.common.base.Optional;
027import com.google.common.base.Stopwatch;
028import com.google.common.base.Supplier;
029import com.google.common.collect.ImmutableSet;
030
031public class S3Scanner implements Supplier<Set<ObjectSummary>> {
032
033        private static final Logger logger = newLogger();
034
035        private final S3Service s3;
036        private final String bucket;
037        private final String hostname;
038        private final Path basedir;
039
040        @Override
041        public Set<ObjectSummary> get() {
042                Stopwatch sw = createStarted();
043                logger.info(format("scanning   -> s3://%s/%s%s", bucket, hostnameToKey().apply(hostname), basedir));
044                String prefix = getBucketScanPrefix();
045                List<ObjectSummary> right = scanRight(prefix);
046                List<ObjectSummary> left = scanLeft();
047                Set<ObjectSummary> set = ImmutableSet.copyOf(concat(right, left));
048                logger.info(format("disk usage -> %s", sumFileObjects(set)));
049                logger.info(format("elapsed    -> %s", getTime(sw)));
050                return set;
051        }
052
053        private List<ObjectSummary> scanRight(String prefix) {
054                return s3.getCompleteList(bucket, prefix);
055        }
056
057        private List<ObjectSummary> scanLeft() {
058                String prefix = getBucketScanPrefix();
059                List<String> dirs = allParentsAsListSingle().apply(prefix);
060                List<ObjectSummary> list = newArrayList();
061                for (String dir : dirs) {
062                        String s3Key = removeStart(dir, "/") + "/";
063                        Optional<ObjectSummary> summary = getObjectSummary(s3Key);
064                        if (summary.isPresent()) {
065                                list.add(summary.get());
066                        }
067                }
068                return list;
069        }
070
071        private Optional<ObjectSummary> getObjectSummary(String key) {
072                ListObjectsRequest request = ListObjectsRequest.builder(bucket).withMax(1).withPrefix(key).build();
073                ObjectListing ol = s3.getObjectListing(request);
074                List<ObjectSummary> list = ol.getSummaries();
075                if (list.isEmpty()) {
076                        return absent();
077                }
078                ObjectSummary summary = list.iterator().next();
079                if (summary.getKey().equals(key)) {
080                        return Optional.of(summary);
081                } else {
082                        return absent();
083                }
084        }
085
086        private String getBucketScanPrefix() {
087                return getHostnamePrefix() + basedir.toString();
088        }
089
090        private String getHostnamePrefix() {
091                return hostnameToKey().apply(hostname);
092        }
093
094        private S3Scanner(Builder builder) {
095                this.s3 = builder.s3;
096                this.bucket = builder.bucket;
097                this.hostname = builder.hostname;
098                this.basedir = builder.basedir;
099        }
100
101        public static Builder builder() {
102                return new Builder();
103        }
104
105        public static class Builder extends ValidatingBuilder<S3Scanner> {
106
107                private S3Service s3;
108                private String bucket;
109                private String hostname;
110                private Path basedir;
111
112                public Builder withS3(S3Service s3) {
113                        this.s3 = s3;
114                        return this;
115                }
116
117                public Builder withBucket(String bucket) {
118                        this.bucket = bucket;
119                        return this;
120                }
121
122                public Builder withHostname(String hostname) {
123                        this.hostname = hostname;
124                        return this;
125                }
126
127                public Builder withBasedir(Path basedir) {
128                        this.basedir = basedir;
129                        return this;
130                }
131
132                @Override
133                public S3Scanner build() {
134                        return validate(new S3Scanner(this));
135                }
136        }
137
138        public S3Service getS3() {
139                return s3;
140        }
141
142        public String getBucket() {
143                return bucket;
144        }
145
146        public String getHostname() {
147                return hostname;
148        }
149
150        public Path getBasedir() {
151                return basedir;
152        }
153
154}