001package org.kuali.common.devops.archive.s3;
002
003import static com.amazonaws.services.s3.Headers.CONTENT_TYPE;
004import static com.google.common.collect.Lists.newArrayList;
005import static com.google.common.collect.Maps.newHashMap;
006import static com.google.common.collect.Sets.difference;
007import static java.lang.String.format;
008import static org.kuali.common.core.io.Files.md5;
009import static org.kuali.common.core.io.Paths.fromFile;
010import static org.kuali.common.util.log.Loggers.newLogger;
011
012import java.io.File;
013import java.util.List;
014import java.util.Map;
015import java.util.Set;
016import java.util.concurrent.Callable;
017
018import org.kuali.common.aws.s3.S3Service;
019import org.kuali.common.aws.s3.model.ObjectMetadata;
020import org.kuali.common.aws.s3.model.PutFileRequest;
021import org.kuali.common.aws.s3.model.PutObjectResult;
022import org.kuali.common.core.build.ValidatingBuilder;
023import org.kuali.common.core.validate.annotation.IdiotProofImmutable;
024import org.kuali.common.devops.archive.ArchivedFileMetadata;
025import org.kuali.common.devops.archive.sas.MetadataProvider;
026import org.slf4j.Logger;
027
028import com.google.common.base.Optional;
029import com.google.common.collect.ImmutableList;
030
031@IdiotProofImmutable
032public final class PutS3FileCallable implements Callable<List<PutS3FileCallableResult>> {
033
034        private static final Logger logger = newLogger();
035
036        private static final String EMPTY_MD5_HASH = "68b329da9893e34099c7d8ad5cb9c940";
037
038        private final String bucket;
039        private final S3Service s3;
040        private final ImmutableList<KeyFile> pairs;
041        private final MetadataProvider provider;
042
043        @Override
044        public List<PutS3FileCallableResult> call() {
045                List<PutS3FileCallableResult> list = newArrayList();
046                for (KeyFile pair : pairs) {
047                        boolean skip = isSkip(pair);
048                        if (!skip) {
049                                PutFileRequest request = getPutFileRequest(pair, bucket);
050                                logger.debug(format("create -> %s", pair.getKey()));
051                                PutObjectResult result = s3.putFile(request);
052                                PutS3FileCallableResult element = PutS3FileCallableResult.builder().withPair(pair).withResult(result).build();
053                                list.add(element);
054                        } else {
055                                logger.debug(format("skip   -> %s", pair.getKey()));
056                        }
057                }
058                return ImmutableList.copyOf(list);
059        }
060
061        private boolean isSkip(KeyFile pair) {
062
063                // Check to see if there is metadata for this key
064                Optional<ObjectMetadata> optional = s3.getOptionalMetadata(bucket, pair.getKey());
065
066                // if there is no metadata there is no archived file
067                if (!optional.isPresent()) {
068                        return false;
069                }
070
071                // extract the metadata
072                ObjectMetadata meta = optional.get();
073
074                // make sure all of the required metadata keys are present
075                Set<String> missingKeys = difference(MetadataConverter.KEYS, meta.getUserMetadata().keySet());
076
077                if (!missingKeys.isEmpty()) {
078                        return false;
079                }
080
081                // If the file we are uploading is a directory and all of the metadata is there, we are good to go
082                if (pair.getFile().isDirectory()) {
083                        return true;
084                }
085
086                // compare the metadata S3 is storing for this key with metadata calculated from the file on the file system
087                ArchivedFileMetadata afd = MetadataConverter.INSTANCE.reverse().convert(meta.getUserMetadata());
088
089                // if the sizes are different we must re-upload
090                if (afd.getSize() != pair.getFile().length()) {
091                        return false;
092                }
093
094                // Both last modified and size are identical, it's ok to skip uploading this file
095                if (afd.getLastModified() == pair.getFile().lastModified()) {
096                        return true;
097                }
098
099                // length is the same, but last modified is different
100                // don't skip unless the md5 checksums match
101                return md5(pair.getFile()).equals(afd.getMd5());
102        }
103
104        private PutFileRequest getPutFileRequest(KeyFile pair, String bucket) {
105                File file = pair.getFile();
106                long size = file.isDirectory() ? 0L : file.length();
107                long lastModified = file.lastModified();
108                String md5 = file.isDirectory() ? EMPTY_MD5_HASH : md5(file);
109                ArchivedFileMetadata meta = ArchivedFileMetadata.builder().withLastModified(lastModified).withMd5(md5).withSize(size).build();
110                Map<String, String> userMetadata = newHashMap(MetadataConverter.INSTANCE.convert(meta));
111                userMetadata.putAll(provider.getUserMetadata(fromFile(pair.getFile())));
112                Map<String, Object> raw = newHashMap();
113                // TODO Provide this somewhere else, maybe make a MetadataProviderProvider, that chains together one or more providers
114                if (pair.getFile().getName().equals("log")) {
115                        raw.put(CONTENT_TYPE, "text/plain");
116                }
117                if (pair.getFile().getName().endsWith(".log")) {
118                        raw.put(CONTENT_TYPE, "text/plain");
119                }
120                if (pair.getFile().getName().endsWith(".out")) {
121                        raw.put(CONTENT_TYPE, "text/plain");
122                }
123                ObjectMetadata omd = ObjectMetadata.builder().withUserMetadata(userMetadata).withRawMetadata(raw).build();
124                return PutFileRequest.builder().withFile(file).withBucket(bucket).withKey(pair.getKey()).withMetadata(omd).build();
125        }
126
127        private PutS3FileCallable(Builder builder) {
128                this.s3 = builder.s3;
129                this.pairs = ImmutableList.copyOf(builder.pairs);
130                this.bucket = builder.bucket;
131                this.provider = builder.provider;
132        }
133
134        public static Builder builder() {
135                return new Builder();
136        }
137
138        public static class Builder extends ValidatingBuilder<PutS3FileCallable> {
139
140                private String bucket;
141                private S3Service s3;
142                private List<KeyFile> pairs;
143                private MetadataProvider provider = EmptyMetadataProvider.INSTANCE;
144
145                public Builder withS3(S3Service s3) {
146                        this.s3 = s3;
147                        return this;
148                }
149
150                public Builder withProvider(MetadataProvider provider) {
151                        this.provider = provider;
152                        return this;
153                }
154
155                public Builder withPairs(List<KeyFile> pairs) {
156                        this.pairs = pairs;
157                        return this;
158                }
159
160                public Builder withBucket(String bucket) {
161                        this.bucket = bucket;
162                        return this;
163                }
164
165                @Override
166                public PutS3FileCallable build() {
167                        return validate(new PutS3FileCallable(this));
168                }
169        }
170
171        public S3Service getS3() {
172                return s3;
173        }
174
175        public List<KeyFile> getPairs() {
176                return pairs;
177        }
178
179        public String getBucket() {
180                return bucket;
181        }
182
183        public MetadataProvider getProvider() {
184                return provider;
185        }
186
187}