1 package org.kuali.common.devops.archive.s3;
2
3 import static com.amazonaws.services.s3.Headers.CONTENT_TYPE;
4 import static com.google.common.collect.Lists.newArrayList;
5 import static com.google.common.collect.Maps.newHashMap;
6 import static com.google.common.collect.Sets.difference;
7 import static java.lang.String.format;
8 import static org.kuali.common.core.io.Files.md5;
9 import static org.kuali.common.core.io.Paths.fromFile;
10 import static org.kuali.common.util.log.Loggers.newLogger;
11
12 import java.io.File;
13 import java.util.List;
14 import java.util.Map;
15 import java.util.Set;
16 import java.util.concurrent.Callable;
17
18 import org.kuali.common.aws.s3.S3Service;
19 import org.kuali.common.aws.s3.model.ObjectMetadata;
20 import org.kuali.common.aws.s3.model.PutFileRequest;
21 import org.kuali.common.aws.s3.model.PutObjectResult;
22 import org.kuali.common.core.build.ValidatingBuilder;
23 import org.kuali.common.core.validate.annotation.IdiotProofImmutable;
24 import org.kuali.common.devops.archive.ArchivedFileMetadata;
25 import org.kuali.common.devops.archive.sas.MetadataProvider;
26 import org.slf4j.Logger;
27
28 import com.google.common.base.Optional;
29 import com.google.common.collect.ImmutableList;
30
31 @IdiotProofImmutable
32 public final class PutS3FileCallable implements Callable<List<PutS3FileCallableResult>> {
33
34 private static final Logger logger = newLogger();
35
36 private static final String EMPTY_MD5_HASH = "68b329da9893e34099c7d8ad5cb9c940";
37
38 private final String bucket;
39 private final S3Service s3;
40 private final ImmutableList<KeyFile> pairs;
41 private final MetadataProvider provider;
42
43 @Override
44 public List<PutS3FileCallableResult> call() {
45 List<PutS3FileCallableResult> list = newArrayList();
46 for (KeyFile pair : pairs) {
47 boolean skip = isSkip(pair);
48 if (!skip) {
49 PutFileRequest request = getPutFileRequest(pair, bucket);
50 logger.debug(format("create -> %s", pair.getKey()));
51 PutObjectResult result = s3.putFile(request);
52 PutS3FileCallableResult element = PutS3FileCallableResult.builder().withPair(pair).withResult(result).build();
53 list.add(element);
54 } else {
55 logger.debug(format("skip -> %s", pair.getKey()));
56 }
57 }
58 return ImmutableList.copyOf(list);
59 }
60
61 private boolean isSkip(KeyFile pair) {
62
63
64 Optional<ObjectMetadata> optional = s3.getOptionalMetadata(bucket, pair.getKey());
65
66
67 if (!optional.isPresent()) {
68 return false;
69 }
70
71
72 ObjectMetadata meta = optional.get();
73
74
75 Set<String> missingKeys = difference(MetadataConverter.KEYS, meta.getUserMetadata().keySet());
76
77 if (!missingKeys.isEmpty()) {
78 return false;
79 }
80
81
82 if (pair.getFile().isDirectory()) {
83 return true;
84 }
85
86
87 ArchivedFileMetadata afd = MetadataConverter.INSTANCE.reverse().convert(meta.getUserMetadata());
88
89
90 if (afd.getSize() != pair.getFile().length()) {
91 return false;
92 }
93
94
95 if (afd.getLastModified() == pair.getFile().lastModified()) {
96 return true;
97 }
98
99
100
101 return md5(pair.getFile()).equals(afd.getMd5());
102 }
103
104 private PutFileRequest getPutFileRequest(KeyFile pair, String bucket) {
105 File file = pair.getFile();
106 long size = file.isDirectory() ? 0L : file.length();
107 long lastModified = file.lastModified();
108 String md5 = file.isDirectory() ? EMPTY_MD5_HASH : md5(file);
109 ArchivedFileMetadata meta = ArchivedFileMetadata.builder().withLastModified(lastModified).withMd5(md5).withSize(size).build();
110 Map<String, String> userMetadata = newHashMap(MetadataConverter.INSTANCE.convert(meta));
111 userMetadata.putAll(provider.getUserMetadata(fromFile(pair.getFile())));
112 Map<String, Object> raw = newHashMap();
113
114 if (pair.getFile().getName().equals("log")) {
115 raw.put(CONTENT_TYPE, "text/plain");
116 }
117 if (pair.getFile().getName().endsWith(".log")) {
118 raw.put(CONTENT_TYPE, "text/plain");
119 }
120 if (pair.getFile().getName().endsWith(".out")) {
121 raw.put(CONTENT_TYPE, "text/plain");
122 }
123 ObjectMetadata omd = ObjectMetadata.builder().withUserMetadata(userMetadata).withRawMetadata(raw).build();
124 return PutFileRequest.builder().withFile(file).withBucket(bucket).withKey(pair.getKey()).withMetadata(omd).build();
125 }
126
127 private PutS3FileCallable(Builder builder) {
128 this.s3 = builder.s3;
129 this.pairs = ImmutableList.copyOf(builder.pairs);
130 this.bucket = builder.bucket;
131 this.provider = builder.provider;
132 }
133
134 public static Builder builder() {
135 return new Builder();
136 }
137
138 public static class Builder extends ValidatingBuilder<PutS3FileCallable> {
139
140 private String bucket;
141 private S3Service s3;
142 private List<KeyFile> pairs;
143 private MetadataProvider provider = EmptyMetadataProvider.INSTANCE;
144
145 public Builder withS3(S3Service s3) {
146 this.s3 = s3;
147 return this;
148 }
149
150 public Builder withProvider(MetadataProvider provider) {
151 this.provider = provider;
152 return this;
153 }
154
155 public Builder withPairs(List<KeyFile> pairs) {
156 this.pairs = pairs;
157 return this;
158 }
159
160 public Builder withBucket(String bucket) {
161 this.bucket = bucket;
162 return this;
163 }
164
165 @Override
166 public PutS3FileCallable build() {
167 return validate(new PutS3FileCallable(this));
168 }
169 }
170
171 public S3Service getS3() {
172 return s3;
173 }
174
175 public List<KeyFile> getPairs() {
176 return pairs;
177 }
178
179 public String getBucket() {
180 return bucket;
181 }
182
183 public MetadataProvider getProvider() {
184 return provider;
185 }
186
187 }