View Javadoc
1   package org.kuali.common.devops.archive.s3;
2   
3   import static com.amazonaws.services.s3.Headers.CONTENT_TYPE;
4   import static com.google.common.collect.Lists.newArrayList;
5   import static com.google.common.collect.Maps.newHashMap;
6   import static com.google.common.collect.Sets.difference;
7   import static java.lang.String.format;
8   import static org.kuali.common.core.io.Files.md5;
9   import static org.kuali.common.core.io.Paths.fromFile;
10  import static org.kuali.common.util.log.Loggers.newLogger;
11  
12  import java.io.File;
13  import java.util.List;
14  import java.util.Map;
15  import java.util.Set;
16  import java.util.concurrent.Callable;
17  
18  import org.kuali.common.aws.s3.S3Service;
19  import org.kuali.common.aws.s3.model.ObjectMetadata;
20  import org.kuali.common.aws.s3.model.PutFileRequest;
21  import org.kuali.common.aws.s3.model.PutObjectResult;
22  import org.kuali.common.core.build.ValidatingBuilder;
23  import org.kuali.common.core.validate.annotation.IdiotProofImmutable;
24  import org.kuali.common.devops.archive.ArchivedFileMetadata;
25  import org.kuali.common.devops.archive.sas.MetadataProvider;
26  import org.slf4j.Logger;
27  
28  import com.google.common.base.Optional;
29  import com.google.common.collect.ImmutableList;
30  
31  @IdiotProofImmutable
32  public final class PutS3FileCallable implements Callable<List<PutS3FileCallableResult>> {
33  
34  	private static final Logger logger = newLogger();
35  
36  	private static final String EMPTY_MD5_HASH = "68b329da9893e34099c7d8ad5cb9c940";
37  
38  	private final String bucket;
39  	private final S3Service s3;
40  	private final ImmutableList<KeyFile> pairs;
41  	private final MetadataProvider provider;
42  
43  	@Override
44  	public List<PutS3FileCallableResult> call() {
45  		List<PutS3FileCallableResult> list = newArrayList();
46  		for (KeyFile pair : pairs) {
47  			boolean skip = isSkip(pair);
48  			if (!skip) {
49  				PutFileRequest request = getPutFileRequest(pair, bucket);
50  				logger.debug(format("create -> %s", pair.getKey()));
51  				PutObjectResult result = s3.putFile(request);
52  				PutS3FileCallableResult element = PutS3FileCallableResult.builder().withPair(pair).withResult(result).build();
53  				list.add(element);
54  			} else {
55  				logger.debug(format("skip   -> %s", pair.getKey()));
56  			}
57  		}
58  		return ImmutableList.copyOf(list);
59  	}
60  
61  	private boolean isSkip(KeyFile pair) {
62  
63  		// Check to see if there is metadata for this key
64  		Optional<ObjectMetadata> optional = s3.getOptionalMetadata(bucket, pair.getKey());
65  
66  		// if there is no metadata there is no archived file
67  		if (!optional.isPresent()) {
68  			return false;
69  		}
70  
71  		// extract the metadata
72  		ObjectMetadata meta = optional.get();
73  
74  		// make sure all of the required metadata keys are present
75  		Set<String> missingKeys = difference(MetadataConverter.KEYS, meta.getUserMetadata().keySet());
76  
77  		if (!missingKeys.isEmpty()) {
78  			return false;
79  		}
80  
81  		// If the file we are uploading is a directory and all of the metadata is there, we are good to go
82  		if (pair.getFile().isDirectory()) {
83  			return true;
84  		}
85  
86  		// compare the metadata S3 is storing for this key with metadata calculated from the file on the file system
87  		ArchivedFileMetadata afd = MetadataConverter.INSTANCE.reverse().convert(meta.getUserMetadata());
88  
89  		// if the sizes are different we must re-upload
90  		if (afd.getSize() != pair.getFile().length()) {
91  			return false;
92  		}
93  
94  		// Both last modified and size are identical, it's ok to skip uploading this file
95  		if (afd.getLastModified() == pair.getFile().lastModified()) {
96  			return true;
97  		}
98  
99  		// length is the same, but last modified is different
100 		// don't skip unless the md5 checksums match
101 		return md5(pair.getFile()).equals(afd.getMd5());
102 	}
103 
104 	private PutFileRequest getPutFileRequest(KeyFile pair, String bucket) {
105 		File file = pair.getFile();
106 		long size = file.isDirectory() ? 0L : file.length();
107 		long lastModified = file.lastModified();
108 		String md5 = file.isDirectory() ? EMPTY_MD5_HASH : md5(file);
109 		ArchivedFileMetadata meta = ArchivedFileMetadata.builder().withLastModified(lastModified).withMd5(md5).withSize(size).build();
110 		Map<String, String> userMetadata = newHashMap(MetadataConverter.INSTANCE.convert(meta));
111 		userMetadata.putAll(provider.getUserMetadata(fromFile(pair.getFile())));
112 		Map<String, Object> raw = newHashMap();
113 		// TODO Provide this somewhere else, maybe make a MetadataProviderProvider, that chains together one or more providers
114 		if (pair.getFile().getName().equals("log")) {
115 			raw.put(CONTENT_TYPE, "text/plain");
116 		}
117 		if (pair.getFile().getName().endsWith(".log")) {
118 			raw.put(CONTENT_TYPE, "text/plain");
119 		}
120 		if (pair.getFile().getName().endsWith(".out")) {
121 			raw.put(CONTENT_TYPE, "text/plain");
122 		}
123 		ObjectMetadata omd = ObjectMetadata.builder().withUserMetadata(userMetadata).withRawMetadata(raw).build();
124 		return PutFileRequest.builder().withFile(file).withBucket(bucket).withKey(pair.getKey()).withMetadata(omd).build();
125 	}
126 
127 	private PutS3FileCallable(Builder builder) {
128 		this.s3 = builder.s3;
129 		this.pairs = ImmutableList.copyOf(builder.pairs);
130 		this.bucket = builder.bucket;
131 		this.provider = builder.provider;
132 	}
133 
134 	public static Builder builder() {
135 		return new Builder();
136 	}
137 
138 	public static class Builder extends ValidatingBuilder<PutS3FileCallable> {
139 
140 		private String bucket;
141 		private S3Service s3;
142 		private List<KeyFile> pairs;
143 		private MetadataProvider provider = EmptyMetadataProvider.INSTANCE;
144 
145 		public Builder withS3(S3Service s3) {
146 			this.s3 = s3;
147 			return this;
148 		}
149 
150 		public Builder withProvider(MetadataProvider provider) {
151 			this.provider = provider;
152 			return this;
153 		}
154 
155 		public Builder withPairs(List<KeyFile> pairs) {
156 			this.pairs = pairs;
157 			return this;
158 		}
159 
160 		public Builder withBucket(String bucket) {
161 			this.bucket = bucket;
162 			return this;
163 		}
164 
165 		@Override
166 		public PutS3FileCallable build() {
167 			return validate(new PutS3FileCallable(this));
168 		}
169 	}
170 
171 	public S3Service getS3() {
172 		return s3;
173 	}
174 
175 	public List<KeyFile> getPairs() {
176 		return pairs;
177 	}
178 
179 	public String getBucket() {
180 		return bucket;
181 	}
182 
183 	public MetadataProvider getProvider() {
184 		return provider;
185 	}
186 
187 }