001/**
002 * Copyright 2004-2014 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.kuali.common.aws.s3;
017
018import static com.amazonaws.services.s3.model.CannedAccessControlList.PublicRead;
019import static com.google.common.base.Optional.absent;
020import static com.google.common.base.Preconditions.checkArgument;
021import static com.google.common.base.Preconditions.checkNotNull;
022import static com.google.common.collect.Lists.newArrayList;
023import static org.apache.commons.io.FileUtils.openOutputStream;
024import static org.apache.commons.io.IOUtils.closeQuietly;
025import static org.kuali.common.aws.ec2.model.Regions.DEFAULT_REGION;
026import static org.kuali.common.core.base.Optionals.fromTrimToNull;
027import static org.kuali.common.util.Encodings.UTF8;
028import static org.kuali.common.util.Str.getUTF8Bytes;
029import static org.kuali.common.util.Str.getUTF8String;
030import static org.kuali.common.util.base.Exceptions.illegalState;
031import static org.kuali.common.util.base.Precondition.checkIsFile;
032import static org.kuali.common.util.base.Precondition.checkNotBlank;
033import static org.kuali.common.util.base.Precondition.checkNotNull;
034
035import java.io.ByteArrayInputStream;
036import java.io.ByteArrayOutputStream;
037import java.io.File;
038import java.io.IOException;
039import java.io.InputStream;
040import java.io.OutputStream;
041import java.util.List;
042import java.util.Map;
043
044import org.apache.commons.io.IOUtils;
045import org.kuali.common.aws.model.ImmutableAWSCredentials;
046import org.kuali.common.aws.s3.model.Bucket;
047import org.kuali.common.aws.s3.model.CopyObjectResult;
048import org.kuali.common.aws.s3.model.ListObjectsRequest;
049import org.kuali.common.aws.s3.model.ObjectListing;
050import org.kuali.common.aws.s3.model.ObjectMetadata;
051import org.kuali.common.aws.s3.model.ObjectSummary;
052import org.kuali.common.aws.s3.model.PutDirRequest;
053import org.kuali.common.aws.s3.model.PutFileRequest;
054import org.kuali.common.aws.s3.model.PutObjectResult;
055import org.kuali.common.core.build.ValidatingBuilder;
056import org.kuali.common.core.validate.annotation.IdiotProofImmutable;
057
058import com.amazonaws.auth.AWSCredentials;
059import com.amazonaws.regions.RegionUtils;
060import com.amazonaws.services.s3.AmazonS3Client;
061import com.amazonaws.services.s3.model.AmazonS3Exception;
062import com.amazonaws.services.s3.model.CannedAccessControlList;
063import com.amazonaws.services.s3.model.CopyObjectRequest;
064import com.amazonaws.services.s3.model.PutObjectRequest;
065import com.amazonaws.services.s3.model.S3Object;
066import com.amazonaws.services.s3.model.S3ObjectSummary;
067import com.google.common.base.Optional;
068import com.google.common.collect.ImmutableList;
069
070@IdiotProofImmutable
071public final class DefaultS3Service implements S3Service {
072
073        private static final int HTTP_NOT_FOUND = 404;
074        private static final byte[] EMPTY_BYTE_ARRAY = {};
075
076        private final ImmutableAWSCredentials credentials;
077        private final String region;
078        private final String directoryContentType;
079        private final CannedAccessControlList defaultPermissions;
080
081        // Mutable! Don't expose via a getter
082        private final AmazonS3Client client;
083
084        @Override
085        public PutObjectResult putDirectory(String bucket, String key) {
086                return putDirectory(PutDirRequest.builder().withBucket(bucket).withKey(key).build());
087        }
088
089        @Override
090        public PutObjectResult putDirectory(PutDirRequest request) {
091                // Setup permissions
092                CannedAccessControlList perms = request.getPermissions().isPresent() ? request.getPermissions().get() : this.defaultPermissions;
093
094                // Always zero bytes for an S3 "directory"
095                ByteArrayInputStream in = new ByteArrayInputStream(EMPTY_BYTE_ARRAY);
096
097                // Convert to a mutable metadata object
098                com.amazonaws.services.s3.model.ObjectMetadata mutable = getAmazonMetadata(request);
099
100                // Create a mutable request object
101                PutObjectRequest aws = new PutObjectRequest(request.getBucket(), request.getKey(), in, mutable);
102
103                // Store the perms
104                aws.setCannedAcl(perms);
105
106                // Make the AWS call, and copy the mutable result into an immutable object
107                return PutObjectResult.copyOf(client.putObject(aws));
108        }
109
110        @Override
111        public ObjectSummary getSummary(String bucket, String key) {
112                checkNotBlank(bucket, "bucket");
113                checkNotBlank(key, "key");
114                ListObjectsRequest request = ListObjectsRequest.builder(bucket).withPrefix(key).build();
115                ObjectListing listing = getObjectListing(request);
116                return listing.getSummaries().iterator().next();
117        }
118
119        @Override
120        public List<ObjectSummary> getCompleteList(final String bucket, final String prefix) {
121                checkNotBlank(bucket, "bucket");
122                checkNotBlank(prefix, "prefix");
123                List<ObjectSummary> summaries = newArrayList();
124                Optional<String> marker = absent();
125                boolean incomplete = true;
126                while (incomplete) {
127                        ListObjectsRequest request = ListObjectsRequest.builder(bucket).withPrefix(prefix).withMarker(marker).build();
128                        ObjectListing listing = getObjectListing(request);
129                        summaries.addAll(listing.getSummaries());
130                        incomplete = listing.isTruncated();
131                        marker = listing.getNextMarker();
132                }
133                return ImmutableList.copyOf(summaries);
134        }
135
136        @Override
137        public ObjectListing getObjectListing(ListObjectsRequest request) {
138                com.amazonaws.services.s3.model.ListObjectsRequest mutable = new com.amazonaws.services.s3.model.ListObjectsRequest();
139                mutable.setBucketName(request.getBucket());
140                mutable.setDelimiter(request.getDelimiter().orNull());
141                mutable.setEncodingType(request.getEncoding().orNull());
142                mutable.setMarker(request.getMarker().orNull());
143                mutable.setMaxKeys(request.getMax());
144                mutable.setPrefix(request.getPrefix().orNull());
145                return copyOf(client.listObjects(mutable));
146        }
147
148        private ObjectListing copyOf(com.amazonaws.services.s3.model.ObjectListing mutable) {
149                ObjectListing.Builder builder = ObjectListing.builder();
150                builder.withBucket(mutable.getBucketName());
151                builder.withCommonPrefixes(mutable.getCommonPrefixes());
152                builder.withDelimiter(fromTrimToNull(mutable.getDelimiter()));
153                builder.withEncoding(fromTrimToNull(mutable.getEncodingType()));
154                builder.withIsTruncated(mutable.isTruncated());
155                builder.withMarker(fromTrimToNull(mutable.getMarker()));
156                builder.withMaxKeys(mutable.getMaxKeys());
157                builder.withNextMarker(fromTrimToNull(mutable.getNextMarker()));
158                builder.withPrefix(fromTrimToNull(mutable.getPrefix()));
159                builder.withSummaries(getSummaries(mutable.getObjectSummaries()));
160                return builder.build();
161        }
162
163        private List<ObjectSummary> getSummaries(List<S3ObjectSummary> mutables) {
164                List<ObjectSummary> summaries = newArrayList();
165                for (S3ObjectSummary mutable : mutables) {
166                        summaries.add(ObjectSummary.copyOf(mutable));
167                }
168                return summaries;
169        }
170
171        @Override
172        public Bucket getOrCreateBucketOwnedByMe(String bucket) {
173                if (client.doesBucketExist(bucket)) {
174                        List<com.amazonaws.services.s3.model.Bucket> buckets = client.listBuckets();
175                        for (com.amazonaws.services.s3.model.Bucket b : buckets) {
176                                if (b.getName().equals(bucket)) {
177                                        return Bucket.copyOf(b);
178                                }
179                        }
180                        throw illegalState("bucket [%s] exists, but is not owned by the currently authenticated user", bucket);
181                } else {
182                        return Bucket.copyOf(client.createBucket(bucket, region));
183                }
184        }
185
186        @Override
187        public Bucket createBucket(String bucket) {
188                checkNotBlank(bucket, "bucket");
189                checkArgument(!client.doesBucketExist(bucket), "bucket [%s] already exists");
190                return Bucket.copyOf(client.createBucket(bucket, region));
191        }
192
193        /**
194         * Return metadata about the object stored in the bucket under the specified key
195         * 
196         * @throws IllegalArgumentException
197         *             If bucket or key are blank, the bucket does not exist, or this key does not exist in this bucket
198         * @throws AmazonS3Exception
199         *             If the bucket exists but permissions deny read access to the bucket or object
200         */
201        @Override
202        public ObjectMetadata getMetadata(String bucket, String key) {
203                // Blanks not allowed
204                checkNotBlank(bucket, "bucket");
205                checkNotBlank(key, "key");
206                Optional<ObjectMetadata> meta = getOptionalMetadata(bucket, key);
207                checkArgument(meta.isPresent(), "[%s] does not exist in bucket [%s]", key, bucket);
208                return meta.get();
209        }
210
211        @Override
212        public Optional<ObjectMetadata> getOptionalMetadata(String bucket, String key) {
213                // Blanks not allowed
214                checkNotBlank(bucket, "bucket");
215                checkNotBlank(key, "key");
216
217                // Throw an exception if they are asking about a key inside a bucket that doesn't exist
218                checkArgument(client.doesBucketExist(bucket), "bucket [%s] does not exist", bucket);
219
220                try {
221                        // Attempt to get metadata for this object in this bucket
222                        org.kuali.common.aws.s3.model.ObjectMetadata omd = org.kuali.common.aws.s3.model.ObjectMetadata.copyOf(client.getObjectMetadata(bucket, key));
223
224                        // If we successfully got metadata, the object definitely exists
225                        return Optional.of(omd);
226                } catch (AmazonS3Exception ase) {
227                        if (ase.getStatusCode() == HTTP_NOT_FOUND) {
228                                // Object definitely doesn't exist
229                                // NOTE: We can only safely return false here because we've already checked to make sure the bucket exists
230                                // Amazon also returns a 404 for buckets that don't exist
231                                return absent();
232                        } else {
233                                // Re-throw all other exceptions, as we now have no way to determine if the object exists.
234                                // A 403 permissions error could just mean we are being denied access to the bucket
235                                // It may, or may not, mean anything about the existence of the indicated key in the indicated bucket.
236                                // This is very similar to what the JDK does with file.exists()
237                                // The JDK throws SecurityException if a security manager exists and its SecurityManager.checkRead(String) method denies read access to the file or directory
238                                throw ase;
239                        }
240                }
241        }
242
243        @Override
244        public boolean exists(String bucket, String key) {
245                return getOptionalMetadata(bucket, key).isPresent();
246        }
247
248        @Override
249        public PutObjectResult putFile(String bucket, String key, File file) {
250                return putFile(bucket, key, file, UTF8);
251        }
252
253        @Override
254        public PutObjectResult putFile(String bucket, String key, File file, String encoding) {
255                PutFileRequest request = PutFileRequest.builder().withBucket(bucket).withKey(key).withFile(file).withEncoding(encoding).build();
256                return putFile(request);
257        }
258
259        /**
260         * Create a file on S3 from a regular file on the local file system
261         */
262        @Override
263        public PutObjectResult putFile(PutFileRequest request) {
264                checkNotNull(request);
265                checkIsFile(request.getFile(), "request.file");
266
267                // Setup metadata
268                com.amazonaws.services.s3.model.ObjectMetadata meta = getAmazonMetadata(request);
269
270                // Convert our immutable PutFileRequest into Amazon's mutable PutObjectRequest
271                PutObjectRequest put = getAmazonRequest(request, meta);
272
273                // Upload the file to S3 and store the return value in an immutable object
274                return PutObjectResult.copyOf(client.putObject(put));
275        }
276
277        private PutObjectRequest getAmazonRequest(PutFileRequest request, com.amazonaws.services.s3.model.ObjectMetadata meta) {
278
279                CannedAccessControlList perms = request.getPermissions().isPresent() ? request.getPermissions().get() : this.defaultPermissions;
280
281                // Create an AWS request object
282                PutObjectRequest put = new PutObjectRequest(request.getBucket(), request.getKey(), request.getFile());
283                put.setMetadata(meta);
284                put.setCannedAcl(perms);
285                return put;
286        }
287
288        private com.amazonaws.services.s3.model.ObjectMetadata getAmazonMetadata(PutFileRequest request) {
289                com.amazonaws.services.s3.model.ObjectMetadata meta = new com.amazonaws.services.s3.model.ObjectMetadata();
290                meta.setUserMetadata(request.getMetadata().getUserMetadata());
291                Map<String, Object> raw = request.getMetadata().getRawMetadata();
292                for (String key : raw.keySet()) {
293                        meta.setHeader(key, raw.get(key));
294                }
295                meta.setContentLength(request.getFile().length());
296                meta.setContentEncoding(request.getEncoding());
297                return meta;
298        }
299
300        private com.amazonaws.services.s3.model.ObjectMetadata getAmazonMetadata(PutDirRequest request) {
301                com.amazonaws.services.s3.model.ObjectMetadata meta = new com.amazonaws.services.s3.model.ObjectMetadata();
302                meta.setUserMetadata(request.getMetadata().getUserMetadata());
303                Map<String, Object> raw = request.getMetadata().getRawMetadata();
304                for (String key : raw.keySet()) {
305                        meta.setHeader(key, raw.get(key));
306                }
307                meta.setContentLength(0L);
308                meta.setContentType(directoryContentType);
309                meta.setContentEncoding(UTF8);
310                return meta;
311        }
312
313        @Override
314        public PutObjectResult writeStringToObject(String bucket, String key, String data) {
315                checkNotBlank(bucket, "bucket");
316                checkNotBlank(key, "key");
317                checkNotNull(data, "data"); // Prevent null, but allow pure whitespace or the empty string
318
319                // Convert the string into bytes using UTF8
320                byte[] bytes = getUTF8Bytes(data);
321
322                // Setup an input stream
323                ByteArrayInputStream in = new ByteArrayInputStream(bytes);
324
325                // Setup metadata
326                com.amazonaws.services.s3.model.ObjectMetadata meta = new com.amazonaws.services.s3.model.ObjectMetadata();
327                meta.setContentEncoding(UTF8);
328                meta.setContentLength(bytes.length);
329
330                // Create a request object with the permission PublicRead
331                PutObjectRequest request = new PutObjectRequest(bucket, key, in, meta);
332                request.setCannedAcl(this.defaultPermissions);
333
334                // Create the object
335                return PutObjectResult.copyOf(client.putObject(request));
336        }
337
338        @Override
339        public String readObjectToString(String bucket, String key) {
340                checkNotBlank(bucket, "bucket");
341                checkNotBlank(key, "key");
342                ByteArrayOutputStream out = new ByteArrayOutputStream();
343                copyObjectToStream(bucket, key, out);
344                return getUTF8String(out.toByteArray());
345        }
346
347        @Override
348        public void copyObjectToStream(String bucket, String key, OutputStream out) {
349                checkNotBlank(bucket, "bucket");
350                checkNotBlank(key, "key");
351                checkNotNull(out, "out");
352                S3Object object = client.getObject(bucket, key);
353                InputStream in = null;
354                try {
355                        in = object.getObjectContent();
356                        IOUtils.copy(in, out);
357                } catch (IOException e) {
358                        throw illegalState(e);
359                } finally {
360                        closeQuietly(in);
361                }
362        }
363
364        @Override
365        public File copyObjectToFile(String bucket, String key, File file) {
366                checkNotBlank(bucket, "bucket");
367                checkNotBlank(key, "key");
368                checkNotNull(file, "file");
369                OutputStream out = null;
370                try {
371                        out = openOutputStream(file);
372                        copyObjectToStream(bucket, key, out);
373                        return file;
374                } catch (IOException e) {
375                        throw illegalState(e);
376                } finally {
377                        closeQuietly(out);
378                }
379        }
380
381        /**
382         * Copy an object in a bucket to another name. Copies the original objects metadata and ACL.
383         */
384        @Override
385        public CopyObjectResult copyObject(String bucket, String srcKey, String dstKey) {
386                checkNotBlank(bucket, "bucket");
387                checkNotBlank(srcKey, "srcKey");
388                checkNotBlank(dstKey, "dstKey");
389                checkArgument(!srcKey.equals(dstKey), "srcKey cannot be the same as dstKey -> [%s] == [%s]", srcKey, dstKey);
390                CopyObjectRequest request = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
391                request.setAccessControlList(client.getObjectAcl(bucket, srcKey));
392                return CopyObjectResult.copyOf(client.copyObject(request));
393        }
394
395        private DefaultS3Service(Builder builder) {
396                this.credentials = ImmutableAWSCredentials.copyOf(builder.credentials);
397                this.region = builder.region;
398                this.defaultPermissions = builder.defaultPermissions;
399                this.directoryContentType = builder.directoryContentType;
400
401                // Setup the client
402                this.client = new AmazonS3Client(credentials);
403                this.client.setRegion(RegionUtils.getRegion(region));
404        }
405
406        public static DefaultS3Service build(AWSCredentials credentials) {
407                return builder(credentials).build();
408        }
409
410        public static Builder builder(AWSCredentials credentials) {
411                return new Builder(credentials);
412        }
413
414        public static class Builder extends ValidatingBuilder<DefaultS3Service> {
415
416                private final AWSCredentials credentials;
417                private String region = DEFAULT_REGION.getName();
418                private CannedAccessControlList defaultPermissions = PublicRead;
419                private String directoryContentType = "application/x-directory";
420
421                public Builder(AWSCredentials credentials) {
422                        this.credentials = credentials;
423                }
424
425                public Builder withRegion(String region) {
426                        this.region = region;
427                        return this;
428                }
429
430                public Builder withDirectoryContentType(String directoryContentType) {
431                        this.directoryContentType = directoryContentType;
432                        return this;
433                }
434
435                public Builder withDefaultPermissions(CannedAccessControlList defaultPermissions) {
436                        this.defaultPermissions = defaultPermissions;
437                        return this;
438                }
439
440                @Override
441                public DefaultS3Service build() {
442                        return validate(new DefaultS3Service(this));
443                }
444        }
445
446        public AWSCredentials getCredentials() {
447                return credentials;
448        }
449
450        @Override
451        public String getRegion() {
452                return region;
453        }
454
455        public CannedAccessControlList getDefaultPermissions() {
456                return defaultPermissions;
457        }
458
459        public String getDirectoryContentType() {
460                return directoryContentType;
461        }
462
463}