1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.kuali.common.aws.s3.old;
17
18 import java.util.ArrayList;
19 import java.util.List;
20
21 import org.apache.commons.beanutils.BeanUtils;
22 import org.apache.commons.lang.StringUtils;
23 import org.kuali.common.util.Assert;
24 import org.kuali.common.util.CollectionUtils;
25 import org.kuali.common.util.Counter;
26 import org.slf4j.Logger;
27 import org.slf4j.LoggerFactory;
28
29 import com.amazonaws.services.s3.AmazonS3Client;
30 import com.amazonaws.services.s3.model.ListObjectsRequest;
31 import com.amazonaws.services.s3.model.ObjectListing;
32
33 public class DefaultBucketService implements BucketService {
34
35 private static final Logger logger = LoggerFactory.getLogger(DefaultBucketService.class);
36
37 @Override
38 public ListingResult getObjectListings(ObjectListingsContext context) {
39
40
41 Assert.notNull(context, "context is null");
42 Assert.notNull(context.getClient(), "client is null");
43 Assert.notNull(context.getRequest(), "request is null");
44 Assert.notNull(context.getBucketContext(), "bucket context is null");
45
46
47 AmazonS3Client client = context.getClient();
48 BucketContext bucketContext = context.getBucketContext();
49 ListingRequest request = context.getRequest();
50
51
52 Assert.hasText(bucketContext.getDelimiter(), "delimiter has no text");
53 Assert.hasText(bucketContext.getName(), "name has no text");
54 boolean exists = client.doesBucketExist(bucketContext.getName());
55 Assert.isTrue(exists, "bucket [" + context.getBucketContext().getName() + "] does not exist");
56
57
58 if (request.getInformer() != null) {
59 logger.debug("starting informer");
60 request.getInformer().start();
61 }
62
63
64 long start = System.currentTimeMillis();
65
66
67 Counter counter = new Counter();
68
69
70
71 List<ObjectListing> listings = accumulateObjectListings(context, context.getRequest(), start, counter);
72
73
74 long stop = System.currentTimeMillis();
75
76
77 if (request.getInformer() != null) {
78 request.getInformer().stop();
79 }
80
81
82 ListingResult result = new ListingResult();
83 result.setListings(listings);
84 result.setStartTime(start);
85 result.setStopTime(stop);
86 result.setElapsed(stop - start);
87 return result;
88 }
89
90
91
92
93 protected List<ObjectListing> accumulateObjectListings(ObjectListingsContext context, ListingRequest request, long startTime, Counter counter) {
94
95
96 validateState(request, startTime, counter);
97
98
99 String prefix = getPrefix(request.getPrefix(), context.getBucketContext().getDelimiter());
100
101
102 List<ObjectListing> listings = new ArrayList<ObjectListing>();
103
104
105 ObjectListing listing = getObjectListing(context, prefix, counter);
106
107
108 listings.add(listing);
109
110
111 for (String subDirectory : listing.getCommonPrefixes()) {
112 doSubDirectory(context, subDirectory, listings, startTime, counter);
113 }
114
115
116 return listings;
117 }
118
119
120
121
122 protected void validateState(ListingRequest request, long startTime, Counter counter) {
123
124
125 if (System.currentTimeMillis() > (startTime + request.getTimeoutMillis())) {
126 throw new IllegalStateException("timeout exceeded");
127 }
128
129
130 if (counter.getValue() > request.getMaxListings()) {
131 throw new IllegalStateException("max listings exceeded");
132 }
133 }
134
135 protected ObjectListing getObjectListing(ObjectListingsContext context, String prefix, Counter counter) {
136
137
138 ListObjectsRequest lor = getListObjectsRequest(context, prefix);
139
140
141 ObjectListing listing = context.getClient().listObjects(lor);
142
143
144 Assert.isFalse(listing.isTruncated(), "listing is truncated");
145
146
147 if (context.getRequest().getInformer() != null) {
148 context.getRequest().getInformer().incrementProgress();
149 }
150
151
152 counter.increment();
153
154
155 return listing;
156 }
157
158 protected void doSubDirectory(ObjectListingsContext context, String subDirectory, List<ObjectListing> listings, long startTime, Counter counter) {
159
160
161 if (isRecurse(context, subDirectory)) {
162
163
164 ListingRequest clone = clone(context.getRequest(), subDirectory);
165
166
167 List<ObjectListing> children = accumulateObjectListings(context, clone, startTime, counter);
168
169
170 listings.addAll(children);
171
172 } else {
173
174
175 ObjectListing subDirectoryListing = getObjectListing(context, subDirectory, counter);
176
177
178 listings.add(subDirectoryListing);
179 }
180 }
181
182
183
184
185 protected String getPrefix(String prefix, String delimiter) {
186 if (StringUtils.isBlank(prefix) || StringUtils.equals(prefix, delimiter)) {
187 return null;
188 } else {
189 return StringUtils.endsWith(prefix, delimiter) ? prefix : prefix + delimiter;
190 }
191 }
192
193
194
195
196
197
198
199
200
201
202
203 protected String getSuffixPattern(String pattern, String delimiter) {
204
205 Assert.hasText(pattern, "pattern has no text");
206 Assert.hasText(delimiter, "delimiter has no text");
207
208 StringBuilder sb = new StringBuilder();
209 if (!StringUtils.startsWith(pattern, delimiter)) {
210 sb.append(delimiter);
211 }
212 sb.append(pattern);
213 if (!StringUtils.endsWith(pattern, delimiter)) {
214 sb.append(delimiter);
215 }
216 return sb.toString();
217 }
218
219 protected boolean isEndsWithMatch(String prefix, String pattern, String delimiter) {
220 String suffix = getSuffixPattern(pattern, delimiter);
221 return StringUtils.endsWith(prefix, suffix);
222 }
223
224 protected boolean isExclude(ObjectListingsContext context, String prefix) {
225 for (String exclude : CollectionUtils.toEmptyList(context.getRequest().getExcludes())) {
226 if (isEndsWithMatch(prefix, exclude, context.getBucketContext().getDelimiter())) {
227 return true;
228 }
229 }
230 return false;
231 }
232
233 protected boolean isInclude(ObjectListingsContext context, String prefix) {
234 if (CollectionUtils.isEmpty(context.getRequest().getIncludes())) {
235 return true;
236 }
237 for (String include : context.getRequest().getIncludes()) {
238 if (isEndsWithMatch(prefix, include, context.getBucketContext().getDelimiter())) {
239 return true;
240 }
241 }
242 return false;
243 }
244
245 protected boolean isRecurse(ObjectListingsContext context, String prefix) {
246 return context.getRequest().isRecursive() && !isExclude(context, prefix) && isInclude(context, prefix);
247 }
248
249 protected ListingRequest clone(ListingRequest request, String prefix) {
250 ListingRequest clone = new ListingRequest();
251 try {
252 BeanUtils.copyProperties(clone, request);
253 } catch (Exception e) {
254 throw new IllegalStateException(e);
255 }
256 clone.setPrefix(prefix);
257 return clone;
258 }
259
260 protected ListObjectsRequest getListObjectsRequest(ObjectListingsContext context, String prefix) {
261 String name = context.getBucketContext().getName();
262 String delimiter = context.getBucketContext().getDelimiter();
263 Integer maxKeys = context.getBucketContext().getMaxKeys();
264 return getListObjectsRequest(name, prefix, delimiter, maxKeys);
265 }
266
267 protected ListObjectsRequest getListObjectsRequest(String bucket, String prefix, String delimiter, Integer maxKeys) {
268 ListObjectsRequest request = new ListObjectsRequest();
269 request.setBucketName(bucket);
270 request.setDelimiter(delimiter);
271 request.setPrefix(prefix);
272 request.setMaxKeys(maxKeys);
273 return request;
274 }
275
276 }