1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.kuali.common.aws.s3;
17
18 import java.util.ArrayList;
19 import java.util.Collections;
20 import java.util.List;
21
22 import javax.swing.tree.DefaultMutableTreeNode;
23
24 import org.kuali.common.aws.s3.monitor.BucketPrefixSummaryHandler;
25 import org.kuali.common.aws.s3.monitor.S3Utils;
26 import org.kuali.common.aws.s3.pojo.BucketPrefixSummary;
27 import org.kuali.common.aws.s3.pojo.S3PrefixContext;
28 import org.kuali.common.threads.ExecutionStatistics;
29 import org.kuali.common.threads.ThreadHandlerContext;
30 import org.kuali.common.threads.ThreadInvoker;
31 import org.kuali.common.threads.listener.PercentCompleteListener;
32 import org.kuali.common.util.FormatUtils;
33 import org.kuali.common.util.Size;
34 import org.slf4j.Logger;
35 import org.slf4j.LoggerFactory;
36
37 import com.amazonaws.auth.AWSCredentials;
38 import com.amazonaws.auth.BasicAWSCredentials;
39 import com.amazonaws.services.s3.AmazonS3Client;
40
41 public class S3UtilsTest {
42 private static final String ACCESSKEY = "AKIAJFD5IM7IPVVUEBNA";
43 private static final String SECRETYKEY = System.getProperty("secret.key");
44
45 private static final Logger log = LoggerFactory.getLogger(S3UtilsTest.class);
46 S3Utils utils = S3Utils.getInstance();
47
48 protected AWSCredentials getCredentials() {
49 log.debug("access key: " + ACCESSKEY);
50 return new BasicAWSCredentials(ACCESSKEY, SECRETYKEY);
51 }
52
53 protected AmazonS3Client getClient() {
54 AWSCredentials credentials = getCredentials();
55 return new AmazonS3Client(credentials);
56 }
57
58
59 public void bruteForceListing() {
60 String bucketName = "maven.kuali.org";
61 AmazonS3Client client = getClient();
62 utils.summarize(client, bucketName);
63 }
64
65
66 public void testGetStructure() {
67 try {
68 org.kuali.common.util.Size[] values = Size.values();
69 for (Size value : values) {
70 log.info(value.getSizeLabel() + " " + value.getValue());
71 }
72 long now = System.currentTimeMillis();
73 long bytes = Long.MAX_VALUE;
74 log.info(FormatUtils.getSize(bytes));
75 log.info(FormatUtils.getRate(now, bytes));
76 log.info(FormatUtils.getTime(now));
77 log.info(FormatUtils.getTime(bytes));
78 String delimiter = "/";
79 String bucket = "maven.kuali.org";
80 AmazonS3Client client = getClient();
81
82 KualiMavenBucketBaseCase baseCase1 = new KualiMavenBucketBaseCase();
83 baseCase1.setDelimiter(delimiter);
84 baseCase1.setToken("latest");
85
86 JavaxServletOnlyBaseCase baseCase2 = new JavaxServletOnlyBaseCase();
87 baseCase2.setDelimiter(delimiter);
88 baseCase2.setToken("latest");
89
90 ExternalOnlyBaseCase baseCase3 = new ExternalOnlyBaseCase();
91 baseCase3.setDelimiter(delimiter);
92 baseCase3.setToken("latest");
93
94 long start1 = System.currentTimeMillis();
95 List<String> prefixes = new ArrayList<String>();
96 utils.buildPrefixList(client, bucket, prefixes, null, delimiter, baseCase1);
97 long elapsed1 = System.currentTimeMillis() - start1;
98 DefaultMutableTreeNode node = utils.buildTree(prefixes, delimiter);
99 log.info("Total Prefixes: " + prefixes.size());
100 log.info("Total Time: " + FormatUtils.getTime(elapsed1));
101 List<DefaultMutableTreeNode> leaves = utils.getLeaves(node);
102 log.info("Total Leaves: " + leaves.size());
103
104
105
106
107
108
109
110 List<BucketPrefixSummary> summaries = utils.getBucketSummaryLeafs(node);
111 Collections.shuffle(summaries);
112 List<S3PrefixContext> contexts = utils.getS3PrefixContexts(client, bucket, summaries);
113
114
115 PercentCompleteListener<S3PrefixContext> listener = new PercentCompleteListener<S3PrefixContext>();
116 listener.setPercentageIncrement(1);
117 ThreadHandlerContext<S3PrefixContext> thc = new ThreadHandlerContext<S3PrefixContext>();
118 thc.setList(contexts);
119 thc.setHandler(new BucketPrefixSummaryHandler());
120 thc.setMax(50);
121 thc.setMin(10);
122 thc.setDivisor(1);
123 thc.setListener(listener);
124 ThreadInvoker invoker = new ThreadInvoker();
125 ExecutionStatistics stats = invoker.invokeThreads(thc);
126
127
128 long millis = stats.getExecutionTime();
129 long iterationCount = stats.getIterationCount();
130 log.info("Elapsed: " + FormatUtils.getTime(millis));
131 log.info("Iteration Count: " + iterationCount);
132 BucketPrefixSummary summary = (BucketPrefixSummary) node.getUserObject();
133 utils.fillInSummaries(node);
134 log.info("Total Bucket Size: " + FormatUtils.getSize(summary.getSize()));
135 log.info("Total Object Count: " + summary.getCount());
136
137
138 } catch (Exception e) {
139 e.printStackTrace();
140 }
141 }
142
143 }