View Javadoc
1   /**
2    * Copyright 2004-2014 The Kuali Foundation
3    *
4    * Licensed under the Educational Community License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    * http://www.opensource.org/licenses/ecl2.php
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package org.kuali.common.aws.s3;
17  
18  import java.util.ArrayList;
19  import java.util.Collections;
20  import java.util.List;
21  
22  import javax.swing.tree.DefaultMutableTreeNode;
23  
24  import org.kuali.common.aws.s3.monitor.BucketPrefixSummaryHandler;
25  import org.kuali.common.aws.s3.monitor.S3Utils;
26  import org.kuali.common.aws.s3.pojo.BucketPrefixSummary;
27  import org.kuali.common.aws.s3.pojo.S3PrefixContext;
28  import org.kuali.common.threads.ExecutionStatistics;
29  import org.kuali.common.threads.ThreadHandlerContext;
30  import org.kuali.common.threads.ThreadInvoker;
31  import org.kuali.common.threads.listener.PercentCompleteListener;
32  import org.kuali.common.util.FormatUtils;
33  import org.kuali.common.util.Size;
34  import org.slf4j.Logger;
35  import org.slf4j.LoggerFactory;
36  
37  import com.amazonaws.auth.AWSCredentials;
38  import com.amazonaws.auth.BasicAWSCredentials;
39  import com.amazonaws.services.s3.AmazonS3Client;
40  
41  public class S3UtilsTest {
42  	private static final String ACCESSKEY = "AKIAJFD5IM7IPVVUEBNA";
43  	private static final String SECRETYKEY = System.getProperty("secret.key");
44  
45  	private static final Logger log = LoggerFactory.getLogger(S3UtilsTest.class);
46  	S3Utils utils = S3Utils.getInstance();
47  
48  	protected AWSCredentials getCredentials() {
49  		log.debug("access key: " + ACCESSKEY);
50  		return new BasicAWSCredentials(ACCESSKEY, SECRETYKEY);
51  	}
52  
53  	protected AmazonS3Client getClient() {
54  		AWSCredentials credentials = getCredentials();
55  		return new AmazonS3Client(credentials);
56  	}
57  
58  	// @Test
59  	public void bruteForceListing() {
60  		String bucketName = "maven.kuali.org";
61  		AmazonS3Client client = getClient();
62  		utils.summarize(client, bucketName);
63  	}
64  
65  	// @Test
66  	public void testGetStructure() {
67  		try {
68  			org.kuali.common.util.Size[] values = Size.values();
69  			for (Size value : values) {
70  				log.info(value.getSizeLabel() + " " + value.getValue());
71  			}
72  			long now = System.currentTimeMillis();
73  			long bytes = Long.MAX_VALUE;
74  			log.info(FormatUtils.getSize(bytes));
75  			log.info(FormatUtils.getRate(now, bytes));
76  			log.info(FormatUtils.getTime(now));
77  			log.info(FormatUtils.getTime(bytes));
78  			String delimiter = "/";
79  			String bucket = "maven.kuali.org";
80  			AmazonS3Client client = getClient();
81  
82  			KualiMavenBucketBaseCase baseCase1 = new KualiMavenBucketBaseCase();
83  			baseCase1.setDelimiter(delimiter);
84  			baseCase1.setToken("latest");
85  
86  			JavaxServletOnlyBaseCase baseCase2 = new JavaxServletOnlyBaseCase();
87  			baseCase2.setDelimiter(delimiter);
88  			baseCase2.setToken("latest");
89  
90  			ExternalOnlyBaseCase baseCase3 = new ExternalOnlyBaseCase();
91  			baseCase3.setDelimiter(delimiter);
92  			baseCase3.setToken("latest");
93  
94  			long start1 = System.currentTimeMillis();
95  			List<String> prefixes = new ArrayList<String>();
96  			utils.buildPrefixList(client, bucket, prefixes, null, delimiter, baseCase1);
97  			long elapsed1 = System.currentTimeMillis() - start1;
98  			DefaultMutableTreeNode node = utils.buildTree(prefixes, delimiter);
99  			log.info("Total Prefixes: " + prefixes.size());
100 			log.info("Total Time: " + FormatUtils.getTime(elapsed1));
101 			List<DefaultMutableTreeNode> leaves = utils.getLeaves(node);
102 			log.info("Total Leaves: " + leaves.size());
103 			// long start2 = System.currentTimeMillis();
104 			// utils.summarize(client, bucket, node);
105 			// long elapsed2 = System.currentTimeMillis() - start2;
106 			// BucketSummary summary = (BucketSummary) node.getUserObject();
107 			// log.info("Total Time: " + sf.getTime(elapsed2));
108 			// log.info("Count: " + summary.getCount());
109 			// log.info("Size: " + sf.getSize(summary.getSize()));
110 			List<BucketPrefixSummary> summaries = utils.getBucketSummaryLeafs(node);
111 			Collections.shuffle(summaries);
112 			List<S3PrefixContext> contexts = utils.getS3PrefixContexts(client, bucket, summaries);
113 
114 			// Store some context for the thread handler
115 			PercentCompleteListener<S3PrefixContext> listener = new PercentCompleteListener<S3PrefixContext>();
116 			listener.setPercentageIncrement(1);
117 			ThreadHandlerContext<S3PrefixContext> thc = new ThreadHandlerContext<S3PrefixContext>();
118 			thc.setList(contexts);
119 			thc.setHandler(new BucketPrefixSummaryHandler());
120 			thc.setMax(50);
121 			thc.setMin(10);
122 			thc.setDivisor(1);
123 			thc.setListener(listener);
124 			ThreadInvoker invoker = new ThreadInvoker();
125 			ExecutionStatistics stats = invoker.invokeThreads(thc);
126 
127 			// Show some stats
128 			long millis = stats.getExecutionTime();
129 			long iterationCount = stats.getIterationCount();
130 			log.info("Elapsed: " + FormatUtils.getTime(millis));
131 			log.info("Iteration Count: " + iterationCount);
132 			BucketPrefixSummary summary = (BucketPrefixSummary) node.getUserObject();
133 			utils.fillInSummaries(node);
134 			log.info("Total Bucket Size: " + FormatUtils.getSize(summary.getSize()));
135 			log.info("Total Object Count: " + summary.getCount());
136 
137 			// log.info("S3 Bucket Summary\n" + utils.toString(node, Size.MB));
138 		} catch (Exception e) {
139 			e.printStackTrace();
140 		}
141 	}
142 
143 }