org.kuali.common.aws.s3.monitor

Class S3Utils

java.lang.Object

org.kuali.common.aws.s3.monitor.S3Utils

public class S3Utils
extends Object

Utility methods related to Amazon S3

Field Summary

Fields

Modifier and Type	Field and Description
static long	MULTI_PART_UPLOAD_THRESHOLD

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	S3Utils()

Method Summary

Methods

Modifier and Type	Method and Description
void	blockingMultiPartUpload(com.amazonaws.services.s3.model.PutObjectRequest request, com.amazonaws.services.s3.transfer.TransferManager manager) Use this method to reliably upload large files and wait until they are fully uploaded before continuing.
void	buildPrefixList(com.amazonaws.services.s3.AmazonS3Client client, String bucketName, List<String> prefixes, String prefix, String delimiter, BaseCase baseCase)
DefaultMutableTreeNode	buildTree(List<String> prefixes, String delimiter)
void	fillInSummaries(DefaultMutableTreeNode node)
AccountSummary	getAccountSummary(AccountSummaryContext context)
AccountSummary	getAccountSummary(String accessKey, String secretKey, List<String> includes, List<String> excludes)
List<String>	getAccountSummaryCSVColumns()
protected String[]	getAccountSummaryCSVRow(BucketSummary summary, Date date)
protected List<String[]>	getAccountSummaryCSVRows(List<BucketSummary> summaries, Date date)
List<BucketDisplay>	getBucketDisplayList(List<BucketPrefixSummary> summaries, org.kuali.common.util.Size size)
List<com.amazonaws.services.s3.model.Bucket>	getBuckets(com.amazonaws.services.s3.AmazonS3Client client, List<String> includes, List<String> excludes)
List<BucketSummary>	getBucketSummaries(com.amazonaws.services.s3.AmazonS3Client client, List<com.amazonaws.services.s3.model.Bucket> buckets)
BucketSummary	getBucketSummary(com.amazonaws.services.s3.AmazonS3Client client, com.amazonaws.services.s3.model.Bucket bucket)
List<String>	getBucketSummaryColumns()
List<BucketPrefixSummary>	getBucketSummaryLeafs(DefaultMutableTreeNode node)
List<BucketPrefixSummary>	getBucketSummaryList(DefaultMutableTreeNode node, Comparator<BucketPrefixSummary> comparator)
List<DefaultMutableTreeNode>	getChildren(DefaultMutableTreeNode node)
com.amazonaws.services.s3.AmazonS3Client	getClient(String accessKey, String secretKey)
int[]	getColumnLengths(List<String> columns, List<String[]> rows)
com.amazonaws.auth.AWSCredentials	getCredentials(String accessKey, String secretKey)
static S3Utils	getInstance()
List<DefaultMutableTreeNode>	getLeaves(DefaultMutableTreeNode node)
com.amazonaws.services.s3.model.ListObjectsRequest	getListObjectsRequest(String bucketName)
com.amazonaws.services.s3.model.ListObjectsRequest	getListObjectsRequest(String bucketName, String prefix)
com.amazonaws.services.s3.model.ListObjectsRequest	getListObjectsRequest(String bucketName, String prefix, String delimiter)
com.amazonaws.services.s3.model.ListObjectsRequest	getListObjectsRequest(String bucketName, String prefix, String delimiter, Integer maxKeys)
String	getParentPrefix(String prefix, String delimiter)
protected String[]	getRow(BucketSummary summary)
List<String[]>	getRows(List<BucketSummary> summaries)
List<S3PrefixContext>	getS3PrefixContexts(com.amazonaws.services.s3.AmazonS3Client client, String bucketName, List<BucketPrefixSummary> summaries)
boolean	include(String bucketName, List<String> includes, List<String> excludes)
String	lpad(String s, int size)
String	rpad(String s, int size)
BucketPrefixSummary	summarize(com.amazonaws.services.s3.AmazonS3Client client, String bucketName)
BucketPrefixSummary	summarize(com.amazonaws.services.s3.AmazonS3Client client, String bucketName, BucketPrefixSummary summary)
void	summarize(com.amazonaws.services.s3.AmazonS3Client client, String bucketName, DefaultMutableTreeNode node)
void	summarize(BucketPrefixSummary summary, List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries)
String	toCSV(AccountSummary summary)
String	toCSV(AccountSummary summary, boolean printColumnHeaders)
String	toCSV(List<String> columns, List<String[]> rows, boolean printColumnHeaders)
List<String>	toList(String csv)
List<String>	toList(String[] tokens)
String	toString(AccountSummary summary)
String	toString(DefaultMutableTreeNode node)
String	toString(DefaultMutableTreeNode node, Comparator<BucketPrefixSummary> comparator)
String	toString(DefaultMutableTreeNode node, org.kuali.common.util.Size size)
String	toString(DefaultMutableTreeNode node, org.kuali.common.util.Size size, Comparator<BucketPrefixSummary> comparator)
String	toString(List<String> columns, List<String[]> rows)
void	updateAccountSummary(AccountSummary summary)
void	updateBucketSummary(BucketSummary summary, List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries)
void	upload(File file, com.amazonaws.services.s3.model.PutObjectRequest request, com.amazonaws.services.s3.AmazonS3Client client, com.amazonaws.services.s3.transfer.TransferManager manager) Upload a single file to Amazon S3.
void	write(File file, String data, boolean append)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

MULTI_PART_UPLOAD_THRESHOLD

public static final long MULTI_PART_UPLOAD_THRESHOLD

Constructor Detail

S3Utils

protected S3Utils()

Method Detail

getInstance

public static S3Utils getInstance()

getCredentials

public com.amazonaws.auth.AWSCredentials getCredentials(String accessKey,
                                               String secretKey)

getClient

public com.amazonaws.services.s3.AmazonS3Client getClient(String accessKey,
                                                 String secretKey)

upload

public void upload(File file,
          com.amazonaws.services.s3.model.PutObjectRequest request,
          com.amazonaws.services.s3.AmazonS3Client client,
          com.amazonaws.services.s3.transfer.TransferManager manager)

Upload a single file to Amazon S3. If the file is larger than MULTI_PART_UPLOAD_THRESHOLD a multi-part upload is used. Multi-part uploads split the file into several smaller chunks with each chunk being uploaded in a different thread. Once all the threads have completed the file is automatically reassembled on S3 as a single file.

blockingMultiPartUpload

public void blockingMultiPartUpload(com.amazonaws.services.s3.model.PutObjectRequest request,
                           com.amazonaws.services.s3.transfer.TransferManager manager)

Use this method to reliably upload large files and wait until they are fully uploaded before continuing. Behind the scenes this is accomplished by splitting the file up into manageable chunks and using separate threads to upload each chunk. Consider using multi-part uploads on files larger than MULTI_PART_UPLOAD_THRESHOLD. When this method returns, all threads have finished and the file has been reassembled on S3. The benefit to this method is that if any one thread fails, only the portion of the file that particular thread was handling will have to be re-uploaded (instead of the entire file). A reasonable number of automatic retries occurs if an individual upload thread fails. If the file upload fails this method throws AmazonS3Exception

getListObjectsRequest

public com.amazonaws.services.s3.model.ListObjectsRequest getListObjectsRequest(String bucketName,
                                                                       String prefix,
                                                                       String delimiter,
                                                                       Integer maxKeys)

getListObjectsRequest

public com.amazonaws.services.s3.model.ListObjectsRequest getListObjectsRequest(String bucketName,
                                                                       String prefix,
                                                                       String delimiter)

getListObjectsRequest

public com.amazonaws.services.s3.model.ListObjectsRequest getListObjectsRequest(String bucketName,
                                                                       String prefix)

getListObjectsRequest

public com.amazonaws.services.s3.model.ListObjectsRequest getListObjectsRequest(String bucketName)

getLeaves

public List<DefaultMutableTreeNode> getLeaves(DefaultMutableTreeNode node)

buildTree

public DefaultMutableTreeNode buildTree(List<String> prefixes,
                               String delimiter)

getParentPrefix

public String getParentPrefix(String prefix,
                     String delimiter)

buildPrefixList

public void buildPrefixList(com.amazonaws.services.s3.AmazonS3Client client,
                   String bucketName,
                   List<String> prefixes,
                   String prefix,
                   String delimiter,
                   BaseCase baseCase)

summarize

public void summarize(com.amazonaws.services.s3.AmazonS3Client client,
             String bucketName,
             DefaultMutableTreeNode node)

getBucketSummaryLeafs

public List<BucketPrefixSummary> getBucketSummaryLeafs(DefaultMutableTreeNode node)

fillInSummaries

public void fillInSummaries(DefaultMutableTreeNode node)

getChildren

public List<DefaultMutableTreeNode> getChildren(DefaultMutableTreeNode node)

getAccountSummary

public AccountSummary getAccountSummary(AccountSummaryContext context)

getAccountSummary

public AccountSummary getAccountSummary(String accessKey,
                               String secretKey,
                               List<String> includes,
                               List<String> excludes)

getBucketSummaries

public List<BucketSummary> getBucketSummaries(com.amazonaws.services.s3.AmazonS3Client client,
                                     List<com.amazonaws.services.s3.model.Bucket> buckets)

getBucketSummary

public BucketSummary getBucketSummary(com.amazonaws.services.s3.AmazonS3Client client,
                             com.amazonaws.services.s3.model.Bucket bucket)

summarize

public BucketPrefixSummary summarize(com.amazonaws.services.s3.AmazonS3Client client,
                            String bucketName)

summarize

public BucketPrefixSummary summarize(com.amazonaws.services.s3.AmazonS3Client client,
                            String bucketName,
                            BucketPrefixSummary summary)

updateBucketSummary

public void updateBucketSummary(BucketSummary summary,
                       List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries)

summarize

public void summarize(BucketPrefixSummary summary,
             List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries)

toString

public String toString(DefaultMutableTreeNode node)

toString

public String toString(DefaultMutableTreeNode node,
              org.kuali.common.util.Size size)

toString

public String toString(DefaultMutableTreeNode node,
              Comparator<BucketPrefixSummary> comparator)

getBucketSummaryList

public List<BucketPrefixSummary> getBucketSummaryList(DefaultMutableTreeNode node,
                                             Comparator<BucketPrefixSummary> comparator)

getBucketDisplayList

public List<BucketDisplay> getBucketDisplayList(List<BucketPrefixSummary> summaries,
                                       org.kuali.common.util.Size size)

getS3PrefixContexts

public List<S3PrefixContext> getS3PrefixContexts(com.amazonaws.services.s3.AmazonS3Client client,
                                        String bucketName,
                                        List<BucketPrefixSummary> summaries)

toString

public String toString(DefaultMutableTreeNode node,
              org.kuali.common.util.Size size,
              Comparator<BucketPrefixSummary> comparator)

getColumnLengths

public int[] getColumnLengths(List<String> columns,
                     List<String[]> rows)

toList

public List<String> toList(String csv)

toList

public List<String> toList(String[] tokens)

updateAccountSummary

public void updateAccountSummary(AccountSummary summary)

toString

public String toString(AccountSummary summary)

toCSV

public String toCSV(AccountSummary summary)

toCSV

public String toCSV(AccountSummary summary,
           boolean printColumnHeaders)

toCSV

public String toCSV(List<String> columns,
           List<String[]> rows,
           boolean printColumnHeaders)

toString

public String toString(List<String> columns,
              List<String[]> rows)

getBuckets

public List<com.amazonaws.services.s3.model.Bucket> getBuckets(com.amazonaws.services.s3.AmazonS3Client client,
                                                      List<String> includes,
                                                      List<String> excludes)

include

public boolean include(String bucketName,
              List<String> includes,
              List<String> excludes)

lpad

public String lpad(String s,
          int size)

rpad

public String rpad(String s,
          int size)

getBucketSummaryColumns

public List<String> getBucketSummaryColumns()

getRows

public List<String[]> getRows(List<BucketSummary> summaries)

getRow

protected String[] getRow(BucketSummary summary)

getAccountSummaryCSVColumns

public List<String> getAccountSummaryCSVColumns()

getAccountSummaryCSVRows

protected List<String[]> getAccountSummaryCSVRows(List<BucketSummary> summaries,
                                      Date date)

getAccountSummaryCSVRow

protected String[] getAccountSummaryCSVRow(BucketSummary summary,
                               Date date)

write

public void write(File file,
         String data,
         boolean append)