1 package org.kuali.common.devops.ci;
2
3 import static com.google.common.base.Predicates.not;
4 import static com.google.common.base.Stopwatch.createStarted;
5 import static com.google.common.collect.Iterables.filter;
6 import static com.google.common.collect.Lists.newArrayList;
7 import static java.lang.Boolean.parseBoolean;
8 import static java.lang.Integer.parseInt;
9 import static java.lang.String.format;
10 import static java.util.Arrays.asList;
11 import static java.util.concurrent.TimeUnit.MILLISECONDS;
12 import static org.apache.commons.lang.StringUtils.removeStart;
13 import static org.kuali.common.aws.model.AWS.decryptedCopy;
14 import static org.kuali.common.aws.s3.S3.diskUsage;
15 import static org.kuali.common.aws.s3.S3.weighObjectSummary;
16 import static org.kuali.common.core.collect.Lists.scatter;
17 import static org.kuali.common.core.io.Files.getCanonicalFile;
18 import static org.kuali.common.devops.aws.EncryptedAWSCredentials.ENCRYPTED_AWS_CREDENTIALS_FOUNDATION;
19 import static org.kuali.common.util.FormatUtils.getRate;
20 import static org.kuali.common.util.FormatUtils.getSize;
21 import static org.kuali.common.util.FormatUtils.getTime;
22 import static org.kuali.common.util.base.Callables.submitCallables;
23 import static org.kuali.common.util.base.Precondition.checkNotBlank;
24 import static org.kuali.common.util.encrypt.Encryption.getDefaultEncryptor;
25 import static org.kuali.common.util.log.Loggers.newLogger;
26
27 import java.io.File;
28 import java.util.List;
29 import java.util.concurrent.Callable;
30
31 import org.kuali.common.aws.s3.CopyObjectToFileCallable;
32 import org.kuali.common.aws.s3.DefaultS3Service;
33 import org.kuali.common.aws.s3.S3Service;
34 import org.kuali.common.aws.s3.model.ObjectSummary;
35 import org.kuali.common.core.base.TimedInterval;
36 import org.kuali.common.core.io.DiskUsage;
37 import org.kuali.common.core.json.api.JsonService;
38 import org.kuali.common.core.json.jackson.JacksonJsonService;
39 import org.kuali.common.core.system.VirtualSystem;
40 import org.kuali.common.util.inform.PercentCompleteInformer;
41 import org.slf4j.Logger;
42
43 import com.amazonaws.auth.AWSCredentials;
44 import com.google.common.base.Function;
45 import com.google.common.base.Predicate;
46 import com.google.common.base.Stopwatch;
47 import com.google.common.collect.ImmutableList;
48
49 public class KualiRepoCacher {
50
51 private static final Logger logger = newLogger();
52
53 public static void main(String[] args) {
54 try {
55 new KualiRepoCacher().cacheKualiS3Repo();
56 System.exit(0);
57 } catch (Throwable e) {
58 e.printStackTrace();
59 System.exit(1);
60 }
61 }
62
63 public TimedInterval cacheKualiS3Repo() {
64 String bucket = "maven.kuali.org";
65 AWSCredentials creds = decryptedCopy(getDefaultEncryptor(), ENCRYPTED_AWS_CREDENTIALS_FOUNDATION);
66 S3Service s3 = DefaultS3Service.build(creds);
67 List<String> prefixes = ImmutableList.of("external", "release");
68 Stopwatch sw = createStarted();
69 VirtualSystem vs = VirtualSystem.build();
70 boolean refresh = parseBoolean(vs.getProperty("refresh", "false"));
71 JsonService json = new JacksonJsonService();
72 Predicate<ObjectSummary> predicate = not(IgnorePredicate.INSTANCE);
73 List<ObjectSummary> objects = getObjects(json, s3, bucket, prefixes, refresh);
74 List<ObjectSummary> relevant = newArrayList(filter(objects, predicate));
75 info("objects -> %s", diskUsage(objects));
76 info("relevant -> %s", diskUsage(relevant));
77 copyToLocal(s3, bucket, prefixes, vs, relevant);
78 elapsed(sw);
79 return TimedInterval.build(sw);
80 }
81
82 private static void copyToLocal(S3Service s3, String bucket, List<String> prefixes, VirtualSystem vs, List<ObjectSummary> objects) {
83 Stopwatch sw = createStarted();
84 int threads = parseInt(System.getProperty("threads", "25"));
85 PercentCompleteInformer informer = new PercentCompleteInformer(objects.size());
86 List<List<ObjectSummary>> distribution = scatter(objects, threads, weighObjectSummary());
87 String repositoryPath = getCanonicalFile(vs.getUser().getHome(), ".m2/repository").getPath();
88 info("caching to -> %s", repositoryPath);
89 Function<String, String> keyToPath = new KeyToPath(repositoryPath, prefixes);
90 List<Callable<List<TimedInterval>>> callables = newArrayList();
91 for (List<ObjectSummary> partition : distribution) {
92 CopyObjectToFileCallable.Builder builder = CopyObjectToFileCallable.builder();
93 builder.withObjects(partition);
94 builder.withBucket(bucket);
95 builder.withInformer(informer);
96 builder.withKeyToPath(keyToPath);
97 builder.withS3(s3);
98 Callable<List<TimedInterval>> callable = builder.build();
99 callables.add(callable);
100 }
101 informer.start();
102 submitCallables(callables);
103 informer.stop();
104 DiskUsage du = diskUsage(objects);
105 info("rate -> %s", getRate(sw.elapsed(MILLISECONDS), du.getSize()));
106 elapsed(sw);
107 }
108
109 private static class KeyToPath implements Function<String, String> {
110
111 public KeyToPath(String repositoryPath, List<String> prefixes) {
112 this.repositoryPath = checkNotBlank(repositoryPath, "repositoryPath");
113 this.prefixes = ImmutableList.copyOf(prefixes);
114 }
115
116 private final String repositoryPath;
117 private final ImmutableList<String> prefixes;
118
119 @Override
120 public String apply(String key) {
121 String path = key;
122 for (String prefix : prefixes) {
123 path = removeStart(path, prefix);
124 }
125 return repositoryPath + "/" + removeStart(path, "/");
126 }
127
128 }
129
130 private enum IgnorePredicate implements Predicate<ObjectSummary> {
131 INSTANCE;
132
133 private final List<String> endsWith = asList("/", "-src.zip", "/jenkins-1.532.2.war", ".license", ".lastUpdated", ".class", "/maven-metadata-local.xml",
134 "/maven-metadata.xml", "-javadoc.jar", ".asc", ".sha1", ".md5", "sources.jar", ".wsdl", "-tests.jar");
135
136 private final List<String> contains = asList("/rice-dist", "/jaxb-api/2.1/extracted/", "/firefox/", "apache-tomcat", "nexus-professional", "-SNAPSHOT",
137 "/jmeter/jmeter/2.3-RC4/", "$folder$", "/oracle-xe/", "/rice-archetype", "/rice-checkstyle", "/rice-config", "-jneal");
138
139 private final List<String> riceVersions = asList("/0.", "/1.", "/2.0.");
140
141 @Override
142 public boolean apply(ObjectSummary object) {
143
144 String key = object.getKey();
145
146
147 if (object.getSize() == 0 && key.endsWith("/")) {
148 return true;
149 }
150
151 for (String suffix : endsWith) {
152 if (key.endsWith(suffix)) {
153 return true;
154 }
155 }
156
157 for (String token : contains) {
158 if (key.contains(token)) {
159 return true;
160 }
161 }
162
163 if (key.contains("org/kuali/student")) {
164 return true;
165 }
166
167 if (key.contains("org/kuali/kpme")) {
168 return true;
169 }
170
171 if (key.contains("org/kuali/mobility")) {
172 return true;
173 }
174
175 if (key.contains("org/kuali/ole")) {
176 return true;
177 }
178
179 for (String token : riceVersions) {
180 if (key.contains("org/kuali/rice/") && key.contains(token)) {
181 return true;
182 }
183 }
184
185 return false;
186 }
187 }
188
189 private static List<ObjectSummary> getObjects(JsonService json, S3Service s3, String bucket, List<String> prefixes, boolean refresh) {
190 Stopwatch sw = createStarted();
191 List<ObjectSummary> objects = newArrayList();
192 for (String prefix : prefixes) {
193 objects.addAll(scan(json, s3, bucket, prefix, refresh));
194 }
195 elapsed(sw);
196 return objects;
197 }
198
199 private static List<ObjectSummary> scan(JsonService json, S3Service s3, String bucket, String prefix, boolean refresh) {
200 Stopwatch sw = createStarted();
201 info("scanning -> s3://%s/%s", bucket, prefix);
202 File file = getCanonicalFile("./target/s3/" + prefix + ".json");
203 if (!refresh && file.exists()) {
204 info("reading -> %s", file);
205 return ImmutableList.copyOf(json.read(file, ObjectSummary[].class));
206 }
207 List<ObjectSummary> objects = s3.getCompleteList(bucket, prefix);
208 DiskUsage du = diskUsage(objects);
209 info("%s", du);
210 info("creating -> %s", file);
211 json.write(file, objects);
212 info("size -> %s", getSize(file.length()));
213 elapsed(sw);
214 return objects;
215 }
216
217 protected static void elapsed(Stopwatch sw) {
218 info("elapsed -> %s", getTime(sw));
219 }
220
221 protected static void debug(String msg, Object... args) {
222 logger.debug((args == null || args.length == 0) ? msg : format(msg, args));
223 }
224
225 protected static void info(String msg, Object... args) {
226 logger.info((args == null || args.length == 0) ? msg : format(msg, args));
227 }
228 }