1 package org.kuali.common.devops.jenkins.scan;
2
3 import static com.google.common.base.Stopwatch.createStarted;
4 import static com.google.common.collect.Lists.newArrayList;
5 import static com.google.common.collect.Maps.newHashMap;
6 import static com.google.common.collect.Maps.newTreeMap;
7 import static com.google.common.collect.Ordering.natural;
8 import static java.lang.Long.parseLong;
9 import static java.lang.Math.ceil;
10 import static java.lang.Math.max;
11 import static java.lang.String.format;
12 import static java.util.Collections.reverse;
13 import static java.util.Collections.sort;
14 import static java.util.TimeZone.getTimeZone;
15 import static org.apache.commons.io.FileUtils.readFileToString;
16 import static org.apache.commons.io.FileUtils.write;
17 import static org.apache.commons.io.FileUtils.writeStringToFile;
18 import static org.kuali.common.core.io.Files.getCanonicalFile;
19 import static org.kuali.common.devops.jenkins.scan.AggregateJobInformationTest.BUCKET;
20 import static org.kuali.common.devops.jenkins.scan.AggregateJobInformationTest.REPO;
21 import static org.kuali.common.devops.jenkins.scan.AggregateJobInformationTest.getKey;
22 import static org.kuali.common.devops.jenkins.scan.Jobs.getCalculatedDurationStatistics;
23 import static org.kuali.common.devops.jenkins.scan.Jobs.getLastBuildNumber;
24 import static org.kuali.common.devops.jenkins.scan.Jobs.getRecordedDurationStatistics;
25 import static org.kuali.common.devops.jenkins.scan.Jobs.getStartTimeStats;
26 import static org.kuali.common.devops.jenkins.scan.Jobs.getTotalExecutions;
27 import static org.kuali.common.util.Encodings.UTF8;
28 import static org.kuali.common.util.FormatUtils.getCount;
29 import static org.kuali.common.util.FormatUtils.getTime;
30 import static org.kuali.common.util.base.Exceptions.illegalState;
31 import static org.kuali.common.util.log.LoggerUtils.logTable;
32 import static org.kuali.common.util.log.Loggers.newLogger;
33
34 import java.io.File;
35 import java.io.IOException;
36 import java.text.SimpleDateFormat;
37 import java.util.Date;
38 import java.util.List;
39 import java.util.Map;
40
41 import org.junit.Ignore;
42 import org.junit.Test;
43 import org.kuali.common.core.json.api.JsonService;
44 import org.kuali.common.core.json.jackson.JacksonJsonService;
45 import org.kuali.common.util.FormatUtils;
46 import org.kuali.common.util.LocationUtils;
47 import org.kuali.common.util.file.CanonicalFile;
48 import org.slf4j.Logger;
49 import org.springframework.core.io.DefaultResourceLoader;
50 import org.springframework.core.io.Resource;
51 import org.springframework.core.io.ResourceLoader;
52
53 import com.google.common.base.Joiner;
54 import com.google.common.base.Stopwatch;
55 import com.google.common.collect.ImmutableList;
56
57 public class AggregatedStatisticsTest {
58
59 private static final Logger logger = newLogger();
60
61 private JsonService json = new JacksonJsonService();
62 private List<String> stacks = ImmutableList.of("prod", "rice", "cifn");
63 private static final Map<Team, List<Job>> jobsByTeam = newTreeMap();
64 private static final Map<Type, List<Job>> jobsByType = newTreeMap();
65 private static final Map<TypeAndTeam, List<Job>> jobsByTypeAndTeam = newTreeMap();
66 private final long msPerSecond = 1000;
67 private final long msPerMinute = msPerSecond * 60;
68 private final long msPerHour = msPerMinute * 60;
69
70
71 private final String timezone = "US/Eastern";
72 private final String dateFormat = "yyyy-MM";
73 private final SimpleDateFormat sdf = getSimpleDateFormat(dateFormat, timezone);
74 private final SimpleDateFormat minuteFormatter = getSimpleDateFormat("yyyyMMdd-HHmm", timezone);
75 private final SimpleDateFormat dayFormatter = getSimpleDateFormat("yyyyMMdd", timezone);
76
77 @Test
78 public void test() {
79 try {
80 Stopwatch sw = createStarted();
81 List<String> keys = getKeys(stacks);
82 List<Jenkins> jenkins = getJenkins(keys);
83 List<JobExecution> builds = getBuilds(jenkins);
84 doBuildSpikes(builds);
85 doHoursPerMonth(builds);
86 info("total builds -> %s", getCount(builds.size()));
87 info("total build time -> %s", getTime(sumDuration(builds)));
88 info("elapsed -> %s", getTime(sw));
89 } catch (Throwable e) {
90 e.printStackTrace();
91 }
92 }
93
94 private void doBuildSpikes(List<JobExecution> builds) throws Exception {
95 Map<String, Integer> map = asMap(builds);
96 info("tracked minutes -> %s", getCount(map.size()));
97 List<String> keys = natural().sortedCopy(map.keySet());
98 List<Spike> list = newArrayList();
99 for (String key : keys) {
100 long minute = parseLong(key);
101 long timestamp = minute * msPerMinute;
102 int count = map.get(key);
103 Spike element = Spike.builder().withCount(count).withTimestamp(timestamp).build();
104 list.add(element);
105 }
106 sort(list, SpikeComparator.INSTANCE);
107 reverse(list);
108 Map<String, Integer> spikeMap = newHashMap();
109 for (Spike spike : list) {
110 long timestamp = spike.getTimestamp();
111 String day = dayFormatter.format(new Date(timestamp));
112 Integer value = spikeMap.get(day);
113 if (value == null) {
114 value = spike.getCount();
115 }
116 Integer max = max(value, spike.getCount());
117 spikeMap.put(day, max);
118 }
119 List<String> values = natural().sortedCopy(spikeMap.keySet());
120 List<String> lines = newArrayList();
121 for (String value : values) {
122 lines.add(value + "," + spikeMap.get(value));
123 }
124 File file = getCanonicalFile("./target/jenkins/aggregated-spikes.txt");
125 info("create -> %s", file);
126 write(file, Joiner.on('\n').join(lines));
127 }
128
129 private void doHoursPerMonth(List<JobExecution> builds) throws Exception {
130 Map<String, Long> map = asMonthlyBuildHours(builds);
131 List<String> csv = asCsv(map);
132 File file = getCanonicalFile("./target/jenkins/hours.txt");
133 info("create -> %s", file);
134 write(file, Joiner.on('\n').join(csv));
135 }
136
137 private long sumDuration(List<JobExecution> builds) {
138 long duration = 0;
139 for (JobExecution build : builds) {
140 duration += build.getDuration();
141 }
142 return duration;
143 }
144
145 private List<String> asCsv(Map<String, Long> map) throws Exception {
146 List<String> lines = newArrayList();
147 Joiner csv = Joiner.on(',');
148 List<String> sorted = natural().sortedCopy(map.keySet());
149 for (String key : sorted) {
150 long timestamp = parseLong(key);
151 long millis = map.get(key);
152 long hours = millis / msPerHour;
153 String line = csv.join(sdf.format(timestamp), hours + "");
154 lines.add(line);
155 }
156 return lines;
157 }
158
159 private SimpleDateFormat getSimpleDateFormat(String dateFormat, String timezone) {
160 SimpleDateFormat sdf = new SimpleDateFormat(dateFormat);
161 sdf.setTimeZone(getTimeZone(timezone));
162 return sdf;
163 }
164
165 private long getMonth(long timestamp) throws Exception {
166 return sdf.parse(sdf.format(new Date(timestamp))).getTime();
167 }
168
169 private Map<String, Integer> asMap(List<JobExecution> builds) throws Exception {
170 Map<String, Integer> map = newHashMap();
171 for (JobExecution build : builds) {
172 long startTime = build.getStartTime();
173 long duration = build.getDuration();
174 int minutes = (int) ceil((duration * 1D) / msPerMinute);
175 for (int i = 0; i < minutes; i++) {
176 long timestamp = startTime + (msPerMinute * i);
177 long minute = timestamp / msPerMinute;
178 String key = minute + "";
179 Integer count = map.get(key);
180 map.put(key, (count == null) ? 1 : (count + 1));
181 }
182 }
183 return map;
184 }
185
186 private Map<String, Long> asMonthlyBuildHours(List<JobExecution> builds) throws Exception {
187 Map<String, Long> map = newHashMap();
188 for (JobExecution build : builds) {
189 long startTime = build.getStartTime();
190 long startMonth = getMonth(startTime);
191 String key = startMonth + "";
192 Long millis = map.get(key);
193 map.put(key, (millis == null) ? 0L : (millis + build.getDuration()));
194 }
195 return map;
196 }
197
198 private List<JobExecution> getBuilds(List<Jenkins> jenkins) {
199 List<JobExecution> builds = newArrayList();
200 for (Jenkins server : jenkins) {
201 for (Job job : server.getJobs()) {
202 builds.addAll(job.getExecutions());
203 }
204 }
205 return builds;
206 }
207
208 @Test
209 @Ignore
210 public void test2() {
211 try {
212 Stopwatch sw = createStarted();
213 List<String> keys = getKeys(stacks);
214 List<Jenkins> jenkins = getJenkins(keys);
215 summarize(jenkins);
216 info("%s jenkins servers - %s", jenkins.size(), getTime(sw));
217 for (TypeAndTeam tt : jobsByTypeAndTeam.keySet()) {
218 List<Job> jobs = jobsByTypeAndTeam.get(tt);
219 long builds = getTotalExecutions(jobs);
220 SummaryStatistics ss = getCalculatedDurationStatistics(jobs);
221 String duration = getTime(new Double(ss.getSum()).longValue());
222 info("%s:%s %s %s %s", tt.getTeam(), tt.getType(), jobs.size(), builds, duration);
223 if (tt.getTeam() == Team.STUDENT && tt.getType() == Type.TEST) {
224 summarizeJobs(jobs);
225 }
226 if (tt.getTeam() == Team.STUDENT && tt.getType() == Type.OTHER) {
227 summarizeJobs(jobs);
228 }
229 if (tt.getTeam() == Team.RICE && tt.getType() == Type.TEST) {
230 summarizeJobs(jobs);
231 }
232 }
233 } catch (Throwable e) {
234 e.printStackTrace();
235 }
236 }
237
238 protected static void summarizeJobs(List<Job> jobs) {
239 List<Job> sorted = newArrayList(jobs);
240 sort(sorted, JobTimeComparator.INSTANCE);
241 reverse(sorted);
242 List<String> columns = ImmutableList.of("name", "builds", "total time");
243 List<Object[]> rows = newArrayList();
244 for (Job job : sorted) {
245 long builds = getLastBuildNumber(job);
246 long time = new Double(getCalculatedDurationStatistics(job).getSummary().getSum()).longValue();
247 Object[] row = { job.getName(), builds, getTime(time) };
248 rows.add(row);
249 }
250 logTable(columns, rows);
251 }
252
253 protected static void summarize(List<Jenkins> jenkins) {
254 List<String> columns = ImmutableList.of("host", "jobs", "recorded builds", "total builds", "duration recorded", "total duration", "first job", "last job", "elapsed");
255 List<Object[]> rows = newArrayList();
256 for (Jenkins element : jenkins) {
257 rows.add(summarize(element));
258 }
259 logTable(columns, rows);
260 }
261
262 protected static Object[] summarize(Jenkins jenkins) {
263 int executions = 0;
264 int totalBuilds = 0;
265 long recordedDuration = 0;
266 long totalDuration = 0;
267
268 SummaryStatistics startTimes = getStartTimeStats(jenkins.getJobs());
269 for (Job job : jenkins.getJobs()) {
270 updateTypeMap(job);
271 updateTeamMap(job);
272 updateTypeAndTeamMap(job);
273 executions += job.getExecutions().size();
274 int lastBuildNumber = getLastBuildNumber(job);
275 recordedDuration += new Double(getRecordedDurationStatistics(job).getSummary().getSum()).longValue();
276 totalDuration += new Double(getCalculatedDurationStatistics(job).getSummary().getSum()).longValue();
277 totalBuilds += lastBuildNumber;
278 }
279 long firstStartTime = new Double(startTimes.getMin()).longValue();
280 long lastStartTime = new Double(startTimes.getMax()).longValue();
281 String jobs = FormatUtils.getCount(jenkins.getJobs().size());
282 String execs = FormatUtils.getCount(executions);
283 String builds = FormatUtils.getCount(totalBuilds);
284 String durationRecorded = FormatUtils.getTime(recordedDuration);
285 String durationTotal = FormatUtils.getTime(totalDuration);
286 String firstJob = FormatUtils.getDate(firstStartTime);
287 String lastJob = FormatUtils.getDate(lastStartTime);
288 String elapsed = FormatUtils.getTime(lastStartTime - firstStartTime);
289 return new Object[] { jenkins.getHostname(), jobs, execs, builds, durationRecorded, durationTotal, firstJob, lastJob, elapsed };
290 }
291
292 protected static void updateTypeAndTeamMap(Job job) {
293 Type type = getType(job);
294 Team team = getTeam(job);
295 TypeAndTeam tt = TypeAndTeam.builder().withTeam(team).withType(type).build();
296 List<Job> jobs = jobsByTypeAndTeam.get(tt);
297 if (jobs == null) {
298 jobs = newArrayList();
299 jobsByTypeAndTeam.put(tt, jobs);
300 }
301 jobs.add(job);
302 }
303
304 protected static void updateTypeMap(Job job) {
305 Type type = getType(job);
306 List<Job> jobs = jobsByType.get(type);
307 if (jobs == null) {
308 jobs = newArrayList();
309 jobsByType.put(type, jobs);
310 }
311 jobs.add(job);
312 }
313
314 protected static void updateTeamMap(Job job) {
315 Team team = getTeam(job);
316 List<Job> jobs = jobsByTeam.get(team);
317 if (jobs == null) {
318 jobs = newArrayList();
319 jobsByTeam.put(team, jobs);
320 }
321 jobs.add(job);
322 }
323
324 protected List<Jenkins> getJenkins(List<String> keys) {
325 List<Jenkins> list = newArrayList();
326 for (String key : keys) {
327 String string = getJson(key);
328 info("parsing -> json");
329 Jenkins element = json.readString(string, Jenkins.class);
330 list.add(element);
331 }
332 return ImmutableList.copyOf(list);
333 }
334
335 protected String getJson(String key) {
336 try {
337 String url = format("http://%s/%s", BUCKET, key);
338 File file = getFile(url);
339 if (file.exists()) {
340 info("loading -> [%s]", file);
341 return readFileToString(file, UTF8);
342 } else {
343 info("loading -> [%s]", url);
344 String content = LocationUtils.toString(url, UTF8);
345 info("creating -> [%s]", file);
346 writeStringToFile(file, content);
347 return content;
348 }
349 } catch (IOException e) {
350 throw illegalState(e);
351 }
352 }
353
354 protected File getFile(String url) {
355 ResourceLoader loader = new DefaultResourceLoader();
356 Resource resource = loader.getResource(url);
357 String filename = resource.getFilename();
358 return new CanonicalFile("./target/jenkins/" + filename);
359 }
360
361 protected List<String> getKeys(List<String> stacks) {
362 List<String> keys = newArrayList();
363 for (String stack : stacks) {
364 String key = getKey(REPO, stack, "latest");
365 keys.add(key);
366 }
367 return ImmutableList.copyOf(keys);
368 }
369
370 protected static void info(String msg, Object... args) {
371 logger.info((args == null) ? msg : format(msg, args));
372 }
373
374 protected static Type getType(Job job) {
375 String name = job.getName();
376 if (name.contains("compile") || name.contains("unit") || name.contains("publish") || name.contains("site") || name.contains("license") || name.contains("nightly")
377 || name.contains("third-party")) {
378 return Type.BUILD;
379 }
380 if (name.contains("-ft") || name.contains("integration-test") || name.contains("test-integration") || name.contains("load-test") || name.contains("smoke-test")
381 || name.contains("test-functional")) {
382 return Type.TEST;
383 }
384 if (name.contains("deploy") || name.contains("-db-") || name.contains("impex")) {
385 return Type.DEPLOY;
386 }
387 if (name.contains("release")) {
388 return Type.RELEASE;
389 }
390 return Type.OTHER;
391 }
392
393 protected static Team getTeam(Job job) {
394 String name = job.getName();
395 if (name.contains("ks-")) {
396 return Team.STUDENT;
397 }
398 if (name.contains("-ks")) {
399 return Team.STUDENT;
400 }
401 if (name.contains("student")) {
402 return Team.STUDENT;
403 }
404 if (name.contains("rice")) {
405 return Team.RICE;
406 }
407 if (name.contains("rice")) {
408 return Team.RICE;
409 }
410 if (name.contains("freemarker")) {
411 return Team.RICE;
412 }
413 if (name.contains("eghm")) {
414 return Team.RICE;
415 }
416 if (name.contains("commons-beanutils")) {
417 return Team.RICE;
418 }
419 if (name.contains("ole")) {
420 return Team.OLE;
421 }
422 if (name.contains("OLE")) {
423 return Team.OLE;
424 }
425 if (name.contains("mobility")) {
426 return Team.MOBILITY;
427 }
428 if (name.contains("kpme")) {
429 return Team.KHR;
430 }
431 if (name.contains("khr")) {
432 return Team.KHR;
433 }
434 if (name.contains("kdo")) {
435 return Team.DEVOPS;
436 }
437 if (name.contains("maven-")) {
438 return Team.DEVOPS;
439 }
440 if (name.contains("update-ci")) {
441 return Team.DEVOPS;
442 }
443 if (name.contains("kuali-")) {
444 return Team.DEVOPS;
445 }
446 if (name.contains("ec2-")) {
447 return Team.DEVOPS;
448 }
449 if (name.contains("devops")) {
450 return Team.DEVOPS;
451 }
452 if (name.contains("restart-jenkins")) {
453 return Team.DEVOPS;
454 }
455 if (name.contains("db-ojb")) {
456 return Team.SHARED;
457 }
458 if (name.contains("impex")) {
459 return Team.SHARED;
460 }
461 return Team.OTHER;
462 }
463
464 }