Source code

001package org.kuali.common.devops.jenkins.scan;
002
003import static com.google.common.base.Stopwatch.createStarted;
004import static com.google.common.collect.Lists.newArrayList;
005import static com.google.common.collect.Maps.newHashMap;
006import static com.google.common.collect.Maps.newTreeMap;
007import static com.google.common.collect.Ordering.natural;
008import static java.lang.Long.parseLong;
009import static java.lang.Math.ceil;
010import static java.lang.Math.max;
011import static java.lang.String.format;
012import static java.util.Collections.reverse;
013import static java.util.Collections.sort;
014import static java.util.TimeZone.getTimeZone;
015import static org.apache.commons.io.FileUtils.readFileToString;
016import static org.apache.commons.io.FileUtils.write;
017import static org.apache.commons.io.FileUtils.writeStringToFile;
018import static org.kuali.common.core.io.Files.getCanonicalFile;
019import static org.kuali.common.devops.jenkins.scan.AggregateJobInformationTest.BUCKET;
020import static org.kuali.common.devops.jenkins.scan.AggregateJobInformationTest.REPO;
021import static org.kuali.common.devops.jenkins.scan.AggregateJobInformationTest.getKey;
022import static org.kuali.common.devops.jenkins.scan.Jobs.getCalculatedDurationStatistics;
023import static org.kuali.common.devops.jenkins.scan.Jobs.getLastBuildNumber;
024import static org.kuali.common.devops.jenkins.scan.Jobs.getRecordedDurationStatistics;
025import static org.kuali.common.devops.jenkins.scan.Jobs.getStartTimeStats;
026import static org.kuali.common.devops.jenkins.scan.Jobs.getTotalExecutions;
027import static org.kuali.common.util.Encodings.UTF8;
028import static org.kuali.common.util.FormatUtils.getCount;
029import static org.kuali.common.util.FormatUtils.getTime;
030import static org.kuali.common.util.base.Exceptions.illegalState;
031import static org.kuali.common.util.log.LoggerUtils.logTable;
032import static org.kuali.common.util.log.Loggers.newLogger;
033
034import java.io.File;
035import java.io.IOException;
036import java.text.SimpleDateFormat;
037import java.util.Date;
038import java.util.List;
039import java.util.Map;
040
041import org.junit.Ignore;
042import org.junit.Test;
043import org.kuali.common.core.json.api.JsonService;
044import org.kuali.common.core.json.jackson.JacksonJsonService;
045import org.kuali.common.util.FormatUtils;
046import org.kuali.common.util.LocationUtils;
047import org.kuali.common.util.file.CanonicalFile;
048import org.slf4j.Logger;
049import org.springframework.core.io.DefaultResourceLoader;
050import org.springframework.core.io.Resource;
051import org.springframework.core.io.ResourceLoader;
052
053import com.google.common.base.Joiner;
054import com.google.common.base.Stopwatch;
055import com.google.common.collect.ImmutableList;
056
057public class AggregatedStatisticsTest {
058
059        private static final Logger logger = newLogger();
060
061        private JsonService json = new JacksonJsonService();
062        private List<String> stacks = ImmutableList.of("prod", "rice", "cifn");
063        private static final Map<Team, List<Job>> jobsByTeam = newTreeMap();
064        private static final Map<Type, List<Job>> jobsByType = newTreeMap();
065        private static final Map<TypeAndTeam, List<Job>> jobsByTypeAndTeam = newTreeMap();
066        private final long msPerSecond = 1000;
067        private final long msPerMinute = msPerSecond * 60;
068        private final long msPerHour = msPerMinute * 60;
069        // private final long msPerDay = msPerHour * 24;
070        // private final long msPerYear = msPerDay * 365;
071        private final String timezone = "US/Eastern";
072        private final String dateFormat = "yyyy-MM";
073        private final SimpleDateFormat sdf = getSimpleDateFormat(dateFormat, timezone);
074        private final SimpleDateFormat minuteFormatter = getSimpleDateFormat("yyyyMMdd-HHmm", timezone);
075        private final SimpleDateFormat dayFormatter = getSimpleDateFormat("yyyyMMdd", timezone);
076
077        @Test
078        public void test() {
079                try {
080                        Stopwatch sw = createStarted();
081                        List<String> keys = getKeys(stacks);
082                        List<Jenkins> jenkins = getJenkins(keys);
083                        List<JobExecution> builds = getBuilds(jenkins);
084                        doBuildSpikes(builds);
085                        doHoursPerMonth(builds);
086                        info("total builds     -> %s", getCount(builds.size()));
087                        info("total build time -> %s", getTime(sumDuration(builds)));
088                        info("elapsed          -> %s", getTime(sw));
089                } catch (Throwable e) {
090                        e.printStackTrace();
091                }
092        }
093
094        private void doBuildSpikes(List<JobExecution> builds) throws Exception {
095                Map<String, Integer> map = asMap(builds);
096                info("tracked minutes -> %s", getCount(map.size()));
097                List<String> keys = natural().sortedCopy(map.keySet());
098                List<Spike> list = newArrayList();
099                for (String key : keys) {
100                        long minute = parseLong(key);
101                        long timestamp = minute * msPerMinute;
102                        int count = map.get(key);
103                        Spike element = Spike.builder().withCount(count).withTimestamp(timestamp).build();
104                        list.add(element);
105                }
106                sort(list, SpikeComparator.INSTANCE);
107                reverse(list);
108                Map<String, Integer> spikeMap = newHashMap();
109                for (Spike spike : list) {
110                        long timestamp = spike.getTimestamp();
111                        String day = dayFormatter.format(new Date(timestamp));
112                        Integer value = spikeMap.get(day);
113                        if (value == null) {
114                                value = spike.getCount();
115                        }
116                        Integer max = max(value, spike.getCount());
117                        spikeMap.put(day, max);
118                }
119                List<String> values = natural().sortedCopy(spikeMap.keySet());
120                List<String> lines = newArrayList();
121                for (String value : values) {
122                        lines.add(value + "," + spikeMap.get(value));
123                }
124                File file = getCanonicalFile("./target/jenkins/aggregated-spikes.txt");
125                info("create  -> %s", file);
126                write(file, Joiner.on('\n').join(lines));
127        }
128
129        private void doHoursPerMonth(List<JobExecution> builds) throws Exception {
130                Map<String, Long> map = asMonthlyBuildHours(builds);
131                List<String> csv = asCsv(map);
132                File file = getCanonicalFile("./target/jenkins/hours.txt");
133                info("create           -> %s", file);
134                write(file, Joiner.on('\n').join(csv));
135        }
136
137        private long sumDuration(List<JobExecution> builds) {
138                long duration = 0;
139                for (JobExecution build : builds) {
140                        duration += build.getDuration();
141                }
142                return duration;
143        }
144
145        private List<String> asCsv(Map<String, Long> map) throws Exception {
146                List<String> lines = newArrayList();
147                Joiner csv = Joiner.on(',');
148                List<String> sorted = natural().sortedCopy(map.keySet());
149                for (String key : sorted) {
150                        long timestamp = parseLong(key);
151                        long millis = map.get(key);
152                        long hours = millis / msPerHour;
153                        String line = csv.join(sdf.format(timestamp), hours + "");
154                        lines.add(line);
155                }
156                return lines;
157        }
158
159        private SimpleDateFormat getSimpleDateFormat(String dateFormat, String timezone) {
160                SimpleDateFormat sdf = new SimpleDateFormat(dateFormat);
161                sdf.setTimeZone(getTimeZone(timezone));
162                return sdf;
163        }
164
165        private long getMonth(long timestamp) throws Exception {
166                return sdf.parse(sdf.format(new Date(timestamp))).getTime();
167        }
168
169        private Map<String, Integer> asMap(List<JobExecution> builds) throws Exception {
170                Map<String, Integer> map = newHashMap();
171                for (JobExecution build : builds) {
172                        long startTime = build.getStartTime();
173                        long duration = build.getDuration();
174                        int minutes = (int) ceil((duration * 1D) / msPerMinute);
175                        for (int i = 0; i < minutes; i++) {
176                                long timestamp = startTime + (msPerMinute * i);
177                                long minute = timestamp / msPerMinute;
178                                String key = minute + "";
179                                Integer count = map.get(key);
180                                map.put(key, (count == null) ? 1 : (count + 1));
181                        }
182                }
183                return map;
184        }
185
186        private Map<String, Long> asMonthlyBuildHours(List<JobExecution> builds) throws Exception {
187                Map<String, Long> map = newHashMap();
188                for (JobExecution build : builds) {
189                        long startTime = build.getStartTime();
190                        long startMonth = getMonth(startTime);
191                        String key = startMonth + "";
192                        Long millis = map.get(key);
193                        map.put(key, (millis == null) ? 0L : (millis + build.getDuration()));
194                }
195                return map;
196        }
197
198        private List<JobExecution> getBuilds(List<Jenkins> jenkins) {
199                List<JobExecution> builds = newArrayList();
200                for (Jenkins server : jenkins) {
201                        for (Job job : server.getJobs()) {
202                                builds.addAll(job.getExecutions());
203                        }
204                }
205                return builds;
206        }
207
208        @Test
209        @Ignore
210        public void test2() {
211                try {
212                        Stopwatch sw = createStarted();
213                        List<String> keys = getKeys(stacks);
214                        List<Jenkins> jenkins = getJenkins(keys);
215                        summarize(jenkins);
216                        info("%s jenkins servers - %s", jenkins.size(), getTime(sw));
217                        for (TypeAndTeam tt : jobsByTypeAndTeam.keySet()) {
218                                List<Job> jobs = jobsByTypeAndTeam.get(tt);
219                                long builds = getTotalExecutions(jobs);
220                                SummaryStatistics ss = getCalculatedDurationStatistics(jobs);
221                                String duration = getTime(new Double(ss.getSum()).longValue());
222                                info("%s:%s  %s %s %s", tt.getTeam(), tt.getType(), jobs.size(), builds, duration);
223                                if (tt.getTeam() == Team.STUDENT && tt.getType() == Type.TEST) {
224                                        summarizeJobs(jobs);
225                                }
226                                if (tt.getTeam() == Team.STUDENT && tt.getType() == Type.OTHER) {
227                                        summarizeJobs(jobs);
228                                }
229                                if (tt.getTeam() == Team.RICE && tt.getType() == Type.TEST) {
230                                        summarizeJobs(jobs);
231                                }
232                        }
233                } catch (Throwable e) {
234                        e.printStackTrace();
235                }
236        }
237
238        protected static void summarizeJobs(List<Job> jobs) {
239                List<Job> sorted = newArrayList(jobs);
240                sort(sorted, JobTimeComparator.INSTANCE);
241                reverse(sorted);
242                List<String> columns = ImmutableList.of("name", "builds", "total time");
243                List<Object[]> rows = newArrayList();
244                for (Job job : sorted) {
245                        long builds = getLastBuildNumber(job);
246                        long time = new Double(getCalculatedDurationStatistics(job).getSummary().getSum()).longValue();
247                        Object[] row = { job.getName(), builds, getTime(time) };
248                        rows.add(row);
249                }
250                logTable(columns, rows);
251        }
252
253        protected static void summarize(List<Jenkins> jenkins) {
254                List<String> columns = ImmutableList.of("host", "jobs", "recorded builds", "total builds", "duration recorded", "total duration", "first job", "last job", "elapsed");
255                List<Object[]> rows = newArrayList();
256                for (Jenkins element : jenkins) {
257                        rows.add(summarize(element));
258                }
259                logTable(columns, rows);
260        }
261
262        protected static Object[] summarize(Jenkins jenkins) {
263                int executions = 0;
264                int totalBuilds = 0;
265                long recordedDuration = 0;
266                long totalDuration = 0;
267
268                SummaryStatistics startTimes = getStartTimeStats(jenkins.getJobs());
269                for (Job job : jenkins.getJobs()) {
270                        updateTypeMap(job);
271                        updateTeamMap(job);
272                        updateTypeAndTeamMap(job);
273                        executions += job.getExecutions().size();
274                        int lastBuildNumber = getLastBuildNumber(job);
275                        recordedDuration += new Double(getRecordedDurationStatistics(job).getSummary().getSum()).longValue();
276                        totalDuration += new Double(getCalculatedDurationStatistics(job).getSummary().getSum()).longValue();
277                        totalBuilds += lastBuildNumber;
278                }
279                long firstStartTime = new Double(startTimes.getMin()).longValue();
280                long lastStartTime = new Double(startTimes.getMax()).longValue();
281                String jobs = FormatUtils.getCount(jenkins.getJobs().size());
282                String execs = FormatUtils.getCount(executions);
283                String builds = FormatUtils.getCount(totalBuilds);
284                String durationRecorded = FormatUtils.getTime(recordedDuration);
285                String durationTotal = FormatUtils.getTime(totalDuration);
286                String firstJob = FormatUtils.getDate(firstStartTime);
287                String lastJob = FormatUtils.getDate(lastStartTime);
288                String elapsed = FormatUtils.getTime(lastStartTime - firstStartTime);
289                return new Object[] { jenkins.getHostname(), jobs, execs, builds, durationRecorded, durationTotal, firstJob, lastJob, elapsed };
290        }
291
292        protected static void updateTypeAndTeamMap(Job job) {
293                Type type = getType(job);
294                Team team = getTeam(job);
295                TypeAndTeam tt = TypeAndTeam.builder().withTeam(team).withType(type).build();
296                List<Job> jobs = jobsByTypeAndTeam.get(tt);
297                if (jobs == null) {
298                        jobs = newArrayList();
299                        jobsByTypeAndTeam.put(tt, jobs);
300                }
301                jobs.add(job);
302        }
303
304        protected static void updateTypeMap(Job job) {
305                Type type = getType(job);
306                List<Job> jobs = jobsByType.get(type);
307                if (jobs == null) {
308                        jobs = newArrayList();
309                        jobsByType.put(type, jobs);
310                }
311                jobs.add(job);
312        }
313
314        protected static void updateTeamMap(Job job) {
315                Team team = getTeam(job);
316                List<Job> jobs = jobsByTeam.get(team);
317                if (jobs == null) {
318                        jobs = newArrayList();
319                        jobsByTeam.put(team, jobs);
320                }
321                jobs.add(job);
322        }
323
324        protected List<Jenkins> getJenkins(List<String> keys) {
325                List<Jenkins> list = newArrayList();
326                for (String key : keys) {
327                        String string = getJson(key);
328                        info("parsing -> json");
329                        Jenkins element = json.readString(string, Jenkins.class);
330                        list.add(element);
331                }
332                return ImmutableList.copyOf(list);
333        }
334
335        protected String getJson(String key) {
336                try {
337                        String url = format("http://%s/%s", BUCKET, key);
338                        File file = getFile(url);
339                        if (file.exists()) {
340                                info("loading -> [%s]", file);
341                                return readFileToString(file, UTF8);
342                        } else {
343                                info("loading -> [%s]", url);
344                                String content = LocationUtils.toString(url, UTF8);
345                                info("creating -> [%s]", file);
346                                writeStringToFile(file, content);
347                                return content;
348                        }
349                } catch (IOException e) {
350                        throw illegalState(e);
351                }
352        }
353
354        protected File getFile(String url) {
355                ResourceLoader loader = new DefaultResourceLoader();
356                Resource resource = loader.getResource(url);
357                String filename = resource.getFilename();
358                return new CanonicalFile("./target/jenkins/" + filename);
359        }
360
361        protected List<String> getKeys(List<String> stacks) {
362                List<String> keys = newArrayList();
363                for (String stack : stacks) {
364                        String key = getKey(REPO, stack, "latest");
365                        keys.add(key);
366                }
367                return ImmutableList.copyOf(keys);
368        }
369
370        protected static void info(String msg, Object... args) {
371                logger.info((args == null) ? msg : format(msg, args));
372        }
373
374        protected static Type getType(Job job) {
375                String name = job.getName();
376                if (name.contains("compile") || name.contains("unit") || name.contains("publish") || name.contains("site") || name.contains("license") || name.contains("nightly")
377                                || name.contains("third-party")) {
378                        return Type.BUILD;
379                }
380                if (name.contains("-ft") || name.contains("integration-test") || name.contains("test-integration") || name.contains("load-test") || name.contains("smoke-test")
381                                || name.contains("test-functional")) {
382                        return Type.TEST;
383                }
384                if (name.contains("deploy") || name.contains("-db-") || name.contains("impex")) {
385                        return Type.DEPLOY;
386                }
387                if (name.contains("release")) {
388                        return Type.RELEASE;
389                }
390                return Type.OTHER;
391        }
392
393        protected static Team getTeam(Job job) {
394                String name = job.getName();
395                if (name.contains("ks-")) {
396                        return Team.STUDENT;
397                }
398                if (name.contains("-ks")) {
399                        return Team.STUDENT;
400                }
401                if (name.contains("student")) {
402                        return Team.STUDENT;
403                }
404                if (name.contains("rice")) {
405                        return Team.RICE;
406                }
407                if (name.contains("rice")) {
408                        return Team.RICE;
409                }
410                if (name.contains("freemarker")) {
411                        return Team.RICE;
412                }
413                if (name.contains("eghm")) {
414                        return Team.RICE;
415                }
416                if (name.contains("commons-beanutils")) {
417                        return Team.RICE;
418                }
419                if (name.contains("ole")) {
420                        return Team.OLE;
421                }
422                if (name.contains("OLE")) {
423                        return Team.OLE;
424                }
425                if (name.contains("mobility")) {
426                        return Team.MOBILITY;
427                }
428                if (name.contains("kpme")) {
429                        return Team.KHR;
430                }
431                if (name.contains("khr")) {
432                        return Team.KHR;
433                }
434                if (name.contains("kdo")) {
435                        return Team.DEVOPS;
436                }
437                if (name.contains("maven-")) {
438                        return Team.DEVOPS;
439                }
440                if (name.contains("update-ci")) {
441                        return Team.DEVOPS;
442                }
443                if (name.contains("kuali-")) {
444                        return Team.DEVOPS;
445                }
446                if (name.contains("ec2-")) {
447                        return Team.DEVOPS;
448                }
449                if (name.contains("devops")) {
450                        return Team.DEVOPS;
451                }
452                if (name.contains("restart-jenkins")) {
453                        return Team.DEVOPS;
454                }
455                if (name.contains("db-ojb")) {
456                        return Team.SHARED;
457                }
458                if (name.contains("impex")) {
459                        return Team.SHARED;
460                }
461                return Team.OTHER;
462        }
463
464}