1 package org.kuali.ole.docstore.utility;
2
3 import org.json.simple.JSONObject;
4 import org.kuali.ole.utility.DateTimeUtil;
5 import org.slf4j.Logger;
6 import org.slf4j.LoggerFactory;
7
8 import java.io.IOException;
9 import java.io.StringWriter;
10 import java.util.ArrayList;
11 import java.util.LinkedHashMap;
12 import java.util.LinkedList;
13 import java.util.List;
14
15
16
17
18
19
20
21
22 public class BulkIngestStatistics {
23 private static final Logger log = LoggerFactory.getLogger(BulkIngestStatistics.class);
24 public List<BatchIngestStatistics> batchStatisticsList = new ArrayList<BatchIngestStatistics>();
25 public List<FileIngestStatistics> fileIngestStatisticsList = new ArrayList<FileIngestStatistics>();
26
27 public String batchSize;
28 public String splitSize;
29 public boolean isFirstBatch;
30 public boolean isLastBatch;
31 public long fileRecCount;
32 public long commitRecCount;
33
34 private static BulkIngestStatistics instance = new BulkIngestStatistics();
35 public FileIngestStatistics fileIngestStatistics = null;
36
37 private BulkIngestStatistics() {
38
39 }
40
41 public static BulkIngestStatistics getInstance() {
42 if (instance == null) {
43 instance = new BulkIngestStatistics();
44 }
45 return instance;
46 }
47
48
49 public FileIngestStatistics startFile() {
50 FileIngestStatistics fileIngestStatistics = new FileIngestStatistics();
51 batchStatisticsList = fileIngestStatistics.getBatchStatisticsList();
52 fileIngestStatistics.setBatchStatisticsList(batchStatisticsList);
53 fileIngestStatisticsList.add(fileIngestStatistics);
54 return fileIngestStatistics;
55 }
56
57
58 public void clearBulkIngestStatistics() {
59 batchStatisticsList = new ArrayList<BatchIngestStatistics>();
60 fileIngestStatisticsList = new ArrayList<FileIngestStatistics>();
61 isLastBatch = false;
62 }
63
64
65 public BatchIngestStatistics getCurrentBatch() {
66 BatchIngestStatistics batchStatistics = null;
67 if (batchStatisticsList != null && batchStatisticsList.size() > 0) {
68 batchStatistics = batchStatisticsList.get(batchStatisticsList.size() - 1);
69 } else {
70 batchStatistics = new BatchIngestStatistics();
71 }
72 return batchStatistics;
73 }
74
75
76 public List<BatchIngestStatistics> getBatchStatisticsList() {
77 return batchStatisticsList;
78 }
79
80 public void setBatchStatisticsList(List<BatchIngestStatistics> batchStatisticsList) {
81 this.batchStatisticsList = batchStatisticsList;
82 }
83
84 public FileIngestStatistics getFileIngestStatistics() {
85 return fileIngestStatistics;
86 }
87
88 public void setFileIngestStatistics(FileIngestStatistics fileIngestStatistics) {
89 this.fileIngestStatistics = fileIngestStatistics;
90 }
91
92 public List<FileIngestStatistics> getFileIngestStatisticsList() {
93 return fileIngestStatisticsList;
94 }
95
96 public void setFileIngestStatisticsList(List<FileIngestStatistics> fileIngestStatisticsList) {
97 this.fileIngestStatisticsList = fileIngestStatisticsList;
98 }
99
100
101 public String getBatchSize() {
102 return batchSize;
103 }
104
105 public void setBatchSize(String batchSize) {
106 this.batchSize = batchSize;
107 }
108
109 public String getSplitSize() {
110 return splitSize;
111 }
112
113 public void setSplitSize(String splitSize) {
114 this.splitSize = splitSize;
115 }
116
117 public boolean isLastBatch() {
118 return isLastBatch;
119 }
120
121 public boolean isFirstBatch() {
122 return isFirstBatch;
123 }
124
125 public void setFirstBatch(boolean firstBatch) {
126 isFirstBatch = firstBatch;
127 }
128
129 public void setLastBatch(boolean lastBatch) {
130 isLastBatch = lastBatch;
131 }
132
133 public long getFileRecCount() {
134 return fileRecCount;
135 }
136
137 public void setFileRecCount(long fileRecCount) {
138 this.fileRecCount = fileRecCount;
139 }
140
141 public long getCommitRecCount() {
142 return commitRecCount;
143 }
144
145 public void setCommitRecCount(long commitRecCount) {
146 this.commitRecCount = commitRecCount;
147 }
148
149 @Override
150 public String toString() {
151 StringBuilder sb = new StringBuilder();
152 sb.append(BatchIngestStatistics.mertixHeading);
153 long totalBatchTime = 0;
154 long totalTimeToconvertStringToReqObj = 0;
155 long totalTimeToCreateNodesInJcr = 0;
156 long totalTimeToSaveJcrSession = 0;
157 long totalIngestingTime = 0;
158 long totalTimeToConvertXmlToPojo = 0;
159 long totalTimeToConvertToSolrInputDocs = 0;
160 long totalTimeToIndexSolrInputDocs = 0;
161 long totalTimeToSolrCommit = 0;
162 long totalIndexingTime = 0;
163 long totalIngestNIndexTotalTime = 0;
164 long totalTimeToSolrOptimize = 0;
165
166 for (FileIngestStatistics fileStatistics : fileIngestStatisticsList) {
167 long batchTime = 0;
168 long timeToConvertStringToReqObj = 0;
169 long timeToCreateNodesInJcr = 0;
170 long timeToSaveJcrSession = 0;
171 long ingestingTime = 0;
172 long timeToConvertXmlToPojo = 0;
173 long timeToConvertToSolrInputDocs = 0;
174 long timeToIndexSolrInputDocs = 0;
175 long timeToSolrCommit = 0;
176 long indexingTime = 0;
177 long ingestNIndexTotalTime = 0;
178 long timeToSolrOptimize = 0;
179 for (BatchIngestStatistics bS : fileStatistics.getBatchStatisticsList()) {
180 timeToConvertStringToReqObj = timeToConvertStringToReqObj + bS.getTimeToConvertStringToReqObj();
181 totalTimeToconvertStringToReqObj = totalTimeToconvertStringToReqObj + bS
182 .getTimeToConvertStringToReqObj();
183 timeToCreateNodesInJcr = timeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
184 totalTimeToCreateNodesInJcr = totalTimeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
185 timeToSaveJcrSession = timeToSaveJcrSession + bS.getTimeToSaveJcrSession();
186 totalTimeToSaveJcrSession = totalTimeToSaveJcrSession + bS.getTimeToSaveJcrSession();
187 ingestingTime = ingestingTime + bS.getIngestingTime();
188 totalIngestingTime = totalIngestingTime + bS.getIngestingTime();
189 timeToConvertXmlToPojo = timeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
190 totalTimeToConvertXmlToPojo = totalTimeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
191 timeToConvertToSolrInputDocs = timeToConvertToSolrInputDocs + bS.getTimeToConvertToSolrInputDocs();
192 totalTimeToConvertToSolrInputDocs = totalTimeToConvertToSolrInputDocs + bS
193 .getTimeToConvertToSolrInputDocs();
194 timeToIndexSolrInputDocs = timeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
195 totalTimeToIndexSolrInputDocs = totalTimeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
196 timeToSolrCommit = timeToSolrCommit + bS.getTimeToSolrCommit();
197 totalTimeToSolrCommit = totalTimeToSolrCommit + bS.getTimeToSolrCommit();
198 indexingTime = indexingTime + bS.getIndexingTime();
199 totalIndexingTime = totalIndexingTime + bS.getIndexingTime();
200 ingestNIndexTotalTime = ingestNIndexTotalTime + bS.getIngestNIndexTotalTime();
201 totalIngestNIndexTotalTime = totalIngestNIndexTotalTime + bS.getIngestNIndexTotalTime();
202 batchTime = batchTime + bS.getBatchTime();
203 totalBatchTime = totalBatchTime + bS.getBatchTime();
204 timeToSolrOptimize = timeToSolrOptimize + bS.getTimeToSolrOptimize();
205 totalTimeToSolrOptimize = totalTimeToSolrOptimize + bS.getTimeToSolrOptimize();
206 bS.buildBatchMetric(sb, bS);
207 }
208 sb.append("\n");
209 sb.append("File Total Time\n");
210 sb.append(" \t");
211 sb.append(DateTimeUtil.formatTime(timeToConvertStringToReqObj));
212 sb.append(" \t");
213 sb.append(DateTimeUtil.formatTime(timeToCreateNodesInJcr));
214 sb.append(" \t");
215 sb.append(DateTimeUtil.formatTime(timeToSaveJcrSession));
216 sb.append(" \t");
217 sb.append(DateTimeUtil.formatTime(ingestingTime));
218 sb.append(" \t");
219 sb.append(DateTimeUtil.formatTime(timeToConvertXmlToPojo));
220 sb.append(" \t");
221 sb.append(DateTimeUtil.formatTime(timeToConvertToSolrInputDocs));
222 sb.append(" \t");
223 sb.append(DateTimeUtil.formatTime(timeToIndexSolrInputDocs));
224 sb.append(" \t");
225 sb.append(DateTimeUtil.formatTime(timeToSolrCommit));
226 sb.append(" \t");
227 sb.append(DateTimeUtil.formatTime(indexingTime));
228 sb.append(" \t");
229 sb.append(DateTimeUtil.formatTime(ingestNIndexTotalTime));
230 sb.append(" \t");
231 sb.append(DateTimeUtil.formatTime(batchTime));
232 sb.append(" \t");
233 sb.append(DateTimeUtil.formatTime(timeToSolrOptimize));
234
235 sb.append("\n");
236 sb.append("Total Time\n");
237 sb.append(" \t");
238 sb.append(DateTimeUtil.formatTime(totalTimeToconvertStringToReqObj));
239 sb.append(" \t");
240 sb.append(DateTimeUtil.formatTime(totalTimeToCreateNodesInJcr));
241 sb.append(" \t");
242 sb.append(DateTimeUtil.formatTime(totalTimeToSaveJcrSession));
243 sb.append(" \t");
244 sb.append(DateTimeUtil.formatTime(totalIngestingTime));
245 sb.append(" \t");
246 sb.append(DateTimeUtil.formatTime(totalTimeToConvertXmlToPojo));
247 sb.append(" \t");
248 sb.append(DateTimeUtil.formatTime(totalTimeToConvertToSolrInputDocs));
249 sb.append(" \t");
250 sb.append(DateTimeUtil.formatTime(totalTimeToIndexSolrInputDocs));
251 sb.append(" \t");
252 sb.append(DateTimeUtil.formatTime(totalTimeToSolrCommit));
253 sb.append(" \t");
254 sb.append(DateTimeUtil.formatTime(totalIndexingTime));
255 sb.append(" \t");
256 sb.append(DateTimeUtil.formatTime(totalIngestNIndexTotalTime));
257 sb.append(" \t");
258 sb.append(DateTimeUtil.formatTime(totalBatchTime));
259 sb.append(" \t");
260 sb.append(DateTimeUtil.formatTime(totalTimeToSolrOptimize));
261
262 }
263
264 return sb.toString();
265 }
266
267
268 public String getJsonString() {
269 StringWriter out = new StringWriter();
270 JSONObject obj = new JSONObject();
271 LinkedHashMap bulkIngestMap = null;
272 LinkedList bulkIngestList = new LinkedList();
273 long totalBatchTime = 0;
274 long totalTimeToconvertStringToReqObj = 0;
275 long totalTimeToCreateNodesInJcr = 0;
276 long totalTimeToSaveJcrSession = 0;
277 long totalIngestingTime = 0;
278 long totalTimeToConvertXmlToPojo = 0;
279 long totalTimeToConvertToSolrInputDocs = 0;
280 long totalTimeToIndexSolrInputDocs = 0;
281 long totalTimeToSolrCommit = 0;
282 long totalIndexingTime = 0;
283 long totalIngestNIndexTotalTime = 0;
284
285
286
287
288
289
290
291
292 for (FileIngestStatistics fileStatistics : fileIngestStatisticsList) {
293 long batchTime = 0;
294 long timeToconvertStringToReqObj = 0;
295 long timeToCreateNodesInJcr = 0;
296 long timeToSaveJcrSession = 0;
297 long ingestingTime = 0;
298 long timeToConvertXmlToPojo = 0;
299 long timeToConvertToSolrInputDocs = 0;
300 long timeToIndexSolrInputDocs = 0;
301 long timeToSolrCommit = 0;
302 long indexingTime = 0;
303 long ingestNIndexTotalTime = 0;
304 bulkIngestMap = getBulkIngestMap(fileStatistics.getFileName(), fileStatistics.getFileStatus(), "", "", "",
305 "", "", "", "", "", "", "", "", "", "");
306 bulkIngestList.add(bulkIngestMap);
307 for (BatchIngestStatistics bS : fileStatistics.getBatchStatisticsList()) {
308 bulkIngestMap = getBulkIngestMap("", "", bS.getBatchStartTime(),
309 DateTimeUtil.formatTime(bS.getTimeToConvertStringToReqObj()),
310 DateTimeUtil.formatTime(bS.getTimeToCreateNodesInJcr()),
311 DateTimeUtil.formatTime(bS.getTimeToSaveJcrSession()),
312 DateTimeUtil.formatTime(bS.getIngestingTime()),
313 DateTimeUtil.formatTime(bS.getTimeToConvertXmlToPojo()),
314 DateTimeUtil.formatTime(bS.getTimeToConvertToSolrInputDocs()),
315 DateTimeUtil.formatTime(bS.getTimeToIndexSolrInputDocs()),
316 DateTimeUtil.formatTime(bS.getTimeToSolrCommit()),
317 DateTimeUtil.formatTime(bS.getIndexingTime()),
318 DateTimeUtil.formatTime(bS.getIngestNIndexTotalTime()),
319 DateTimeUtil.formatTime(bS.getBatchTime()), bS.getBatchEndTime());
320 timeToconvertStringToReqObj = timeToconvertStringToReqObj + bS.getTimeToConvertStringToReqObj();
321 totalTimeToconvertStringToReqObj = totalTimeToconvertStringToReqObj + bS
322 .getTimeToConvertStringToReqObj();
323 timeToCreateNodesInJcr = timeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
324 totalTimeToCreateNodesInJcr = totalTimeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
325 timeToSaveJcrSession = timeToSaveJcrSession + bS.getTimeToSaveJcrSession();
326 totalTimeToSaveJcrSession = totalTimeToSaveJcrSession + bS.getTimeToSaveJcrSession();
327 ingestingTime = ingestingTime + bS.getIngestingTime();
328 totalIngestingTime = totalIngestingTime + bS.getIngestingTime();
329 timeToConvertXmlToPojo = timeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
330 totalTimeToConvertXmlToPojo = totalTimeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
331 timeToConvertToSolrInputDocs = timeToConvertToSolrInputDocs + bS.getTimeToConvertToSolrInputDocs();
332 totalTimeToConvertToSolrInputDocs = totalTimeToConvertToSolrInputDocs + bS
333 .getTimeToConvertToSolrInputDocs();
334 timeToIndexSolrInputDocs = timeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
335 totalTimeToIndexSolrInputDocs = totalTimeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
336 timeToSolrCommit = timeToSolrCommit + bS.getTimeToSolrCommit();
337 totalTimeToSolrCommit = totalTimeToSolrCommit + bS.getTimeToSolrCommit();
338 indexingTime = indexingTime + bS.getIndexingTime();
339 totalIndexingTime = totalIndexingTime + bS.getIndexingTime();
340 ingestNIndexTotalTime = ingestNIndexTotalTime + bS.getIngestNIndexTotalTime();
341 totalIngestNIndexTotalTime = totalIngestNIndexTotalTime + bS.getIngestNIndexTotalTime();
342 batchTime = batchTime + bS.getBatchTime();
343 totalBatchTime = totalBatchTime + bS.getBatchTime();
344 bulkIngestList.add(bulkIngestMap);
345 }
346 bulkIngestMap = getBulkIngestMap("", "", "", "", "", "", "", "", "", "", "", "", "", "", "");
347 bulkIngestList.add(bulkIngestMap);
348 bulkIngestMap = getBulkIngestMap("File Total Time", "", "",
349 DateTimeUtil.formatTime(timeToconvertStringToReqObj),
350 DateTimeUtil.formatTime(timeToCreateNodesInJcr),
351 DateTimeUtil.formatTime(timeToSaveJcrSession),
352 DateTimeUtil.formatTime(ingestingTime),
353 DateTimeUtil.formatTime(timeToConvertXmlToPojo),
354 DateTimeUtil.formatTime(timeToConvertToSolrInputDocs),
355 DateTimeUtil.formatTime(timeToIndexSolrInputDocs),
356 DateTimeUtil.formatTime(timeToSolrCommit),
357 DateTimeUtil.formatTime(indexingTime),
358 DateTimeUtil.formatTime(ingestNIndexTotalTime),
359 DateTimeUtil.formatTime(batchTime), "");
360 bulkIngestList.add(bulkIngestMap);
361 bulkIngestMap = getBulkIngestMap("Total Time", "", "",
362 DateTimeUtil.formatTime(totalTimeToconvertStringToReqObj),
363 DateTimeUtil.formatTime(totalTimeToCreateNodesInJcr),
364 DateTimeUtil.formatTime(totalTimeToSaveJcrSession),
365 DateTimeUtil.formatTime(totalIngestingTime),
366 DateTimeUtil.formatTime(totalTimeToConvertXmlToPojo),
367 DateTimeUtil.formatTime(totalTimeToConvertToSolrInputDocs),
368 DateTimeUtil.formatTime(totalTimeToIndexSolrInputDocs),
369 DateTimeUtil.formatTime(totalTimeToSolrCommit),
370 DateTimeUtil.formatTime(totalIndexingTime),
371 DateTimeUtil.formatTime(totalIngestNIndexTotalTime),
372 DateTimeUtil.formatTime(totalBatchTime), "");
373 bulkIngestList.add(bulkIngestMap);
374 }
375
376 obj.put("rows", bulkIngestList);
377 try {
378 obj.writeJSONString(out);
379 } catch (IOException e) {
380 log.error("Error occurred due to :", e);
381 }
382 return out.toString();
383 }
384
385 public LinkedHashMap getBulkIngestMap(String fileName, String status, String batchStartTime,
386 String timeToConvertStringToReqObj, String timeToCreateNodesInJcr,
387 String timeToSaveJcrSession, String ingestingTime,
388 String timeToConvertXmlToPojo, String timeToConvertToSolrInputDocs,
389 String timeToIndexSolrInputDocs, String timeToSolrCommit, String indexingTime,
390 String ingestNIndexTotalTime, String batchTime, String batchEndTime) {
391 LinkedHashMap bulkIngestMap = new LinkedHashMap();
392 bulkIngestMap.put("fileName", fileName);
393 bulkIngestMap.put("status", status);
394 bulkIngestMap.put("batchStartTime", batchStartTime);
395 bulkIngestMap.put("timeToConvertStringToReqObj", timeToConvertStringToReqObj);
396 bulkIngestMap.put("timeToCreateNodesInJcr", timeToCreateNodesInJcr);
397 bulkIngestMap.put("timeToSaveJcrSession", timeToSaveJcrSession);
398 bulkIngestMap.put("ingestingTime", ingestingTime);
399 bulkIngestMap.put("timeToConvertXmlToPojo", timeToConvertXmlToPojo);
400 bulkIngestMap.put("timeToConvertToSolrInputDocs", timeToConvertToSolrInputDocs);
401 bulkIngestMap.put("timeToIndexSolrInputDocs", timeToIndexSolrInputDocs);
402 bulkIngestMap.put("timeToSolrCommit", timeToSolrCommit);
403 bulkIngestMap.put("indexingTime", indexingTime);
404 bulkIngestMap.put("ingestNIndexTotalTime", ingestNIndexTotalTime);
405 bulkIngestMap.put("batchTime", batchTime);
406 bulkIngestMap.put("batchEndTime", batchEndTime);
407 return bulkIngestMap;
408 }
409
410 }