1 package org.kuali.ole.docstore.utility;
2
3 import org.json.simple.JSONObject;
4 import org.kuali.ole.utility.DateTimeUtil;
5 import org.slf4j.Logger;
6 import org.slf4j.LoggerFactory;
7
8 import java.io.IOException;
9 import java.io.StringWriter;
10 import java.util.ArrayList;
11 import java.util.LinkedHashMap;
12 import java.util.LinkedList;
13 import java.util.List;
14
15
16
17
18
19
20
21
22 public class BulkIngestStatistics {
23 private static final Logger log = LoggerFactory.getLogger(BulkIngestStatistics.class);
24 public List<BatchIngestStatistics> batchStatisticsList = new ArrayList<BatchIngestStatistics>();
25 public List<FileIngestStatistics> fileIngestStatisticsList = new ArrayList<FileIngestStatistics>();
26
27 public String batchSize;
28 public String splitSize;
29 public boolean isFirstBatch;
30 public boolean isLastBatch;
31 public long fileRecCount;
32 public long commitRecCount;
33
34 private static BulkIngestStatistics instance = new BulkIngestStatistics();
35 public FileIngestStatistics fileIngestStatistics = null;
36
37 private BulkIngestStatistics() {
38
39 }
40
41 public static BulkIngestStatistics getInstance() {
42 if (instance == null) {
43 instance = new BulkIngestStatistics();
44 }
45 return instance;
46 }
47
48
49 public FileIngestStatistics startFile() {
50 FileIngestStatistics fileIngestStatistics = new FileIngestStatistics();
51 batchStatisticsList = fileIngestStatistics.getBatchStatisticsList();
52 fileIngestStatistics.setBatchStatisticsList(batchStatisticsList);
53 fileIngestStatisticsList.add(fileIngestStatistics);
54 return fileIngestStatistics;
55 }
56
57
58 public void clearBulkIngestStatistics() {
59 batchStatisticsList = new ArrayList<BatchIngestStatistics>();
60 fileIngestStatisticsList = new ArrayList<FileIngestStatistics>();
61 isLastBatch = false;
62 }
63
64
65 public BatchIngestStatistics getCurrentBatch() {
66 BatchIngestStatistics batchStatistics = null;
67 if (batchStatisticsList != null && batchStatisticsList.size() > 0) {
68 batchStatistics = batchStatisticsList.get(batchStatisticsList.size() - 1);
69 }
70 else {
71 batchStatistics = new BatchIngestStatistics();
72 }
73 return batchStatistics;
74 }
75
76
77 public List<BatchIngestStatistics> getBatchStatisticsList() {
78 return batchStatisticsList;
79 }
80
81 public void setBatchStatisticsList(List<BatchIngestStatistics> batchStatisticsList) {
82 this.batchStatisticsList = batchStatisticsList;
83 }
84
85 public FileIngestStatistics getFileIngestStatistics() {
86 return fileIngestStatistics;
87 }
88
89 public void setFileIngestStatistics(FileIngestStatistics fileIngestStatistics) {
90 this.fileIngestStatistics = fileIngestStatistics;
91 }
92 public List<FileIngestStatistics> getFileIngestStatisticsList() {
93 return fileIngestStatisticsList;
94 }
95
96 public void setFileIngestStatisticsList(List<FileIngestStatistics> fileIngestStatisticsList) {
97 this.fileIngestStatisticsList = fileIngestStatisticsList;
98 }
99
100
101
102 public String getBatchSize() {
103 return batchSize;
104 }
105
106 public void setBatchSize(String batchSize) {
107 this.batchSize = batchSize;
108 }
109
110 public String getSplitSize() {
111 return splitSize;
112 }
113
114 public void setSplitSize(String splitSize) {
115 this.splitSize = splitSize;
116 }
117
118 public boolean isLastBatch() {
119 return isLastBatch;
120 }
121
122 public boolean isFirstBatch() {
123 return isFirstBatch;
124 }
125
126 public void setFirstBatch(boolean firstBatch) {
127 isFirstBatch = firstBatch;
128 }
129
130 public void setLastBatch(boolean lastBatch) {
131 isLastBatch = lastBatch;
132 }
133
134 public long getFileRecCount() {
135 return fileRecCount;
136 }
137
138 public void setFileRecCount(long fileRecCount) {
139 this.fileRecCount = fileRecCount;
140 }
141
142 public long getCommitRecCount() {
143 return commitRecCount;
144 }
145
146 public void setCommitRecCount(long commitRecCount) {
147 this.commitRecCount = commitRecCount;
148 }
149
150 @Override
151 public String toString() {
152 StringBuilder sb = new StringBuilder();
153 sb.append(BatchIngestStatistics.mertixHeading);
154 long totalBatchTime = 0;
155 long totalTimeToconvertStringToReqObj = 0;
156 long totalTimeToCreateNodesInJcr = 0;
157 long totalTimeToSaveJcrSession = 0;
158 long totalIngestingTime = 0;
159 long totalTimeToConvertXmlToPojo = 0;
160 long totalTimeToConvertToSolrInputDocs = 0;
161 long totalTimeToIndexSolrInputDocs = 0;
162 long totalTimeToSolrCommit = 0;
163 long totalIndexingTime = 0;
164 long totalIngestNIndexTotalTime = 0;
165 long totalTimeToSolrOptimize = 0;
166
167 for (FileIngestStatistics fileStatistics : fileIngestStatisticsList) {
168 long batchTime = 0;
169 long timeToConvertStringToReqObj = 0;
170 long timeToCreateNodesInJcr = 0;
171 long timeToSaveJcrSession = 0;
172 long ingestingTime = 0;
173 long timeToConvertXmlToPojo = 0;
174 long timeToConvertToSolrInputDocs = 0;
175 long timeToIndexSolrInputDocs = 0;
176 long timeToSolrCommit = 0;
177 long indexingTime = 0;
178 long ingestNIndexTotalTime = 0;
179 long timeToSolrOptimize = 0;
180 for (BatchIngestStatistics bS : fileStatistics.getBatchStatisticsList()) {
181 timeToConvertStringToReqObj = timeToConvertStringToReqObj + bS.getTimeToConvertStringToReqObj();
182 totalTimeToconvertStringToReqObj = totalTimeToconvertStringToReqObj + bS
183 .getTimeToConvertStringToReqObj();
184 timeToCreateNodesInJcr = timeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
185 totalTimeToCreateNodesInJcr = totalTimeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
186 timeToSaveJcrSession = timeToSaveJcrSession + bS.getTimeToSaveJcrSession();
187 totalTimeToSaveJcrSession = totalTimeToSaveJcrSession + bS.getTimeToSaveJcrSession();
188 ingestingTime = ingestingTime + bS.getIngestingTime();
189 totalIngestingTime = totalIngestingTime + bS.getIngestingTime();
190 timeToConvertXmlToPojo = timeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
191 totalTimeToConvertXmlToPojo = totalTimeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
192 timeToConvertToSolrInputDocs = timeToConvertToSolrInputDocs + bS.getTimeToConvertToSolrInputDocs();
193 totalTimeToConvertToSolrInputDocs = totalTimeToConvertToSolrInputDocs + bS
194 .getTimeToConvertToSolrInputDocs();
195 timeToIndexSolrInputDocs = timeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
196 totalTimeToIndexSolrInputDocs = totalTimeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
197 timeToSolrCommit = timeToSolrCommit + bS.getTimeToSolrCommit();
198 totalTimeToSolrCommit = totalTimeToSolrCommit + bS.getTimeToSolrCommit();
199 indexingTime = indexingTime + bS.getIndexingTime();
200 totalIndexingTime = totalIndexingTime + bS.getIndexingTime();
201 ingestNIndexTotalTime = ingestNIndexTotalTime + bS.getIngestNIndexTotalTime();
202 totalIngestNIndexTotalTime = totalIngestNIndexTotalTime + bS.getIngestNIndexTotalTime();
203 batchTime = batchTime + bS.getBatchTime();
204 totalBatchTime = totalBatchTime + bS.getBatchTime();
205 timeToSolrOptimize = timeToSolrOptimize + bS.getTimeToSolrOptimize();
206 totalTimeToSolrOptimize = totalTimeToSolrOptimize + bS.getTimeToSolrOptimize();
207 bS.buildBatchMetric(sb, bS);
208 }
209 sb.append("\n");
210 sb.append("File Total Time\n");
211 sb.append(" \t");
212 sb.append(DateTimeUtil.formatTime(timeToConvertStringToReqObj));
213 sb.append(" \t");
214 sb.append(DateTimeUtil.formatTime(timeToCreateNodesInJcr));
215 sb.append(" \t");
216 sb.append(DateTimeUtil.formatTime(timeToSaveJcrSession));
217 sb.append(" \t");
218 sb.append(DateTimeUtil.formatTime(ingestingTime));
219 sb.append(" \t");
220 sb.append(DateTimeUtil.formatTime(timeToConvertXmlToPojo));
221 sb.append(" \t");
222 sb.append(DateTimeUtil.formatTime(timeToConvertToSolrInputDocs));
223 sb.append(" \t");
224 sb.append(DateTimeUtil.formatTime(timeToIndexSolrInputDocs));
225 sb.append(" \t");
226 sb.append(DateTimeUtil.formatTime(timeToSolrCommit));
227 sb.append(" \t");
228 sb.append(DateTimeUtil.formatTime(indexingTime));
229 sb.append(" \t");
230 sb.append(DateTimeUtil.formatTime(ingestNIndexTotalTime));
231 sb.append(" \t");
232 sb.append(DateTimeUtil.formatTime(batchTime));
233 sb.append(" \t");
234 sb.append(DateTimeUtil.formatTime(timeToSolrOptimize));
235
236 sb.append("\n");
237 sb.append("Total Time\n");
238 sb.append(" \t");
239 sb.append(DateTimeUtil.formatTime(totalTimeToconvertStringToReqObj));
240 sb.append(" \t");
241 sb.append(DateTimeUtil.formatTime(totalTimeToCreateNodesInJcr));
242 sb.append(" \t");
243 sb.append(DateTimeUtil.formatTime(totalTimeToSaveJcrSession));
244 sb.append(" \t");
245 sb.append(DateTimeUtil.formatTime(totalIngestingTime));
246 sb.append(" \t");
247 sb.append(DateTimeUtil.formatTime(totalTimeToConvertXmlToPojo));
248 sb.append(" \t");
249 sb.append(DateTimeUtil.formatTime(totalTimeToConvertToSolrInputDocs));
250 sb.append(" \t");
251 sb.append(DateTimeUtil.formatTime(totalTimeToIndexSolrInputDocs));
252 sb.append(" \t");
253 sb.append(DateTimeUtil.formatTime(totalTimeToSolrCommit));
254 sb.append(" \t");
255 sb.append(DateTimeUtil.formatTime(totalIndexingTime));
256 sb.append(" \t");
257 sb.append(DateTimeUtil.formatTime(totalIngestNIndexTotalTime));
258 sb.append(" \t");
259 sb.append(DateTimeUtil.formatTime(totalBatchTime));
260 sb.append(" \t");
261 sb.append(DateTimeUtil.formatTime(totalTimeToSolrOptimize));
262
263 }
264
265 return sb.toString();
266 }
267
268
269 public String getJsonString() {
270 StringWriter out = new StringWriter();
271 JSONObject obj = new JSONObject();
272 LinkedHashMap bulkIngestMap = null;
273 LinkedList bulkIngestList = new LinkedList();
274 long totalBatchTime = 0;
275 long totalTimeToconvertStringToReqObj = 0;
276 long totalTimeToCreateNodesInJcr = 0;
277 long totalTimeToSaveJcrSession = 0;
278 long totalIngestingTime = 0;
279 long totalTimeToConvertXmlToPojo = 0;
280 long totalTimeToConvertToSolrInputDocs = 0;
281 long totalTimeToIndexSolrInputDocs = 0;
282 long totalTimeToSolrCommit = 0;
283 long totalIndexingTime = 0;
284 long totalIngestNIndexTotalTime = 0;
285 bulkIngestMap = getBulkIngestMap("File Name", "Bulk Ingest Status", "Batch Start Time",
286 "Time To Convert String To ReqObj", "Time To Create Nodes In Jcr",
287 "Time To Save Jcr Session", "Ingesting Time", "Time To Convert Xml To Pojo",
288 "Time To Convert To Solr InputDocs", "Time To Index Solr InputDocs",
289 "Time To Solr Commit", "Indexing Time", "Ingest and Index Total Time",
290 "Batch Time", "Batch End Time");
291 bulkIngestList.add(bulkIngestMap);
292
293 for (FileIngestStatistics fileStatistics : fileIngestStatisticsList) {
294 long batchTime = 0;
295 long timeToconvertStringToReqObj = 0;
296 long timeToCreateNodesInJcr = 0;
297 long timeToSaveJcrSession = 0;
298 long ingestingTime = 0;
299 long timeToConvertXmlToPojo = 0;
300 long timeToConvertToSolrInputDocs = 0;
301 long timeToIndexSolrInputDocs = 0;
302 long timeToSolrCommit = 0;
303 long indexingTime = 0;
304 long ingestNIndexTotalTime = 0;
305 bulkIngestMap = getBulkIngestMap(fileStatistics.getFileName(), fileStatistics.getFileStatus(), "", "", "",
306 "", "", "", "", "", "", "", "", "", "");
307 bulkIngestList.add(bulkIngestMap);
308 for (BatchIngestStatistics bS : fileStatistics.getBatchStatisticsList()) {
309 bulkIngestMap = getBulkIngestMap("", "", bS.getBatchStartTime(),
310 DateTimeUtil.formatTime(bS.getTimeToConvertStringToReqObj()),
311 DateTimeUtil.formatTime(bS.getTimeToCreateNodesInJcr()),
312 DateTimeUtil.formatTime(bS.getTimeToSaveJcrSession()),
313 DateTimeUtil.formatTime(bS.getIngestingTime()),
314 DateTimeUtil.formatTime(bS.getTimeToConvertXmlToPojo()),
315 DateTimeUtil.formatTime(bS.getTimeToConvertToSolrInputDocs()),
316 DateTimeUtil.formatTime(bS.getTimeToIndexSolrInputDocs()),
317 DateTimeUtil.formatTime(bS.getTimeToSolrCommit()),
318 DateTimeUtil.formatTime(bS.getIndexingTime()),
319 DateTimeUtil.formatTime(bS.getIngestNIndexTotalTime()),
320 DateTimeUtil.formatTime(bS.getBatchTime()), bS.getBatchEndTime());
321 timeToconvertStringToReqObj = timeToconvertStringToReqObj + bS.getTimeToConvertStringToReqObj();
322 totalTimeToconvertStringToReqObj = totalTimeToconvertStringToReqObj + bS
323 .getTimeToConvertStringToReqObj();
324 timeToCreateNodesInJcr = timeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
325 totalTimeToCreateNodesInJcr = totalTimeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
326 timeToSaveJcrSession = timeToSaveJcrSession + bS.getTimeToSaveJcrSession();
327 totalTimeToSaveJcrSession = totalTimeToSaveJcrSession + bS.getTimeToSaveJcrSession();
328 ingestingTime = ingestingTime + bS.getIngestingTime();
329 totalIngestingTime = totalIngestingTime + bS.getIngestingTime();
330 timeToConvertXmlToPojo = timeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
331 totalTimeToConvertXmlToPojo = totalTimeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
332 timeToConvertToSolrInputDocs = timeToConvertToSolrInputDocs + bS.getTimeToConvertToSolrInputDocs();
333 totalTimeToConvertToSolrInputDocs = totalTimeToConvertToSolrInputDocs + bS
334 .getTimeToConvertToSolrInputDocs();
335 timeToIndexSolrInputDocs = timeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
336 totalTimeToIndexSolrInputDocs = totalTimeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
337 timeToSolrCommit = timeToSolrCommit + bS.getTimeToSolrCommit();
338 totalTimeToSolrCommit = totalTimeToSolrCommit + bS.getTimeToSolrCommit();
339 indexingTime = indexingTime + bS.getIndexingTime();
340 totalIndexingTime = totalIndexingTime + bS.getIndexingTime();
341 ingestNIndexTotalTime = ingestNIndexTotalTime + bS.getIngestNIndexTotalTime();
342 totalIngestNIndexTotalTime = totalIngestNIndexTotalTime + bS.getIngestNIndexTotalTime();
343 batchTime = batchTime + bS.getBatchTime();
344 totalBatchTime = totalBatchTime + bS.getBatchTime();
345 bulkIngestList.add(bulkIngestMap);
346 }
347 bulkIngestMap = getBulkIngestMap("", "", "", "", "", "", "", "", "", "", "", "", "", "", "");
348 bulkIngestList.add(bulkIngestMap);
349 bulkIngestMap = getBulkIngestMap("File Total Time", "", "",
350 DateTimeUtil.formatTime(timeToconvertStringToReqObj),
351 DateTimeUtil.formatTime(timeToCreateNodesInJcr),
352 DateTimeUtil.formatTime(timeToSaveJcrSession),
353 DateTimeUtil.formatTime(ingestingTime),
354 DateTimeUtil.formatTime(timeToConvertXmlToPojo),
355 DateTimeUtil.formatTime(timeToConvertToSolrInputDocs),
356 DateTimeUtil.formatTime(timeToIndexSolrInputDocs),
357 DateTimeUtil.formatTime(timeToSolrCommit),
358 DateTimeUtil.formatTime(indexingTime),
359 DateTimeUtil.formatTime(ingestNIndexTotalTime),
360 DateTimeUtil.formatTime(batchTime), "");
361 bulkIngestList.add(bulkIngestMap);
362 bulkIngestMap = getBulkIngestMap("Total Time", "", "",
363 DateTimeUtil.formatTime(totalTimeToconvertStringToReqObj),
364 DateTimeUtil.formatTime(totalTimeToCreateNodesInJcr),
365 DateTimeUtil.formatTime(totalTimeToSaveJcrSession),
366 DateTimeUtil.formatTime(totalIngestingTime),
367 DateTimeUtil.formatTime(totalTimeToConvertXmlToPojo),
368 DateTimeUtil.formatTime(totalTimeToConvertToSolrInputDocs),
369 DateTimeUtil.formatTime(totalTimeToIndexSolrInputDocs),
370 DateTimeUtil.formatTime(totalTimeToSolrCommit),
371 DateTimeUtil.formatTime(totalIndexingTime),
372 DateTimeUtil.formatTime(totalIngestNIndexTotalTime),
373 DateTimeUtil.formatTime(totalBatchTime), "");
374 bulkIngestList.add(bulkIngestMap);
375 }
376
377 obj.put("rows", bulkIngestList);
378 try {
379 obj.writeJSONString(out);
380 }
381 catch (IOException e) {
382 log.error("Error occurred due to :", e);
383 }
384 return out.toString();
385 }
386
387 public LinkedHashMap getBulkIngestMap(String fileName, String status, String batchStartTime,
388 String timeToConvertStringToReqObj, String timeToCreateNodesInJcr,
389 String timeToSaveJcrSession, String ingestingTime,
390 String timeToConvertXmlToPojo, String timeToConvertToSolrInputDocs,
391 String timeToIndexSolrInputDocs, String timeToSolrCommit, String indexingTime,
392 String ingestNIndexTotalTime, String batchTime, String batchEndTime) {
393 LinkedHashMap bulkIngestMap = new LinkedHashMap();
394 bulkIngestMap.put("fileName", fileName);
395 bulkIngestMap.put("status", status);
396 bulkIngestMap.put("batchStartTime", batchStartTime);
397 bulkIngestMap.put("timeToConvertStringToReqObj", timeToConvertStringToReqObj);
398 bulkIngestMap.put("timeToCreateNodesInJcr", timeToCreateNodesInJcr);
399 bulkIngestMap.put("timeToSaveJcrSession", timeToSaveJcrSession);
400 bulkIngestMap.put("ingestingTime", ingestingTime);
401 bulkIngestMap.put("timeToConvertXmlToPojo", timeToConvertXmlToPojo);
402 bulkIngestMap.put("timeToConvertToSolrInputDocs", timeToConvertToSolrInputDocs);
403 bulkIngestMap.put("timeToIndexSolrInputDocs", timeToIndexSolrInputDocs);
404 bulkIngestMap.put("timeToSolrCommit", timeToSolrCommit);
405 bulkIngestMap.put("indexingTime", indexingTime);
406 bulkIngestMap.put("ingestNIndexTotalTime", ingestNIndexTotalTime);
407 bulkIngestMap.put("batchTime", batchTime);
408 bulkIngestMap.put("batchEndTime", batchEndTime);
409 return bulkIngestMap;
410 }
411
412 }