001package org.kuali.ole.docstore.utility;
002
003import org.json.simple.JSONObject;
004import org.kuali.ole.utility.DateTimeUtil;
005import org.slf4j.Logger;
006import org.slf4j.LoggerFactory;
007
008import java.io.IOException;
009import java.io.StringWriter;
010import java.util.ArrayList;
011import java.util.LinkedHashMap;
012import java.util.LinkedList;
013import java.util.List;
014
015/**
016 * Created by IntelliJ IDEA.
017 * User: Pranitha
018 * Date: 6/11/12
019 * Time: 12:38 PM
020 * To change this template use File | Settings | File Templates.
021 */
022public class BulkIngestStatistics {
023    private static final Logger log = LoggerFactory.getLogger(BulkIngestStatistics.class);
024    public List<BatchIngestStatistics> batchStatisticsList = new ArrayList<BatchIngestStatistics>();
025    public List<FileIngestStatistics> fileIngestStatisticsList = new ArrayList<FileIngestStatistics>();
026
027    public String batchSize;
028    public String splitSize;
029    public boolean isFirstBatch;
030    public boolean isLastBatch;
031    public long fileRecCount;
032    public long commitRecCount;
033
034    private static BulkIngestStatistics instance = new BulkIngestStatistics();
035    public FileIngestStatistics fileIngestStatistics = null;
036
037    private BulkIngestStatistics() {
038
039    }
040
041    public static BulkIngestStatistics getInstance() {
042        if (instance == null) {
043            instance = new BulkIngestStatistics();
044        }
045        return instance;
046    }
047
048
049    public FileIngestStatistics startFile() {
050        FileIngestStatistics fileIngestStatistics = new FileIngestStatistics();
051        batchStatisticsList = fileIngestStatistics.getBatchStatisticsList();
052        fileIngestStatistics.setBatchStatisticsList(batchStatisticsList);
053        fileIngestStatisticsList.add(fileIngestStatistics);
054        return fileIngestStatistics;
055    }
056
057
058    public void clearBulkIngestStatistics() {
059        batchStatisticsList = new ArrayList<BatchIngestStatistics>();
060        fileIngestStatisticsList = new ArrayList<FileIngestStatistics>();
061        isLastBatch = false;
062    }
063
064
065    public BatchIngestStatistics getCurrentBatch() {
066        BatchIngestStatistics batchStatistics = null;
067        if (batchStatisticsList != null && batchStatisticsList.size() > 0) {
068            batchStatistics = batchStatisticsList.get(batchStatisticsList.size() - 1);
069        } else {
070            batchStatistics = new BatchIngestStatistics();
071        }
072        return batchStatistics;
073    }
074
075
076    public List<BatchIngestStatistics> getBatchStatisticsList() {
077        return batchStatisticsList;
078    }
079
080    public void setBatchStatisticsList(List<BatchIngestStatistics> batchStatisticsList) {
081        this.batchStatisticsList = batchStatisticsList;
082    }
083
084    public FileIngestStatistics getFileIngestStatistics() {
085        return fileIngestStatistics;
086    }
087
088    public void setFileIngestStatistics(FileIngestStatistics fileIngestStatistics) {
089        this.fileIngestStatistics = fileIngestStatistics;
090    }
091
092    public List<FileIngestStatistics> getFileIngestStatisticsList() {
093        return fileIngestStatisticsList;
094    }
095
096    public void setFileIngestStatisticsList(List<FileIngestStatistics> fileIngestStatisticsList) {
097        this.fileIngestStatisticsList = fileIngestStatisticsList;
098    }
099
100
101    public String getBatchSize() {
102        return batchSize;
103    }
104
105    public void setBatchSize(String batchSize) {
106        this.batchSize = batchSize;
107    }
108
109    public String getSplitSize() {
110        return splitSize;
111    }
112
113    public void setSplitSize(String splitSize) {
114        this.splitSize = splitSize;
115    }
116
117    public boolean isLastBatch() {
118        return isLastBatch;
119    }
120
121    public boolean isFirstBatch() {
122        return isFirstBatch;
123    }
124
125    public void setFirstBatch(boolean firstBatch) {
126        isFirstBatch = firstBatch;
127    }
128
129    public void setLastBatch(boolean lastBatch) {
130        isLastBatch = lastBatch;
131    }
132
133    public long getFileRecCount() {
134        return fileRecCount;
135    }
136
137    public void setFileRecCount(long fileRecCount) {
138        this.fileRecCount = fileRecCount;
139    }
140
141    public long getCommitRecCount() {
142        return commitRecCount;
143    }
144
145    public void setCommitRecCount(long commitRecCount) {
146        this.commitRecCount = commitRecCount;
147    }
148
149    @Override
150    public String toString() {
151        StringBuilder sb = new StringBuilder();
152        sb.append(BatchIngestStatistics.mertixHeading);
153        long totalBatchTime = 0;
154        long totalTimeToconvertStringToReqObj = 0;
155        long totalTimeToCreateNodesInJcr = 0;
156        long totalTimeToSaveJcrSession = 0;
157        long totalIngestingTime = 0;
158        long totalTimeToConvertXmlToPojo = 0;
159        long totalTimeToConvertToSolrInputDocs = 0;
160        long totalTimeToIndexSolrInputDocs = 0;
161        long totalTimeToSolrCommit = 0;
162        long totalIndexingTime = 0;
163        long totalIngestNIndexTotalTime = 0;
164        long totalTimeToSolrOptimize = 0;
165
166        for (FileIngestStatistics fileStatistics : fileIngestStatisticsList) {
167            long batchTime = 0;
168            long timeToConvertStringToReqObj = 0;
169            long timeToCreateNodesInJcr = 0;
170            long timeToSaveJcrSession = 0;
171            long ingestingTime = 0;
172            long timeToConvertXmlToPojo = 0;
173            long timeToConvertToSolrInputDocs = 0;
174            long timeToIndexSolrInputDocs = 0;
175            long timeToSolrCommit = 0;
176            long indexingTime = 0;
177            long ingestNIndexTotalTime = 0;
178            long timeToSolrOptimize = 0;
179            for (BatchIngestStatistics bS : fileStatistics.getBatchStatisticsList()) {
180                timeToConvertStringToReqObj = timeToConvertStringToReqObj + bS.getTimeToConvertStringToReqObj();
181                totalTimeToconvertStringToReqObj = totalTimeToconvertStringToReqObj + bS
182                        .getTimeToConvertStringToReqObj();
183                timeToCreateNodesInJcr = timeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
184                totalTimeToCreateNodesInJcr = totalTimeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
185                timeToSaveJcrSession = timeToSaveJcrSession + bS.getTimeToSaveJcrSession();
186                totalTimeToSaveJcrSession = totalTimeToSaveJcrSession + bS.getTimeToSaveJcrSession();
187                ingestingTime = ingestingTime + bS.getIngestingTime();
188                totalIngestingTime = totalIngestingTime + bS.getIngestingTime();
189                timeToConvertXmlToPojo = timeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
190                totalTimeToConvertXmlToPojo = totalTimeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
191                timeToConvertToSolrInputDocs = timeToConvertToSolrInputDocs + bS.getTimeToConvertToSolrInputDocs();
192                totalTimeToConvertToSolrInputDocs = totalTimeToConvertToSolrInputDocs + bS
193                        .getTimeToConvertToSolrInputDocs();
194                timeToIndexSolrInputDocs = timeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
195                totalTimeToIndexSolrInputDocs = totalTimeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
196                timeToSolrCommit = timeToSolrCommit + bS.getTimeToSolrCommit();
197                totalTimeToSolrCommit = totalTimeToSolrCommit + bS.getTimeToSolrCommit();
198                indexingTime = indexingTime + bS.getIndexingTime();
199                totalIndexingTime = totalIndexingTime + bS.getIndexingTime();
200                ingestNIndexTotalTime = ingestNIndexTotalTime + bS.getIngestNIndexTotalTime();
201                totalIngestNIndexTotalTime = totalIngestNIndexTotalTime + bS.getIngestNIndexTotalTime();
202                batchTime = batchTime + bS.getBatchTime();
203                totalBatchTime = totalBatchTime + bS.getBatchTime();
204                timeToSolrOptimize = timeToSolrOptimize + bS.getTimeToSolrOptimize();
205                totalTimeToSolrOptimize = totalTimeToSolrOptimize + bS.getTimeToSolrOptimize();
206                bS.buildBatchMetric(sb, bS);
207            }
208            sb.append("\n");
209            sb.append("File Total Time\n");
210            sb.append(" \t");
211            sb.append(DateTimeUtil.formatTime(timeToConvertStringToReqObj));
212            sb.append(" \t");
213            sb.append(DateTimeUtil.formatTime(timeToCreateNodesInJcr));
214            sb.append(" \t");
215            sb.append(DateTimeUtil.formatTime(timeToSaveJcrSession));
216            sb.append(" \t");
217            sb.append(DateTimeUtil.formatTime(ingestingTime));
218            sb.append(" \t");
219            sb.append(DateTimeUtil.formatTime(timeToConvertXmlToPojo));
220            sb.append(" \t");
221            sb.append(DateTimeUtil.formatTime(timeToConvertToSolrInputDocs));
222            sb.append(" \t");
223            sb.append(DateTimeUtil.formatTime(timeToIndexSolrInputDocs));
224            sb.append(" \t");
225            sb.append(DateTimeUtil.formatTime(timeToSolrCommit));
226            sb.append(" \t");
227            sb.append(DateTimeUtil.formatTime(indexingTime));
228            sb.append(" \t");
229            sb.append(DateTimeUtil.formatTime(ingestNIndexTotalTime));
230            sb.append(" \t");
231            sb.append(DateTimeUtil.formatTime(batchTime));
232            sb.append(" \t");
233            sb.append(DateTimeUtil.formatTime(timeToSolrOptimize));
234
235            sb.append("\n");
236            sb.append("Total Time\n");
237            sb.append(" \t");
238            sb.append(DateTimeUtil.formatTime(totalTimeToconvertStringToReqObj));
239            sb.append(" \t");
240            sb.append(DateTimeUtil.formatTime(totalTimeToCreateNodesInJcr));
241            sb.append(" \t");
242            sb.append(DateTimeUtil.formatTime(totalTimeToSaveJcrSession));
243            sb.append(" \t");
244            sb.append(DateTimeUtil.formatTime(totalIngestingTime));
245            sb.append(" \t");
246            sb.append(DateTimeUtil.formatTime(totalTimeToConvertXmlToPojo));
247            sb.append(" \t");
248            sb.append(DateTimeUtil.formatTime(totalTimeToConvertToSolrInputDocs));
249            sb.append(" \t");
250            sb.append(DateTimeUtil.formatTime(totalTimeToIndexSolrInputDocs));
251            sb.append(" \t");
252            sb.append(DateTimeUtil.formatTime(totalTimeToSolrCommit));
253            sb.append(" \t");
254            sb.append(DateTimeUtil.formatTime(totalIndexingTime));
255            sb.append(" \t");
256            sb.append(DateTimeUtil.formatTime(totalIngestNIndexTotalTime));
257            sb.append(" \t");
258            sb.append(DateTimeUtil.formatTime(totalBatchTime));
259            sb.append(" \t");
260            sb.append(DateTimeUtil.formatTime(totalTimeToSolrOptimize));
261
262        }
263
264        return sb.toString();
265    }
266
267
268    public String getJsonString() {
269        StringWriter out = new StringWriter();
270        JSONObject obj = new JSONObject();
271        LinkedHashMap bulkIngestMap = null;
272        LinkedList bulkIngestList = new LinkedList();
273        long totalBatchTime = 0;
274        long totalTimeToconvertStringToReqObj = 0;
275        long totalTimeToCreateNodesInJcr = 0;
276        long totalTimeToSaveJcrSession = 0;
277        long totalIngestingTime = 0;
278        long totalTimeToConvertXmlToPojo = 0;
279        long totalTimeToConvertToSolrInputDocs = 0;
280        long totalTimeToIndexSolrInputDocs = 0;
281        long totalTimeToSolrCommit = 0;
282        long totalIndexingTime = 0;
283        long totalIngestNIndexTotalTime = 0;
284/*        bulkIngestMap = getBulkIngestMap("File Name", "Bulk Ingest Status", "Batch Start Time",
285                                         "Time To Convert String To ReqObj", "Time To Create Nodes In Jcr",
286                                         "Time To Save Jcr Session", "Ingesting Time", "Time To Convert Xml To Pojo",
287                                         "Time To Convert To Solr InputDocs", "Time To Index Solr InputDocs",
288                                         "Time To Solr Commit", "Indexing Time", "Ingest and Index Total Time",
289                                         "Batch Time", "Batch End Time");
290        bulkIngestList.add(bulkIngestMap);*/
291
292        for (FileIngestStatistics fileStatistics : fileIngestStatisticsList) {
293            long batchTime = 0;
294            long timeToconvertStringToReqObj = 0;
295            long timeToCreateNodesInJcr = 0;
296            long timeToSaveJcrSession = 0;
297            long ingestingTime = 0;
298            long timeToConvertXmlToPojo = 0;
299            long timeToConvertToSolrInputDocs = 0;
300            long timeToIndexSolrInputDocs = 0;
301            long timeToSolrCommit = 0;
302            long indexingTime = 0;
303            long ingestNIndexTotalTime = 0;
304            bulkIngestMap = getBulkIngestMap(fileStatistics.getFileName(), fileStatistics.getFileStatus(), "", "", "",
305                    "", "", "", "", "", "", "", "", "", "");
306            bulkIngestList.add(bulkIngestMap);
307            for (BatchIngestStatistics bS : fileStatistics.getBatchStatisticsList()) {
308                bulkIngestMap = getBulkIngestMap("", "", bS.getBatchStartTime(),
309                        DateTimeUtil.formatTime(bS.getTimeToConvertStringToReqObj()),
310                        DateTimeUtil.formatTime(bS.getTimeToCreateNodesInJcr()),
311                        DateTimeUtil.formatTime(bS.getTimeToSaveJcrSession()),
312                        DateTimeUtil.formatTime(bS.getIngestingTime()),
313                        DateTimeUtil.formatTime(bS.getTimeToConvertXmlToPojo()),
314                        DateTimeUtil.formatTime(bS.getTimeToConvertToSolrInputDocs()),
315                        DateTimeUtil.formatTime(bS.getTimeToIndexSolrInputDocs()),
316                        DateTimeUtil.formatTime(bS.getTimeToSolrCommit()),
317                        DateTimeUtil.formatTime(bS.getIndexingTime()),
318                        DateTimeUtil.formatTime(bS.getIngestNIndexTotalTime()),
319                        DateTimeUtil.formatTime(bS.getBatchTime()), bS.getBatchEndTime());
320                timeToconvertStringToReqObj = timeToconvertStringToReqObj + bS.getTimeToConvertStringToReqObj();
321                totalTimeToconvertStringToReqObj = totalTimeToconvertStringToReqObj + bS
322                        .getTimeToConvertStringToReqObj();
323                timeToCreateNodesInJcr = timeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
324                totalTimeToCreateNodesInJcr = totalTimeToCreateNodesInJcr + bS.getTimeToCreateNodesInJcr();
325                timeToSaveJcrSession = timeToSaveJcrSession + bS.getTimeToSaveJcrSession();
326                totalTimeToSaveJcrSession = totalTimeToSaveJcrSession + bS.getTimeToSaveJcrSession();
327                ingestingTime = ingestingTime + bS.getIngestingTime();
328                totalIngestingTime = totalIngestingTime + bS.getIngestingTime();
329                timeToConvertXmlToPojo = timeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
330                totalTimeToConvertXmlToPojo = totalTimeToConvertXmlToPojo + bS.getTimeToConvertXmlToPojo();
331                timeToConvertToSolrInputDocs = timeToConvertToSolrInputDocs + bS.getTimeToConvertToSolrInputDocs();
332                totalTimeToConvertToSolrInputDocs = totalTimeToConvertToSolrInputDocs + bS
333                        .getTimeToConvertToSolrInputDocs();
334                timeToIndexSolrInputDocs = timeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
335                totalTimeToIndexSolrInputDocs = totalTimeToIndexSolrInputDocs + bS.getTimeToIndexSolrInputDocs();
336                timeToSolrCommit = timeToSolrCommit + bS.getTimeToSolrCommit();
337                totalTimeToSolrCommit = totalTimeToSolrCommit + bS.getTimeToSolrCommit();
338                indexingTime = indexingTime + bS.getIndexingTime();
339                totalIndexingTime = totalIndexingTime + bS.getIndexingTime();
340                ingestNIndexTotalTime = ingestNIndexTotalTime + bS.getIngestNIndexTotalTime();
341                totalIngestNIndexTotalTime = totalIngestNIndexTotalTime + bS.getIngestNIndexTotalTime();
342                batchTime = batchTime + bS.getBatchTime();
343                totalBatchTime = totalBatchTime + bS.getBatchTime();
344                bulkIngestList.add(bulkIngestMap);
345            }
346            bulkIngestMap = getBulkIngestMap("", "", "", "", "", "", "", "", "", "", "", "", "", "", "");
347            bulkIngestList.add(bulkIngestMap);
348            bulkIngestMap = getBulkIngestMap("File Total Time", "", "",
349                    DateTimeUtil.formatTime(timeToconvertStringToReqObj),
350                    DateTimeUtil.formatTime(timeToCreateNodesInJcr),
351                    DateTimeUtil.formatTime(timeToSaveJcrSession),
352                    DateTimeUtil.formatTime(ingestingTime),
353                    DateTimeUtil.formatTime(timeToConvertXmlToPojo),
354                    DateTimeUtil.formatTime(timeToConvertToSolrInputDocs),
355                    DateTimeUtil.formatTime(timeToIndexSolrInputDocs),
356                    DateTimeUtil.formatTime(timeToSolrCommit),
357                    DateTimeUtil.formatTime(indexingTime),
358                    DateTimeUtil.formatTime(ingestNIndexTotalTime),
359                    DateTimeUtil.formatTime(batchTime), "");
360            bulkIngestList.add(bulkIngestMap);
361            bulkIngestMap = getBulkIngestMap("Total Time", "", "",
362                    DateTimeUtil.formatTime(totalTimeToconvertStringToReqObj),
363                    DateTimeUtil.formatTime(totalTimeToCreateNodesInJcr),
364                    DateTimeUtil.formatTime(totalTimeToSaveJcrSession),
365                    DateTimeUtil.formatTime(totalIngestingTime),
366                    DateTimeUtil.formatTime(totalTimeToConvertXmlToPojo),
367                    DateTimeUtil.formatTime(totalTimeToConvertToSolrInputDocs),
368                    DateTimeUtil.formatTime(totalTimeToIndexSolrInputDocs),
369                    DateTimeUtil.formatTime(totalTimeToSolrCommit),
370                    DateTimeUtil.formatTime(totalIndexingTime),
371                    DateTimeUtil.formatTime(totalIngestNIndexTotalTime),
372                    DateTimeUtil.formatTime(totalBatchTime), "");
373            bulkIngestList.add(bulkIngestMap);
374        }
375
376        obj.put("rows", bulkIngestList);
377        try {
378            obj.writeJSONString(out);
379        } catch (IOException e) {
380            log.error("Error occurred due to :", e);
381        }
382        return out.toString();
383    }
384
385    public LinkedHashMap getBulkIngestMap(String fileName, String status, String batchStartTime,
386                                          String timeToConvertStringToReqObj, String timeToCreateNodesInJcr,
387                                          String timeToSaveJcrSession, String ingestingTime,
388                                          String timeToConvertXmlToPojo, String timeToConvertToSolrInputDocs,
389                                          String timeToIndexSolrInputDocs, String timeToSolrCommit, String indexingTime,
390                                          String ingestNIndexTotalTime, String batchTime, String batchEndTime) {
391        LinkedHashMap bulkIngestMap = new LinkedHashMap();
392        bulkIngestMap.put("fileName", fileName);
393        bulkIngestMap.put("status", status);
394        bulkIngestMap.put("batchStartTime", batchStartTime);
395        bulkIngestMap.put("timeToConvertStringToReqObj", timeToConvertStringToReqObj);
396        bulkIngestMap.put("timeToCreateNodesInJcr", timeToCreateNodesInJcr);
397        bulkIngestMap.put("timeToSaveJcrSession", timeToSaveJcrSession);
398        bulkIngestMap.put("ingestingTime", ingestingTime);
399        bulkIngestMap.put("timeToConvertXmlToPojo", timeToConvertXmlToPojo);
400        bulkIngestMap.put("timeToConvertToSolrInputDocs", timeToConvertToSolrInputDocs);
401        bulkIngestMap.put("timeToIndexSolrInputDocs", timeToIndexSolrInputDocs);
402        bulkIngestMap.put("timeToSolrCommit", timeToSolrCommit);
403        bulkIngestMap.put("indexingTime", indexingTime);
404        bulkIngestMap.put("ingestNIndexTotalTime", ingestNIndexTotalTime);
405        bulkIngestMap.put("batchTime", batchTime);
406        bulkIngestMap.put("batchEndTime", batchEndTime);
407        return bulkIngestMap;
408    }
409
410}