1 package org.kuali.ole.docstore.process;
2
3 import org.apache.commons.lang.time.StopWatch;
4 import org.apache.solr.client.solrj.SolrServer;
5 import org.apache.solr.client.solrj.SolrServerException;
6 import org.apache.solr.common.SolrDocument;
7 import org.apache.solr.common.SolrInputDocument;
8 import org.kuali.ole.DocumentUniqueIDPrefix;
9 import org.kuali.ole.RepositoryBrowser;
10 import org.kuali.ole.RepositoryManager;
11 import org.kuali.ole.docstore.common.document.*;
12 import org.kuali.ole.docstore.common.document.content.instance.FormerIdentifier;
13 import org.kuali.ole.docstore.common.document.content.instance.Instance;
14 import org.kuali.ole.docstore.common.document.content.instance.xstream.InstanceOlemlRecordProcessor;
15 import org.kuali.ole.docstore.common.util.BatchBibTreeDBUtil;
16 import org.kuali.ole.docstore.discovery.service.SolrServerManager;
17 //import org.kuali.ole.docstore.discovery.solr.work.bib.marc.WorkBibMarcDocBuilder;
18 //import org.kuali.ole.docstore.document.rdbms.RdbmsWorkBibMarcDocumentManager;
19 //import org.kuali.ole.docstore.document.rdbms.RdbmsWorkEInstanceDocumentManager;
20 //import org.kuali.ole.docstore.document.rdbms.RdbmsWorkInstanceDocumentManager;
21 import org.kuali.ole.docstore.engine.service.index.solr.BibMarcIndexer;
22 import org.kuali.ole.docstore.engine.service.index.solr.DocumentIndexer;
23 //import org.kuali.ole.docstore.indexer.solr.IndexerService;
24 import org.kuali.ole.docstore.metrics.reindex.ReIndexingBatchStatus;
25 import org.kuali.ole.docstore.metrics.reindex.ReIndexingStatus;
26 import org.kuali.ole.docstore.model.enums.DocCategory;
27 import org.kuali.ole.docstore.model.enums.DocFormat;
28 import org.kuali.ole.docstore.model.enums.DocType;
29 import org.kuali.ole.docstore.engine.service.storage.rdbms.pojo.BibRecord;
30 import org.kuali.ole.docstore.engine.service.storage.rdbms.pojo.EInstanceRecord;
31 import org.kuali.ole.docstore.engine.service.storage.rdbms.pojo.InstanceRecord;
32 //import org.kuali.ole.docstore.model.xmlpojo.ingest.AdditionalAttributes;
33 //import org.kuali.ole.docstore.model.xmlpojo.ingest.Content;
34 //import org.kuali.ole.docstore.model.xmlpojo.ingest.RequestDocument;
35 //import org.kuali.ole.docstore.model.xmlpojo.ingest.ResponseDocument;
36 import org.kuali.ole.docstore.common.document.content.instance.InstanceCollection;
37 import org.kuali.ole.docstore.model.xstream.work.oleml.WorkEInstanceOlemlRecordProcessor;
38 import org.kuali.ole.docstore.service.BeanLocator;
39 //import org.kuali.ole.docstore.service.DocumentIngester;
40 import org.kuali.ole.docstore.service.ServiceLocator;
41 import org.kuali.ole.pojo.OleException;
42 //import org.kuali.ole.repository.CheckoutManager;
43 //import org.kuali.ole.repository.NodeHandler;
44 import org.kuali.rice.core.api.criteria.QueryByCriteria;
45 import org.kuali.rice.krad.data.DataObjectService;
46 import org.kuali.rice.krad.service.BusinessObjectService;
47 import org.kuali.rice.kns.service.KNSServiceLocator;
48 import org.kuali.rice.krad.service.KRADServiceLocator;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
51 import org.kuali.rice.core.api.config.property.ConfigContext;
52
53 import javax.jcr.Binary;
54 import javax.jcr.Node;
55 import javax.jcr.NodeIterator;
56 import javax.jcr.Session;
57 import java.io.ByteArrayInputStream;
58 import java.io.IOException;
59 import java.sql.*;
60 import java.text.DateFormat;
61 import java.text.SimpleDateFormat;
62 import java.util.*;
63 import java.util.Date;
64
65 /**
66 * Class to Rebuild Indexes.
67 *
68 * @author Rajesh Chowdary K
69 * @created May 2, 2012
70 */
71 public class RebuildIndexesHandler
72 implements Runnable {
73
74 private static final Logger LOG = LoggerFactory.getLogger(RebuildIndexesHandler.class);
75 private static RebuildIndexesHandler reBuilder = null;
76 private boolean isRunning = false;
77 private boolean isStop = false;
78 private static final Logger logger = LoggerFactory.getLogger(RebuildIndexesHandler.class);
79 private String docCategory;
80 private String docType;
81 private String docFormat;
82 //private CheckoutManager checkoutManager;
83 // private ReIndexingStatus reIndexingStatus;
84 private int batchSize;
85
86 public static String EXCEPION_FILE_NAME = "";
87 public static String STATUS_FILE_NAME = "";
88 private String filePath = System.getProperty("solr.solr.home");
89 public synchronized void setRunning(boolean running) {
90 isRunning = running;
91 }
92
93 public synchronized void setStop(boolean stop) {
94 isStop = stop;
95 }
96
97 private RebuildIndexesHandler() {
98 //checkoutManager = new CheckoutManager();
99 }
100
101 public static RebuildIndexesHandler getInstance() {
102 if (reBuilder == null) {
103 reBuilder = new RebuildIndexesHandler();
104 }
105 return reBuilder;
106 }
107
108 /**
109 * Method to get running status.
110 *
111 * @return
112 */
113 public synchronized boolean isRunning() {
114 return isRunning;
115 }
116
117 public synchronized boolean isStop() {
118 return isStop;
119 }
120
121 /**
122 * Method to startProcess
123 */
124 public String startProcess(String docCategory, String docType, String docFormat) throws InterruptedException {
125 String status = null;
126 if (isRunning()) {
127 status = "ReIndexing process is already running. Click 'Show Status' button to know the status. ";
128 } else {
129 setRunning(true);
130 setStop(false);
131 status = "ReIndexing process has started. Click 'Show Status' button to know the status. ";
132 ReIndexingStatus reIndexingStatus = ReIndexingStatus.getInstance();
133 reIndexingStatus.reset();
134 if (docCategory == null || docCategory.equals("")) {
135 docCategory = "all";
136 }
137 if (docType == null || docType.equals("")) {
138 docType = "all";
139 }
140 if (docFormat == null || docType.equals("")) {
141 docFormat = "all";
142 }
143 this.docCategory = docCategory;
144 this.docType = docType;
145 this.docFormat = docFormat;
146 Thread reBuilderThread = new Thread(this);
147 reBuilderThread.start();
148 // reBuilderThread.join();
149 setRunning(false);
150 }
151 return status;
152 }
153 public String startProcess(String docCategory, String docType, String docFormat, int batchSize) throws InterruptedException {
154 String status = null;
155 if (isRunning()) {
156 status = "ReIndexing process is already running. Click 'Show Status' button to know the status. ";
157 } else {
158 setRunning(true);
159 setStop(false);
160 status = "ReIndexing process has started. Click 'Show Status' button to know the status. ";
161 ReIndexingStatus reIndexingStatus = ReIndexingStatus.getInstance();
162 reIndexingStatus.reset();
163 if (docCategory == null || docCategory.equals("")) {
164 docCategory = "all";
165 }
166 if (docType == null || docType.equals("")) {
167 docType = "all";
168 }
169 if (docFormat == null || docType.equals("")) {
170 docFormat = "all";
171 }
172 this.docCategory = docCategory;
173 this.docType = docType;
174 this.docFormat = docFormat;
175 this.batchSize = batchSize;
176 Thread reBuilderThread = new Thread(this);
177 reBuilderThread.start();
178 // reBuilderThread.join();
179 setRunning(false);
180 }
181 return status;
182 }
183
184 public String stopProcess() throws Exception {
185 String status = null;
186 if (isRunning()) {
187 status = "ReIndexing process is running. ReIndexing will stop after current batch. ";
188 setStop(true);
189 setRunning(false);
190 } else {
191 status = "ReIndexing process is not running.";
192 }
193 return status;
194
195 }
196
197 public void run() {
198 DocCategoryTypeFormat docCategoryTypeFormat = new DocCategoryTypeFormat();
199 List<String> categoryList = docCategoryTypeFormat.getCategories();
200 List<String> typeList = null;
201 List<String> formatList = null;
202 for (String docCategoryCurr : categoryList) {
203 if (docCategory.equals("all") || docCategory.equals(docCategoryCurr)) {
204 typeList = docCategoryTypeFormat.getDocTypes(docCategoryCurr);
205 for (String docTypeCurr : typeList) {
206 if (docType.equals("all") || docType.equals(docTypeCurr)) {
207 formatList = docCategoryTypeFormat.getDocFormats(docCategoryCurr, docTypeCurr);
208 for (String docFormatCurr : formatList) {
209 if (docFormat.equals("all") || docFormat.equals(docFormatCurr)) {
210 if (!isStop()) {
211 ReIndexingStatus.getInstance()
212 .startDocType(docCategoryCurr, docTypeCurr, docFormatCurr);
213 reIndex(docCategoryCurr, docTypeCurr, docFormatCurr);
214 } else {
215 return;
216 }
217 }
218 }
219 }
220 }
221 }
222 }
223 setRunning(false);
224 }
225
226 private void reIndex(String docCategory, String docType, String docFormat) {
227 Session session = null;
228 setRunning(true);
229 logger.info("Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat + "): ");
230 try {
231 if (docCategory.equals(DocCategory.WORK.getCode())) {
232 if (docType.equals(DocType.BIB.getDescription())) {
233 if (docFormat.equals(DocFormat.MARC.getCode()) || docFormat.equals(DocFormat.DUBLIN_CORE.getCode())
234 || docFormat.equals(DocFormat.DUBLIN_UNQUALIFIED.getCode())) {
235 org.springframework.util.StopWatch stopWatch = new org.springframework.util.StopWatch();
236 stopWatch.start("total time taken");
237 Date date = new Date();
238 EXCEPION_FILE_NAME = "ReindexErrors-" + date.toString() + ".txt";
239 STATUS_FILE_NAME = "ReindexBatchStatus-" + date.toString() + ".txt";
240 BatchBibTreeDBUtil.writeStatusToFile(filePath, RebuildIndexesHandler.EXCEPION_FILE_NAME, "Reindex started at:" + date);
241 BibHoldingItemReindexer bibHoldingItemReindexer = BibHoldingItemReindexer.getInstance();
242 bibHoldingItemReindexer.index(batchSize);
243 date = new Date();
244 BatchBibTreeDBUtil.writeStatusToFile(filePath, RebuildIndexesHandler.EXCEPION_FILE_NAME, "Reindex ended at:" + date);
245 stopWatch.stop();
246 logger.info(stopWatch.prettyPrint());
247 // workBibMarcAndDublinAll(docCategory, docType, docFormat);
248 } else {
249 logger.info(
250 "Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat + "): FAIL");
251 }
252 } else if (docType.equals(DocType.INSTANCE.getDescription())) {
253 if (docFormat.equals(DocFormat.OLEML.getCode())) {
254 workInstanceOLEML(docCategory, docType, docFormat);
255 } else {
256 logger.info(
257 "Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat + "): FAIL");
258 }
259 } else if (docType.equals(DocType.LICENSE.getDescription())) {
260 if (docFormat.equals(DocFormat.ONIXPL.getCode()) || docFormat.equals(DocFormat.PDF.getCode())
261 || docFormat.equals(DocFormat.DOC.getCode())) {
262 workLicense(docCategory, docType, docFormat);
263 } else {
264 logger.info(
265 "Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat + "): FAIL");
266 }
267 } else if (docType.equals(DocType.EINSTANCE.getCode())) {
268 if (docFormat.equals(DocFormat.OLEML.getCode())) {
269 workEInstanceOLEML(docCategory, docType, docFormat);
270 } else {
271 logger.info(
272 "Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat + "): FAIL");
273 }
274 }
275 }
276 } catch (Exception e) {
277 logger.info(e.getMessage(), e);
278 } finally {
279 try {
280 if (isStop) {
281 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
282 } else {
283 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
284 }
285 RepositoryManager.getRepositoryManager().logout(session);
286 } catch (OleException e) {
287 logger.error(e.getMessage(), e);
288 }
289 }
290
291 }
292
293 private void workEInstanceOLEML(String docCategory, String docType, String docFormat) {
294 long totalCount = 0;
295 long nodeCount = 0;
296 // List<RequestDocument> docs = new ArrayList<RequestDocument>();
297 WorkEInstanceOlemlRecordProcessor workEInstanceOlemlRecordProcessor = new WorkEInstanceOlemlRecordProcessor();
298 try {
299 /*RequestDocument rd = new RequestDocument();
300 rd.setCategory(docCategory);
301 rd.setType(docType);
302 rd.setFormat(docFormat);*/
303 List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
304 DataObjectService dataObjectService = KRADServiceLocator.getDataObjectService();
305 List<EInstanceRecord> instanceRecords = (List<EInstanceRecord>) dataObjectService.findMatching(EInstanceRecord.class, QueryByCriteria.Builder.create().build()).getResults();
306 StopWatch loadTimer = new StopWatch();
307 StopWatch batchTimer = new StopWatch();
308 loadTimer.start();
309 batchTimer.start();
310 for (int i = 0; i < instanceRecords.size(); i++) {
311 /*if (docs.size() == ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE) {
312 if (!isStop()) {
313 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
314 //indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
315 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
316 resetTimers(batchTimer, loadTimer);
317 totalCount = 0;
318 logger.info("Rebuild");
319 } else {
320 return;
321 }
322 } else {
323 EInstanceRecord instanceRecord = instanceRecords.get(i);
324 String uuid = DocumentUniqueIDPrefix.getPrefixedId(instanceRecord.getUniqueIdPrefix(), instanceRecord.geteInstanceIdentifier());
325 //RequestDocument requestDocument = buildRequestDocumentForCheckout(docCategory, docType, docFormat, uuid);
326 //ResponseDocument responseDocument = null; //RdbmsWorkEInstanceDocumentManager.getInstance().checkoutContent(requestDocument, dataObjectService);
327 //String content = responseDocument.getContent().getContent();
328 //RequestDocument requestDocumentForIndex = (RequestDocument) rd.clone();
329 *//*requestDocumentForIndex.setAdditionalAttributes(responseDocument.getAdditionalAttributes());
330 requestDocumentForIndex.setId(uuid);
331 requestDocumentForIndex.setUuid(uuid);
332 org.kuali.ole.docstore.model.xmlpojo.work.einstance.oleml.InstanceCollection instanceCollection = workEInstanceOlemlRecordProcessor.fromXML(content);
333 content = workEInstanceOlemlRecordProcessor.toXML(instanceCollection);
334 Content contentObj = new Content();
335 contentObj.setContent(content);
336 contentObj.setContentObject(instanceCollection);
337 requestDocumentForIndex.setContent(contentObj);
338 docs.add(requestDocumentForIndex);*//*
339 totalCount++;
340 }*/
341 }
342 /*if (docs.size() > 0 && !isStop()) {
343 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
344 indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
345 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
346 }*/
347 } catch (Exception e) {
348 /*logger.error(
349 "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed(" + (
350 totalCount - docs.size()) + "), Failed @ batch(" + docs.size() + "): Cause: " + e, e);*/
351 } finally {
352 if (isStop) {
353 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
354 } else {
355 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
356 }
357 }
358
359 }
360
361
362 private void indexBibDocs(List<BibTree> bibTreeList, long records, long recCount,
363 List<ReIndexingBatchStatus> batchStatusList, ReIndexingBatchStatus reIndexingBatchStatus) {
364 StopWatch indexTimer = new StopWatch();
365 DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss:SSS");
366 try {
367 Date startDate = new Date();
368 reIndexingBatchStatus.setBatchStartTime(dateFormat.format(startDate));
369 indexTimer.start();
370 reIndexingBatchStatus.setStatus("Indexing");
371 reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
372 reIndexingBatchStatus.setRecordsProcessed(records);
373 reIndexingBatchStatus.setBatchEndTime(" ");
374 batchStatusList.add(reIndexingBatchStatus);
375 ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
376 DocumentIndexer documentIndexer = BibMarcIndexer.getInstance();
377 BibTrees bibTrees = new BibTrees();
378 bibTrees.getBibTrees().addAll(bibTreeList);
379 documentIndexer.createTrees(bibTrees);
380 //logger.debug(result);
381 indexTimer.stop();
382 Date endDate = new Date();
383 reIndexingBatchStatus.setBatchEndTime(dateFormat.format(endDate));
384 reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
385 reIndexingBatchStatus.setRecordsProcessed(records);
386 reIndexingBatchStatus.setStatus("Done");
387 reIndexingBatchStatus.setRecordsRemaining(recCount - records);
388 ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
389 bibTreeList.clear();
390 } catch (Exception e) {
391 String firstBibId = bibTreeList.get(0).getBib().getId();
392 String lastBibId = bibTreeList.get(bibTreeList.size()-1).getBib().getId();
393 logger.error(
394 "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed(" + (
395 records - bibTreeList.size()) + "), Failed @ bibId( First BibId: " + firstBibId + " : Last BibId : "+ lastBibId +"): Cause: " + e, e);
396 indexTimer.stop();
397 Date endDate = new Date();
398 reIndexingBatchStatus.setBatchEndTime(dateFormat.format(endDate));
399 reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
400 reIndexingBatchStatus.setRecordsProcessed(0L);
401 reIndexingBatchStatus.setStatus("Done");
402 reIndexingBatchStatus.setRecordsRemaining(recCount - records);
403 ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
404 bibTreeList.clear();
405 }
406 }
407
408
409 /*private void indexDocs(List<RequestDocument> docs, long records, long recCount,
410 List<ReIndexingBatchStatus> batchStatusList, ReIndexingBatchStatus reIndexingBatchStatus) {
411 try {
412 StopWatch indexTimer = new StopWatch();
413 DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss:SSS");
414 Date startDate = new Date();
415 reIndexingBatchStatus.setBatchStartTime(dateFormat.format(startDate));
416 indexTimer.start();
417 reIndexingBatchStatus.setStatus("Indexing");
418 reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
419 reIndexingBatchStatus.setRecordsProcessed(records);
420 reIndexingBatchStatus.setBatchEndTime(" ");
421 batchStatusList.add(reIndexingBatchStatus);
422 ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
423 IndexerService indexerService = BeanLocator.getDocstoreFactory().getDocumentIndexManager(docs.get(0).getCategory(), docs.get(0).getType(), docs.get(0).getFormat());
424 String result = indexerService.indexDocuments(docs, false);
425 logger.debug(result);
426 indexTimer.stop();
427 Date endDate = new Date();
428 reIndexingBatchStatus.setBatchEndTime(dateFormat.format(endDate));
429 reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
430 reIndexingBatchStatus.setRecordsProcessed(records);
431 reIndexingBatchStatus.setStatus("Done");
432 reIndexingBatchStatus.setRecordsRemaining(recCount - records);
433 ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
434 docs.clear();
435 } catch (Exception e) {
436 logger.error("Rebuild Indexes Processed(" + (records - docs.size()) + "), Failed @ batch(" + docs.size()
437 + "): Cause: " + e + "\n\tContinuous", e);
438 }
439 }*/
440
441 private void workBibMarcAndDublinAll(String docCategory, String docType, String docFormat) throws SolrServerException, IOException {
442 long totalCount = 0;
443 long nodeCount = 0;
444 int start = 0;
445 String sqlQuery = null;
446 long startTime = 0;
447 long commitEndTime = 0;
448 long commitStartTime = 0;
449 int batchSize = 50000; //ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE;
450 int commitSize = 50000;
451 long endIndexBatch = 0;
452 String dbVendor = ConfigContext.getCurrentContextConfig().getProperty("db.vendor");
453 List<BibTree> bibTrees = new ArrayList<BibTree>();
454 //Map<String, BibTree> bibMap = new HashMap<String, BibTree>();
455 try {
456 String prefix = DocumentUniqueIDPrefix.getPrefix(docCategory, docType, docFormat);
457 Map prefixMap = new HashMap(0);
458 prefixMap.put("uniqueIdPrefix", prefix);
459 startTime = System.currentTimeMillis(); //t1
460
461 BusinessObjectService dataObjectService = KNSServiceLocator.getBusinessObjectService();
462 int bibCount = dataObjectService.countMatching(BibRecord.class, prefixMap);
463 Connection connection = null;
464 PreparedStatement preparedStatement = null;
465 if (bibCount > 0) {
466 connection = getConnection();
467 if (dbVendor.equalsIgnoreCase("mysql")) {
468 sqlQuery = "select * from ole_ds_bib_t b ORDER BY b.bib_id LIMIT ?,?";
469 } else {
470 sqlQuery = "select * from (select b.*,ROWNUM r from OLE_DS_BIB_T b) where r between ? and ?";
471 }
472 preparedStatement = connection.prepareStatement(sqlQuery);
473 }
474 List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
475 StopWatch loadTimer = new StopWatch();
476 StopWatch batchTimer = new StopWatch();
477 loadTimer.start();
478 batchTimer.start();
479
480 for (int i = 0; i < bibCount; i++) {
481 if (bibTrees.size() == batchSize) {
482 if (!isStop()) {
483
484 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
485 indexBibDocs(bibTrees, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
486 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
487 resetTimers(batchTimer, loadTimer);
488 totalCount = 0;
489 logger.info("Rebuild");
490 i = start;
491 if (start % commitSize == 0) {
492 commitStartTime = System.currentTimeMillis(); //t2
493 logger.info("Time elapsed since start ====>>>>>> " + (commitStartTime - startTime)); // t2-t1
494 logger.info("Time elapsed since last commit ====>>>>>> " + (commitStartTime - commitEndTime)); //t2-t3
495 logger.info("commit started ====>>>>>> " + commitStartTime);
496 SolrServer solr = SolrServerManager.getInstance().getSolrServer();
497 solr.commit();
498 logger.info("No..of records committed ====>>>>>> " + start);
499 commitEndTime = System.currentTimeMillis(); //t3
500 logger.info("Time Taken for commit ====>>>>>> " + (commitEndTime - commitStartTime)); //t3-t2
501
502 }
503 } else {
504 return;
505 }
506 } else {
507 if (start < bibCount) {
508 long b2time = System.currentTimeMillis();
509 if (dbVendor.equalsIgnoreCase("mysql")) {
510 preparedStatement.setInt(1, start);
511 preparedStatement.setInt(2, batchSize);
512 } else {
513 preparedStatement.setInt(1, start + 1);
514 preparedStatement.setInt(2, start + batchSize);
515 }
516 ResultSet resultSet = preparedStatement.executeQuery();
517 logger.info("time taking for getting records from DB end======>>>>>" + (System.currentTimeMillis() - b2time));
518 while (resultSet.next()) {
519
520 BibTree bibTree = new BibTree();
521 bibTree.setCategory(docCategory);
522 bibTree.setType(docType);
523 bibTree.setFormat(docFormat);
524 Bib bib = new BibMarc();
525 bib.setCategory(docCategory);
526 bib.setType(docType);
527 bib.setFormat(docFormat);
528 bib.setCreatedBy(resultSet.getString("CREATED_BY"));
529 bib.setCreatedOn(resultSet.getString("DATE_CREATED"));
530 bib.setStaffOnly((resultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE));
531 bib.setContent(resultSet.getString("CONTENT"));
532 bib.setUpdatedBy(resultSet.getString("UPDATED_BY"));
533 bib.setUpdatedOn(resultSet.getString("DATE_UPDATED"));
534 bib.setLastUpdated(resultSet.getString("DATE_UPDATED"));
535 bib.setStatus(resultSet.getString("STATUS"));
536 bib.setStatusUpdatedBy(resultSet.getString("STATUS_UPDATED_BY"));
537 bib.setStatusUpdatedOn(resultSet.getString("STATUS_UPDATED_DATE"));
538 String uuid = DocumentUniqueIDPrefix.getPrefixedId(resultSet.getString("UNIQUE_ID_PREFIX"), resultSet.getString(1));
539 bib.setId(uuid);
540 bib.setLocalId(uuid);
541 bibTree.setBib(bib);
542
543 start++;
544 totalCount++;
545 bibTrees.add(bibTree);
546 }
547 resultSet.close();
548 }
549 }
550
551 }
552 if (bibTrees.size() > 0 && !isStop()) {
553 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
554 indexBibDocs(bibTrees, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
555 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
556 commitStartTime = System.currentTimeMillis();
557 logger.info("commit started : ----> " + commitStartTime);
558 SolrServer solr = SolrServerManager.getInstance().getSolrServer();
559 solr.commit();
560 logger.info("No..of records committed : ----> " + start);
561 commitEndTime = System.currentTimeMillis();
562 logger.info("Time Taken for commit ======>>> " + (commitEndTime - commitStartTime));
563
564 }
565 endIndexBatch = System.currentTimeMillis(); //t1
566 logger.info("Time elapsed since end ====>>>>>> " + endIndexBatch);
567 } catch (Exception e) {
568 logger.error(
569 "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed(" + (
570 totalCount - bibTrees.size()) + "), Failed @ batch(" + bibTrees.size() + "): Cause: " + e, e);
571 } finally {
572 if (isStop) {
573 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
574 } else {
575 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
576 }
577 }
578 }
579
580 private void workInstanceOLEML(String docCategory, String docType, String docFormat) {
581 long totalCount = 0;
582 long nodeCount = 0;
583 //List<RequestDocument> docs = new ArrayList<RequestDocument>();
584 InstanceOlemlRecordProcessor instanceOlemlRecordProcessor = new InstanceOlemlRecordProcessor();
585 try {
586 /*RequestDocument rd = new RequestDocument();
587 rd.setCategory(docCategory);
588 rd.setType(docType);
589 rd.setFormat(docFormat);*/
590 List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
591 DataObjectService dataObjectService = KRADServiceLocator.getDataObjectService();
592 List<InstanceRecord> instanceRecords = (List<InstanceRecord>) dataObjectService.findMatching(InstanceRecord.class,QueryByCriteria.Builder.create().build()).getResults();
593 StopWatch loadTimer = new StopWatch();
594 StopWatch batchTimer = new StopWatch();
595 loadTimer.start();
596 batchTimer.start();
597 for (int i = 0; i < instanceRecords.size(); i++) {
598 /*if (docs.size() == ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE) {
599 if (!isStop()) {
600 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
601 //indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
602 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
603 resetTimers(batchTimer, loadTimer);
604 totalCount = 0;
605 logger.info("Rebuild");
606 } else {
607 return;
608 }
609 } else {
610 InstanceRecord instanceRecord = instanceRecords.get(i);
611 String uuid = DocumentUniqueIDPrefix.getPrefixedId(instanceRecord.getUniqueIdPrefix(), instanceRecord.getInstanceId());
612 //RequestDocument requestDocument = buildRequestDocumentForCheckout(docCategory, docType, docFormat, uuid);
613 //ResponseDocument responseDocument = RdbmsWorkInstanceDocumentManager.getInstance().checkoutContent(requestDocument, dataObjectService);
614 String content = "";//responseDocument.getContent().getContent();
615 //RequestDocument requestDocumentForIndex = (RequestDocument) rd.clone();
616 //requestDocumentForIndex.setAdditionalAttributes(responseDocument.getAdditionalAttributes());
617 //requestDocumentForIndex.setId(uuid);
618 //requestDocumentForIndex.setUuid(uuid);
619 //InstanceCollection instanceCollection = instanceOlemlRecordProcessor.fromXML(content);
620 // logger.debug("REBUILD_INDEXING_LINKING " + ProcessParameters.REBUILD_INDEXING_LINKING);
621 // if (!ProcessParameters.REBUILD_INDEXING_LINKING) {
622 // instanceCollection.getInstance().get(0).getResourceIdentifier().clear();
623 // }
624 content = instanceOlemlRecordProcessor.toXML(instanceCollection);
625 Content contentObj = new Content();
626 contentObj.setContent(content);
627 contentObj.setContentObject(instanceCollection);
628 requestDocumentForIndex.setContent(contentObj);
629 docs.add(requestDocumentForIndex);
630 totalCount++;
631 }*/
632 }
633 /*if (docs.size() > 0 && !isStop()) {
634 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
635 //indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
636 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
637 }*/
638 } catch (Exception e) {
639 /*logger.error(
640 "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed(" + (
641 totalCount - docs.size()) + "), Failed @ batch(" + docs.size() + "): Cause: " + e, e);*/
642 } finally {
643 if (isStop) {
644 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
645 } else {
646 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
647 }
648 }
649 }
650
651 /*private RequestDocument buildRequestDocumentForCheckout(String docCategory, String docType, String docFormat, String uuid) {
652 RequestDocument requestDocument = new RequestDocument();
653 requestDocument.setCategory(docCategory);
654 requestDocument.setType(docType);
655 requestDocument.setFormat(docFormat);
656 requestDocument.setUuid(uuid);
657 return requestDocument;
658 }*/
659
660 /*
661 private void linkingInstanceWithBib(InstanceCollection instanceCollection, Session session, Node fileNode) {
662 for (Instance instance : instanceCollection.getInstance()) {
663 instance.getResourceIdentifier().clear();
664 for (FormerIdentifier frids : instance.getFormerResourceIdentifier()) {
665 try {
666 if (frids != null && frids.getIdentifier() != null &&
667 frids.getIdentifier().getIdentifierValue() != null &&
668 frids.getIdentifier().getIdentifierValue().trim().length() >= 0) {
669 List<SolrDocument> solrBibDocs = ServiceLocator.getIndexerService()
670 .getSolrDocument("SystemControlNumber",
671 frids.getIdentifier()
672 .getIdentifierValue());
673 SolrInputDocument solrInputDocument = new SolrInputDocument();
674 WorkBibMarcDocBuilder marcDocBuilder = new WorkBibMarcDocBuilder();
675 List<SolrInputDocument> solrInputDocs = new ArrayList<SolrInputDocument>();
676 if (solrBibDocs != null && solrBibDocs.size() > 0) {
677 for (SolrDocument solrbibDoc : solrBibDocs) {
678 if (checkApplicability(frids.getIdentifier().getIdentifierValue(),
679 solrbibDoc.getFieldValue("SystemControlNumber"))) {
680
681 compareObjNAddValue(instance.getInstanceIdentifier(),
682 solrbibDoc.getFieldValue("instanceIdentifier"), solrbibDoc,
683 "instanceIdentifier");
684 solrInputDocument = new SolrInputDocument();
685 marcDocBuilder.buildSolrInputDocFromSolrDoc(solrbibDoc, solrInputDocument);
686 solrInputDocs.add(solrInputDocument);
687 String bibId = compareListRString(solrbibDoc.getFieldValue("id"));
688 instance.getResourceIdentifier().add(bibId);
689 modifyContentAddLinkedIdsInDocStore(instance, bibId, session, fileNode);
690 indexSolrDocs(solrInputDocs);
691 }
692 }
693 }
694 }
695 } catch (Exception e) {
696 logger.error("error message" + e.getMessage(), e);
697 }
698 }
699 }
700 }
701 */
702
703 /*private void modifyContentAddLinkedIdsInDocStore(Instance instance, String id, Session session, Node fileNode) {
704
705 try {
706 Node bibNode = getNodeByUUID(session, id);
707 bibNode.setProperty("instanceIdentifier", instance.getInstanceIdentifier());
708 fileNode.setProperty("bibIdentifier", id);
709
710 InstanceOlemlRecordProcessor recordProcessor = new InstanceOlemlRecordProcessor();
711 NodeIterator nodeIterator = fileNode.getNodes();
712 while (nodeIterator.hasNext()) {
713 Node instNode = nodeIterator.nextNode();
714 if (instNode.getName().equalsIgnoreCase("instanceFile")) {
715 InstanceCollection instCol = new InstanceCollection();
716 Instance inst = new Instance();
717 inst.setResourceIdentifier(instance.getResourceIdentifier());
718 inst.setFormerResourceIdentifier(instance.getFormerResourceIdentifier());
719 inst.setExtension(instance.getExtension());
720 inst.setInstanceIdentifier(instance.getInstanceIdentifier());
721 List<Instance> instanceList = new ArrayList<Instance>();
722 instanceList.add(inst);
723 instCol.setInstance(instanceList);
724
725 byte[] documentBytes = recordProcessor.toXML(instCol).getBytes();
726 Binary binary = null;
727 if (documentBytes != null && instNode != null && documentBytes.length > 0) {
728 binary = session.getValueFactory().createBinary(new ByteArrayInputStream(documentBytes));
729 instNode.getNode("jcr:content").setProperty("jcr:data", binary);
730 }
731 }
732 }
733 } catch (Exception e) {
734 logger.error("error while updating Docstore in reindexing Process" + e.getMessage(), e);
735 }
736 }*/
737
738 /*private void indexSolrDocs(List<SolrInputDocument> solrInputDocs) {
739
740 try {
741 ServiceLocator.getIndexerService().indexSolrDocuments(solrInputDocs);
742 logger.info("Linking Bib and Instance Records (" + solrInputDocs.size() + "): ");
743 solrInputDocs.clear();
744 } catch (Exception e) {
745 logger.error(
746 "Linking Bib and Instance Records (" + (solrInputDocs.size()) + "), Failed @ batch(" + solrInputDocs
747 .size() + "): Cause: " + e + "\n\tContinuous", e);
748 }
749 }*/
750
751
752 private boolean checkApplicability(Object value, Object fieldValue) {
753 if (fieldValue instanceof Collection) {
754 for (Object object : (Collection) fieldValue) {
755 if (object.equals(value)) {
756 return true;
757 }
758 }
759 return false;
760 } else {
761 return value.equals(fieldValue);
762 }
763 }
764
765
766 private String compareListRString(Object id) {
767 if (id != null) {
768 if (id instanceof List) {
769 List<String> idList = (List<String>) id;
770 return idList.get(0);
771 } else if (id instanceof String) {
772 String strId = (String) id;
773 return strId;
774 }
775 }
776 return null;
777 }
778
779 private void compareObjNAddValue(String id, Object idObj, SolrDocument solrDoc, String identifier) {
780 if (idObj != null) {
781 if (idObj instanceof List) {
782 List<String> instBibIdList = (List<String>) idObj;
783 if (!instBibIdList.contains(id)) {
784 solrDoc.addField(identifier, id);
785 }
786 } else if (idObj instanceof String) {
787 String instBibId = (String) idObj;
788 if (!instBibId.equalsIgnoreCase(id)) {
789 solrDoc.addField(identifier, id);
790 }
791 }
792 } else {
793 solrDoc.addField(identifier, id);
794 }
795 }
796
797 private void workLicense(String docCategory, String docType, String docFormat) {
798 Session session = null;
799 long totalCount = 0;
800 long nodeCount = 0;
801 //List<RequestDocument> docs = new ArrayList<RequestDocument>();
802 try {
803 session = RepositoryManager.getRepositoryManager().getSession(ProcessParameters.BULK_DEFAULT_USER,
804 ProcessParameters.BULK_DEFUALT_ACTION);
805 /*RequestDocument rd = new RequestDocument();
806 rd.setCategory(docCategory);
807 rd.setType(docType);
808 rd.setFormat(docFormat);*/
809 //DocumentIngester docIngester = new DocumentIngester();
810 Node nodeFormat = null;//docIngester.getStaticFormatNode(rd, session);
811 NodeIterator nodesL1 = nodeFormat.getNodes();
812 List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
813 StopWatch loadTimer = new StopWatch();
814 StopWatch batchTimer = new StopWatch();
815 loadTimer.start();
816 RepositoryBrowser repositoryBrowser = new RepositoryBrowser();
817 while (nodesL1.hasNext()) {
818 Node nodeL1 = nodesL1.nextNode();
819 NodeIterator nodesFile = nodeL1.getNodes();
820 nodeCount = nodesFile.getSize();
821 batchTimer.start();
822 while (nodesFile.hasNext()) {
823 /* if (docs.size() == ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE && !isStop()) {
824 if (!isStop()) {
825 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
826 //indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
827 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
828 resetTimers(batchTimer, loadTimer);
829 totalCount = 0;
830 logger.info("Rebuild");
831 } else {
832 return;
833 }
834 } else {
835
836 Node fileNode = nodesFile.nextNode();
837 String content = null;
838 if (docFormat.equals(DocFormat.ONIXPL.getCode())) {
839 content = "";//checkoutManager.getData(fileNode);
840 } else if (docFormat.equals(DocFormat.PDF.getCode()) || docFormat
841 .equals(DocFormat.DOC.getCode())) {
842 *//*content = checkoutManager
843 .checkOutBinary(fileNode.getIdentifier(), ProcessParameters.BULK_DEFAULT_USER,
844 ProcessParameters.BULK_DEFUALT_ACTION, docFormat);*//*
845 }
846 *//*RequestDocument reqDoc = (RequestDocument) rd.clone();
847 reqDoc.setId(fileNode.getIdentifier());
848 reqDoc.setUuid(fileNode.getIdentifier());
849 Content contentObj = new Content();
850 contentObj.setContent(content);
851 reqDoc.setContent(contentObj);
852 docs.add(reqDoc);
853 totalCount++;*//*
854 }*/
855 }
856 }
857 /*if (docs.size() > 0 && !isStop()) {
858 ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
859 //indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
860 indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
861 }*/
862 } catch (Exception e) {
863 /*logger.error(
864 "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed(" + (
865 totalCount - docs.size()) + "), Failed @ batch(" + docs.size() + "): Cause: " + e, e);*/
866 } finally {
867 try {
868 if (isStop) {
869 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
870 } else {
871 ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
872 }
873 RepositoryManager.getRepositoryManager().logout(session);
874 } catch (OleException e) {
875 logger.error(e.getMessage(), e);
876 }
877 }
878 }
879
880 private void resetTimers(StopWatch batchTimer, StopWatch loadTimer) {
881 batchTimer.reset();
882 batchTimer.start();
883 loadTimer.reset();
884 loadTimer.start();
885 }
886
887 private void indexAfterParams(StopWatch batchTimer, ReIndexingBatchStatus reIndexingBatchStatus,
888 List<ReIndexingBatchStatus> batchStatusList) {
889 batchTimer.stop();
890 reIndexingBatchStatus.setBatchTotalTime(batchTimer.toString());
891 ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
892 }
893
894 private ReIndexingBatchStatus indexBeforeParams(StopWatch loadTimer) {
895 loadTimer.stop();
896 ReIndexingBatchStatus reIndexingBatchStatus = new ReIndexingBatchStatus();
897 reIndexingBatchStatus.setBatchTotalTime(" ");
898 reIndexingBatchStatus.setBatchLoadTime(loadTimer.toString());
899 return reIndexingBatchStatus;
900 }
901
902 /*private Node getNodeByUUID(Session newSession, String uuid) throws OleException {
903 return new NodeHandler().getNodeByUUID(newSession, uuid);
904 }*/
905
906 public Connection getConnection() {
907 Connection connection = null;
908 try {
909 /* InputStream in =getClass().getResourceAsStream("/mysql.properties");
910 Properties properties = new Properties();
911 properties.load(in);*/
912 String connectionUrl = ConfigContext.getCurrentContextConfig().getProperty("datasource.url");
913 String userName = ConfigContext.getCurrentContextConfig().getProperty("datasource.username");
914 String passWord = ConfigContext.getCurrentContextConfig().getProperty("datasource.password");
915 String driverName = ConfigContext.getCurrentContextConfig().getProperty("jdbc.driver");
916 Class.forName(driverName);
917 connection = DriverManager.getConnection(connectionUrl, userName, passWord);
918 } catch (Exception e) {
919 LOG.error("Exception : ", e);
920 }
921 return connection;
922 }
923
924 public String showStatus() {
925 BibHoldingItemReindexer bibHoldingItemReindexer = BibHoldingItemReindexer.getInstance();
926 return bibHoldingItemReindexer.showStats();
927 }
928 }