1 package org.kuali.ole.docstore.discovery.service;
2
3 import org.apache.commons.io.FileUtils;
4 import org.apache.commons.lang.StringUtils;
5 import org.apache.solr.client.solrj.SolrQuery;
6 import org.apache.solr.client.solrj.SolrServer;
7 import org.apache.solr.client.solrj.response.QueryResponse;
8 import org.apache.solr.common.SolrDocument;
9 import org.apache.solr.common.SolrDocumentList;
10 import org.apache.solr.common.SolrInputDocument;
11 import org.junit.Before;
12 import org.junit.Ignore;
13 import org.junit.Test;
14 import org.kuali.ole.docstore.common.document.content.instance.Instance;
15 import org.kuali.ole.docstore.common.document.content.instance.InstanceCollection;
16 import org.kuali.ole.docstore.common.document.content.instance.Item;
17 import org.kuali.ole.docstore.common.document.content.instance.xstream.InstanceOlemlRecordProcessor;
18 import org.kuali.ole.docstore.discovery.BaseTestCase;
19 import org.kuali.ole.docstore.indexer.solr.DocumentIndexerManagerFactory;
20 import org.kuali.ole.docstore.indexer.solr.IndexerService;
21 import org.kuali.ole.docstore.indexer.solr.WorkBibDocumentIndexer;
22 import org.kuali.ole.docstore.model.enums.DocCategory;
23 import org.kuali.ole.docstore.model.enums.DocFormat;
24 import org.kuali.ole.docstore.model.enums.DocType;
25 import org.kuali.ole.docstore.model.xmlpojo.ingest.Content;
26 import org.kuali.ole.docstore.model.xmlpojo.ingest.Request;
27 import org.kuali.ole.docstore.model.xmlpojo.ingest.RequestDocument;
28 import org.kuali.ole.docstore.common.document.content.instance.OleHoldings;
29 import org.kuali.ole.docstore.model.xstream.ingest.RequestHandler;
30
31 import java.io.File;
32 import java.net.URL;
33 import java.util.ArrayList;
34 import java.util.Iterator;
35 import java.util.List;
36
37 import static org.junit.Assert.*;
38
39
40
41
42
43
44 public class IndexerService_UT extends BaseTestCase {
45
46 private IndexerService indexerService = null;
47 private int oleItemCount = 0;
48
49 public IndexerService_UT() {
50
51 }
52
53 @Before
54 public void setUp() throws Exception {
55 super.setUp();
56 }
57
58
59
60
61 @Ignore
62 @Test
63 public void testInstanceBulkIndex() {
64 String result = "";
65 try {
66
67 printInitialLog();
68 List<String> ids = new ArrayList<String>();
69 ids.add("1234");
70 ids.add("14445");
71 ids.add("144446");
72 ids.add("144447");
73 ids.add("144448");
74 ids.add("144449");
75 ids.add("144450");
76 ids.add("144451");
77 ids.add("144452");
78 ids.add("144453");
79 ids.add("144454");
80 ids.add("144455");
81 ids.add("144456");
82 String filePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
83 URL resource = getClass().getResource(filePath);
84 File file = new File(resource.toURI());
85 String instanceRequestContent = readFile(file);
86 RequestHandler requestHandler = new RequestHandler();
87 Request request = requestHandler.toObject(instanceRequestContent);
88 List<RequestDocument> marcRequestDocumentList = request.getRequestDocuments();
89 LOG.info("requestDocuments size " + marcRequestDocumentList.size());
90 int i = 0;
91 for (RequestDocument requestDocument : marcRequestDocumentList) {
92 if (requestDocument.getUuid() == null)
93 requestDocument.setUuid(ids.get(i));
94 i++;
95 }
96 result = getIndexerService(marcRequestDocumentList.get(0)).bulkIndexDocuments(marcRequestDocumentList, true);
97 for (RequestDocument doc : marcRequestDocumentList) {
98 LOG.info("Doc Marc : " + doc.getUuid());
99 }
100 LOG.info("RESULT: " + result);
101 if (!result.toLowerCase().contains("success"))
102 fail("Bulk Index Failed.");
103
104 filePath = "/work/instance/oleml/OLE-Instance-bulkIngest-IU-Set1-split.xml";
105 resource = getClass().getResource(filePath);
106 file = new File(resource.toURI());
107 instanceRequestContent = readFile(file);
108 requestHandler = new RequestHandler();
109 Request instanceRequest = requestHandler.toObject(instanceRequestContent);
110 List<RequestDocument> newInstanceRequestDocumentList = new ArrayList<RequestDocument>();
111 List<RequestDocument> instanceRequestDocumentList = instanceRequest.getRequestDocuments();
112 LOG.info("requestDocuments size " + instanceRequestDocumentList.size());
113 for (RequestDocument requestDocument : instanceRequestDocumentList) {
114 if (requestDocument.getUuid() == null)
115 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
116 if (requestDocument.getContent().getContentObject() == null) {
117 InstanceOlemlRecordProcessor oleMlProcessor = new InstanceOlemlRecordProcessor();
118 InstanceCollection instanceCollection = oleMlProcessor.fromXML(requestDocument.getContent()
119 .getContent());
120 for (Instance instance : instanceCollection.getInstance()) {
121 if (instance.getInstanceIdentifier() == null)
122 instance.setInstanceIdentifier(requestDocument.getUuid());
123 if (instance.getOleHoldings().getHoldingsIdentifier() == null)
124 instance.getOleHoldings().setHoldingsIdentifier(String.valueOf(Math.round(Math.random())));
125 for (Item item : instance.getItems().getItem())
126 if (item.getItemIdentifier() == null)
127 item.setItemIdentifier(String.valueOf(Math.round(Math.random())));
128 }
129 requestDocument.getContent().setContentObject(instanceCollection);
130 newInstanceRequestDocumentList.add(requestDocument);
131 }
132
133 }
134 result = getIndexerService(newInstanceRequestDocumentList.get(0)).bulkIndexDocuments(newInstanceRequestDocumentList, true);
135 LOG.info("RESULT: " + result);
136 if (!result.toLowerCase().contains("success"))
137 fail("Bulk Index Failed.");
138
139 for (RequestDocument bibDoc : marcRequestDocumentList) {
140 List<SolrDocument> bibSolrDocs = getIndexerService(bibDoc).getSolrDocumentBySolrId(bibDoc.getUuid());
141 if (bibSolrDocs == null || bibSolrDocs.size() <= 0)
142 fail(" Bulk Indexing Failed.");
143 for (SolrDocument outDoc : bibSolrDocs) {
144 if (outDoc.getFieldValues("instanceIdentifier") != null)
145 for (Object instanceId : outDoc.getFieldValues("instanceIdentifier")) {
146 List<SolrDocument> linkedInstanceDocs = getIndexerService(bibDoc).getSolrDocumentBySolrId(instanceId
147 .toString());
148 if (linkedInstanceDocs.size() <= 0)
149 fail(" Bulk Indexing Failed.");
150 else
151 for (SolrDocument instDoc : linkedInstanceDocs)
152 if (!instDoc.getFieldValues("bibIdentifier").contains(bibDoc.getUuid()))
153 fail(" Bulk Indexing Failed.");
154 }
155 }
156 }
157
158 printFinalLog();
159
160 } catch (Exception e) {
161 LOG.error(e.getMessage() , e);
162 fail("Bulk Index Failed.");
163 }
164 }
165
166 @Test
167 public void testDublinBulkIndex() throws Exception {
168 try {
169
170 printInitialLog();
171 String filePath = "/work/bib/dublin/DublinQ-Test1.xml";
172 List<RequestDocument> dublinRequestDocumentList = getRequestDocumentList(filePath);
173 String result = getIndexerService(dublinRequestDocumentList.get(0)).bulkIndexDocuments(dublinRequestDocumentList, true);
174 LOG.info("result:" + result);
175
176 printFinalLog();
177
178 } catch (Exception e) {
179 LOG.error(e.getMessage() , e);
180 }
181 }
182
183 @Test
184 public void testDublinUnqBulkIndex() throws Exception {
185 try {
186
187 printInitialLog();
188 String filePath = "/work/bib/dublin/unqualified/DublinUnq-Test1.xml";
189 List<RequestDocument> dublinUnqRequestDocumentList = getRequestDocumentList(filePath);
190 System.out.println("size:" + dublinUnqRequestDocumentList.size());
191 String result = getIndexerService(dublinUnqRequestDocumentList.get(0)).bulkIndexDocuments(dublinUnqRequestDocumentList, true);
192 LOG.info("result:" + result);
193
194 printFinalLog();
195
196 } catch (Exception e) {
197 LOG.error(e.getMessage() , e);
198 }
199 }
200
201 public List<RequestDocument> getRequestDocumentList(String path) throws Exception {
202 URL resource = getClass().getResource(path);
203 File file = new File(resource.toURI());
204 String requestContent = readFile(file);
205 RequestHandler requestHandler = new RequestHandler();
206 Request request = requestHandler.toObject(requestContent);
207 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
208 int i = 1000004;
209 for (RequestDocument requestDocument : requestDocumentList) {
210 if (requestDocument.getUuid() == null) {
211 requestDocument.setUuid("" + i + "");
212 }
213 i++;
214 }
215 return requestDocumentList;
216 }
217
218 @Test
219 public void testDeleteDocuments() throws Exception {
220 try {
221
222 printInitialLog();
223 List<String> UUIDList = new ArrayList<String>();
224
225
226 String inputFilePath = "/work/instance/oleml/Instance-Request.xml";
227 Request request = getRequest(inputFilePath);
228 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
229 for (RequestDocument requestDocument : requestDocumentList) {
230 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
231 UUIDList.add(requestDocument.getUuid());
232 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
233 LOG.info("result:" + result);
234
235
236 }
237
238 printLogInfo("Record count after indexing:" + getRecordCount());
239 getIndexerService(requestDocumentList.get(0)).deleteDocuments(DocCategory.WORK.getCode(), UUIDList);
240 printLogInfo("Record count after deleting:" + getRecordCount());
241
242
243 printFinalLog();
244
245 } catch (Exception e) {
246 LOG.info("Exception+" + e.getMessage(), e);
247 LOG.error(e.getMessage() , e);
248 }
249 }
250
251 @Test
252 public void testDeleteDocument() throws Exception {
253 try {
254
255 printInitialLog();
256
257 String inputFilePath = "/work/instance/oleml/Instance-Request.xml";
258 Request request = getRequest(inputFilePath);
259 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
260 RequestDocument requestDocument = requestDocumentList.get(0);
261 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
262 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
263 LOG.info("result:" + result);
264 printLogInfo("Record count after indexing:" + getRecordCount());
265
266 getIndexerService(requestDocument).deleteDocument(DocCategory.WORK.getCode(), requestDocument.getUuid());
267
268 printLogInfo("Record count after deleting:" + getRecordCount());
269
270
271 printFinalLog();
272
273 } catch (Exception e) {
274 LOG.info("Exception+" + e.getMessage(), e);
275 }
276
277 }
278 @Ignore
279 @Test
280 public void testIndexDocumentsFromFilesBySolrDoc() throws Exception {
281
282 printLogInfo("======");
283 printLogInfo("Total record count before cleanup = " + getRecordCount());
284 cleanUpData();
285 printLogInfo("Total record count after cleanup = " + getRecordCount());
286
287
288 LOG.info("testIndexDocumentsFromFileBySolrDoc() : Marc Records Test");
289 String docCategory = DocCategory.WORK.getCode();
290 String docType = DocType.BIB.getDescription();
291 String docFormat = DocFormat.MARC.getCode();
292 String requestFile
293 = "/bib/bib/marc/marc-one-record-xstream.xml";
294 invokeIndexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, requestFile);
295
296
297 LOG.info("testIndexDocumentsFromFileBySolrDoc() : Dublin Records Test");
298 docFormat = DocFormat.DUBLIN_CORE.getCode();
299 requestFile = "/bib/bib/dublin/Bib-Bib-DublinQ-Test1.xml";
300 invokeIndexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, requestFile);
301
302
303 LOG.info("testIndexDocumentsFromFileBySolrDoc() : Dublin Unqualified Records Test");
304 requestFile = "/bib/bib/dublin/unqualified/Bib-Bib-DublinUnQ-Test1.xml";
305 docFormat = DocFormat.DUBLIN_UNQUALIFIED.getCode();
306 invokeIndexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, requestFile);
307
308
309 printLogInfo("Total record count after the test = " + getRecordCount());
310 printLogInfo("======");
311 }
312 @Ignore
313 @Test
314
315
316
317
318
319
320
321 public void testIndexDocumentsFromDirBySolrDocWithConfirm() throws Exception {
322
323 printLogInfo("======");
324 printLogInfo("Total record count before cleanup = " + getRecordCount());
325 cleanUpData();
326 printLogInfo("Total record count after cleanup = " + getRecordCount());
327 String dirPath = "/bib/bib/marc/request";
328 invokeIndexDocumentsFromDirBySolrDoc(dirPath);
329
330 printLogInfo("Total record count after the test = " + getRecordCount());
331 printLogInfo("======");
332 }
333
334 @Test
335 public void testIndexDocumentsWithInvalidDirectoryORFile() throws Exception {
336 try {
337 String docCategory = DocCategory.WORK.getCode();
338 String docType = DocType.BIB.getDescription();
339 String docFormat = DocFormat.MARC.getCode();
340
341 printInitialLog();
342 String dirPath = "/bib/bib/marc/request/invalidDir";
343 getIndexerService(docCategory, docType, docFormat).indexDocumentsFromDirBySolrDoc(docCategory, docType, docFormat, dirPath);
344
345 dirPath = "/work/emptyDir";
346 getIndexerService(docCategory, docType, docFormat).indexDocumentsFromDirBySolrDoc(docCategory, docType, docFormat, dirPath);
347
348 printFinalLog();
349 } catch (Exception e) {
350 LOG.error(e.getMessage() , e);
351 }
352
353 }
354
355 public void invokeIndexDocumentsFromDirBySolrDoc(String path) {
356 try {
357 printLogInfo("-----");
358 printLogInfo("Input Dir/File : " + path);
359 String docCategory = DocCategory.WORK.getCode();
360 String docType = DocType.BIB.getDescription();
361 String docFormat = DocFormat.MARC.getCode();
362 URL resource = getClass().getResource(path);
363 File file = new File(resource.toURI());
364 String dataDir = file.getAbsolutePath();
365 if (null == dataDir) {
366 LOG.info("dataDir is null.");
367 System.out.println("dataDir is null.");
368 return;
369 }
370 String result = getIndexerService(docCategory, docType, docFormat).indexDocumentsFromDirBySolrDoc(docCategory, docType, docFormat, dataDir);
371 LOG.info("result=" + result);
372 System.out.println("result=" + result);
373 assertNotNull(result);
374 assertTrue(result.contains(IndexerService.SUCCESS));
375 printLogInfo("-----");
376 } catch (Exception e) {
377 LOG.error("Exception:", e);
378 }
379 }
380 @Ignore
381 @Test
382 public void testIndexDocumentFromStringBySolr() throws Exception {
383
384 printInitialLog();
385 String requestFile = "/bib/bib/marc/marc-one-record-xstream.xml";
386 invokeIndexDocumentsFromStringBySolrDoc(requestFile);
387
388 printFinalLog();
389
390 }
391
392 public void invokeIndexDocumentsFromStringBySolrDoc(String requestFile) throws Exception {
393 try {
394 printLogInfo("-----");
395 printLogInfo("Input Dir/File : " + requestFile);
396 String docCategory = DocCategory.WORK.getCode();
397 String docType = DocType.BIB.getDescription();
398 String docFormat = DocFormat.MARC.getCode();
399 URL resource = getClass().getResource(requestFile);
400 File file = new File(resource.toURI());
401 String requestContent = FileUtils.readFileToString(file);
402 getIndexerService(docCategory, docType, docFormat).indexDocumentsFromStringBySolrDoc(docCategory, docType, docFormat, requestContent);
403 printLogInfo("-----");
404 } catch (Exception e) {
405 LOG.error("Exception:", e);
406 }
407
408 }
409
410 public void invokeIndexDocumentsFromFileBySolrDoc(String docCategory, String docType, String docFormat,
411 String requestFile) throws Exception {
412 URL resource = getClass().getResource(requestFile);
413 File file = new File(resource.toURI());
414 String path = file.getAbsolutePath();
415 String result = getIndexerService(docCategory, docType, docFormat).indexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, path);
416 System.out.println("result=" + result);
417 LOG.info("result=" + result);
418 assertNotNull(result);
419 assertTrue(result.contains(IndexerService.SUCCESS));
420 }
421
422 @Test
423
424
425
426
427 public void deleteTestDataWithConfirm() {
428 String confirmDelete = System.getProperty("confirmDelete");
429 if (confirmDelete != null) {
430 boolean confirm = Boolean.valueOf(confirmDelete);
431 if (!confirm) {
432 LOG.info("Delete is not confirmed through a system property.");
433 return;
434 }
435 deleteTestData();
436 }
437 }
438
439 @Test
440 public void testSpecialCharsInInput() {
441 assert (true);
442 }
443
444 protected void indexTestFile(String docCategory, String docType, String docFormat, String resFile)
445 throws Exception {
446 URL resource = getClass().getResource(resFile);
447 File file = new File(resource.toURI());
448 String filePath = file.getAbsolutePath();
449 String result = getIndexerService(docCategory, docType, docFormat).indexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, filePath);
450 LOG.info("result =" + result);
451 assertNotNull(result);
452 assertTrue(result.contains(IndexerService.SUCCESS));
453 String countStr = result.substring(result.indexOf("-") + 1);
454 int count = Integer.parseInt(countStr);
455 assertTrue(count > 0);
456 }
457
458 protected void deleteTestData() {
459 String docCategory = DocCategory.WORK.getCode();
460
461 String result = WorkBibDocumentIndexer.getInstance().deleteDocument(docCategory, indexerService.ID_FIELD_PREFIX + "*");
462 LOG.info("result=" + result);
463 assertNotNull(result);
464 assertTrue(result.contains(IndexerService.SUCCESS));
465 }
466
467 protected List<String> getUUIDsByAuthor(String authorName) throws Exception {
468 List<String> uuidList = new ArrayList<String>();
469 SolrServer solr = SolrServerManager.getInstance().getSolrServer();
470 SolrQuery query = new SolrQuery();
471 query.setQuery("Author_display:" + authorName);
472 QueryResponse queryResponse = solr.query(query);
473 Iterator<SolrDocument> iter = queryResponse.getResults().iterator();
474 while (iter.hasNext()) {
475 SolrDocument resultDoc = iter.next();
476 String id = (String) resultDoc.getFieldValue("id");
477 uuidList.add(id);
478 LOG.info("uuidList=" + uuidList);
479 }
480 return uuidList;
481 }
482 @Ignore
483 @Test
484 public void testWorkInstanceOlemlIndexDocuments() throws Exception {
485 cleanUpData();
486 RequestDocument rd = null;
487 QueryResponse response;
488 String instanceRequestFile = "/work/instance/oleml/Instance-Request.xml";
489 URL resource = getClass().getResource(instanceRequestFile);
490 File file = new File(resource.toURI());
491 String instanceRequestContent = readFile(file);
492 RequestHandler requestHandler = new RequestHandler();
493 org.kuali.ole.docstore.model.xmlpojo.ingest.Request request = requestHandler.toObject(instanceRequestContent);
494 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
495 LOG.info("requestDocuments size " + requestDocumentList.size());
496 int i = 0;
497
498 for (RequestDocument requestDocument : requestDocumentList) {
499 modifyRequestDocument(requestDocument);
500 display(requestDocument);
501 getIndexerService(requestDocument).indexDocument(requestDocument);
502 response = searchBibRecord(DocCategory.WORK.getCode(), DocType.BIB.getDescription(),
503 DocFormat.MARC.getCode(), "id", requestDocument.getUuid());
504
505 SolrDocumentList solrDocumentList = response.getResults();
506
507
508 LOG.info("getNumFound marc " + solrDocumentList.getNumFound());
509
510
511 for (RequestDocument requestDocumentLinks : requestDocument.getLinkedRequestDocuments()) {
512 InstanceCollection ic = (InstanceCollection) requestDocumentLinks.getContent().getContentObject();
513 for (Instance oleInstance : ic.getInstance()) {
514 response = searchInstanceRecord(requestDocumentLinks.getCategory(), requestDocumentLinks.getType(),
515 requestDocumentLinks.getFormat(), "id",
516 oleInstance.getInstanceIdentifier());
517 solrDocumentList = response.getResults();
518 LOG.info("getNumFound oleinstance " + solrDocumentList.getNumFound());
519
520 response = searchHoldingRecord(DocCategory.WORK.getCode(), "holding",
521 DocFormat.OLEML.getCode(), "id",
522 oleInstance.getOleHoldings().getHoldingsIdentifier());
523 solrDocumentList = response.getResults();
524 LOG.info("getNumFound oleholding " + solrDocumentList.getNumFound());
525
526 for (Item oleItem : oleInstance.getItems().getItem()) {
527 response = searchItemsRecord(DocCategory.WORK.getCode(), "item",
528 DocFormat.OLEML.getCode(), "id", oleItem.getItemIdentifier());
529 solrDocumentList = response.getResults();
530 LOG.info("getNumFound oleitem " + solrDocumentList.getNumFound());
531
532 }
533 }
534 }
535 }
536 }
537
538
539 public QueryResponse searchRecord(String docCat, String docType, String docFormat, String fieldName,
540 String fieldValue) throws Exception {
541 LOG.info("fieldNameIdentifier " + fieldName);
542 String identifier_args = "(" + fieldName + ":" + fieldValue + ")";
543 String docCategory_args = "(DocCategory" + ":" + docCat + ")";
544 String docType_args = "(DocType" + ":" + docType + ")";
545 String docFormat_args = "(DocFormat" + ":" + docFormat + ")";
546 String args = identifier_args + "AND" + docCategory_args + "AND" + docType_args + "AND" + docFormat_args;
547
548 SolrServer solr = SolrServerManager.getInstance().getSolrServer();
549 SolrQuery query = new SolrQuery();
550 LOG.info("args " + args);
551 query.setQuery(args);
552
553
554
555
556
557
558 QueryResponse response = solr.query(query);
559
560 return response;
561 }
562
563 public QueryResponse searchBibRecord(String docCat, String docType, String docFormat, String fieldNameIdentifier,
564 String fieldValueIdentifier) throws Exception {
565 return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
566 }
567
568 public QueryResponse searchInstanceRecord(String docCat, String docType, String docFormat,
569 String fieldNameIdentifier, String fieldValueIdentifier)
570 throws Exception {
571 return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
572 }
573
574 public QueryResponse searchHoldingRecord(String docCat, String docType, String docFormat,
575 String fieldNameIdentifier, String fieldValueIdentifier) throws Exception {
576 return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
577 }
578
579 public QueryResponse searchItemsRecord(String docCat, String docType, String docFormat, String fieldNameIdentifier,
580 String fieldValueIdentifier) throws Exception {
581 return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
582
583 }
584
585 private static int count = 0;
586
587 public void modifyRequestDocument(RequestDocument requestDocument) {
588 count++;
589 LOG.info("count " + count);
590 oleItemCount = 0;
591 String marcID = String.valueOf(Math.round(Math.random()));
592 LOG.info("modifyRequestDocument : marcID " + marcID);
593 String oleHoldingID = String.valueOf(Math.round(Math.random()));
594
595 LOG.info("modifyRequestDocument : oleHoldingID " + oleHoldingID);
596 String oleItemID = "";
597 String instanceID = String.valueOf(Math.round(Math.random()));
598 LOG.info("modifyRequestDocument : instanceID " + instanceID);
599
600 requestDocument.setUuid(marcID);
601 InstanceOlemlRecordProcessor instanceOlemlRecordProcessor = new InstanceOlemlRecordProcessor();
602 List<RequestDocument> linkedRequestDocumentsModified = new ArrayList<RequestDocument>();
603 Content content = null;
604 for (RequestDocument req : requestDocument.getLinkedRequestDocuments()) {
605 InstanceCollection ic = instanceOlemlRecordProcessor.fromXML(req.getContent().getContent());
606 List<Instance> oleInstances = ic.getInstance();
607 req.getContent().setContentObject(ic);
608 req.setUuid(instanceID);
609 for (Instance oleInstance : oleInstances) {
610 oleInstance.setInstanceIdentifier(instanceID);
611
612
613
614
615
616
617 OleHoldings holding = oleInstance.getOleHoldings();
618 holding.setHoldingsIdentifier(oleHoldingID);
619
620
621
622
623
624 for (Item oleItem : oleInstance.getItems().getItem()) {
625 oleItemCount++;
626 oleItemID = String.valueOf(Math.round(Math.random()));
627
628 oleItem.setItemIdentifier(oleItemID);
629
630
631
632
633
634
635
636 }
637 }
638
639 }
640
641 }
642
643 public void display(RequestDocument requestDocument) {
644 LOG.info("in display");
645 LOG.info("requestDocument.getId " + requestDocument.getId());
646 LOG.info("requestDocument.getCategory " + requestDocument.getCategory());
647 LOG.info("requestDocument.getFormat " + requestDocument.getFormat());
648 LOG.info("requestDocument.getType " + requestDocument.getType());
649
650 List<RequestDocument> links = requestDocument.getLinkedRequestDocuments();
651 LOG.info("links.size " + links.size());
652 for (RequestDocument requestDocumentLinks : links) {
653 LOG.info("requestDocumentLinks.getId " + requestDocumentLinks.getId());
654 LOG.info("requestDocumentLinks.getUuid " + requestDocumentLinks.getUuid());
655 LOG.info("requestDocumentLinks.getCategory " + requestDocumentLinks.getCategory());
656 LOG.info("requestDocumentLinks.getFormat " + requestDocumentLinks.getFormat());
657 LOG.info("requestDocumentLinks.getType " + requestDocumentLinks.getType());
658 LOG.info("requestDocumentLinks getContentObject " + requestDocumentLinks.getContent().getContentObject());
659 InstanceCollection ic = (InstanceCollection) requestDocumentLinks.getContent().getContentObject();
660 for (Instance oleInstance : ic.getInstance()) {
661 LOG.info("getResourceIdentifier " + oleInstance.getResourceIdentifier());
662 LOG.info("getHoldingsIdentifier " + oleInstance.getOleHoldings().getHoldingsIdentifier());
663
664 for (Item oleItem : oleInstance.getItems().getItem()) {
665 LOG.info("getItemIdentifier " + oleItem.getItemIdentifier());
666
667
668 }
669
670 }
671
672 if (requestDocumentLinks.getContent().getContentObject() instanceof OleHoldings) {
673 LOG.info("requestDocumentLinks.getHoldingsIdentifier " + ((OleHoldings) requestDocumentLinks.getContent()
674 .getContentObject())
675 .getHoldingsIdentifier());
676
677
678
679
680 }
681 if (requestDocumentLinks.getContent().getContentObject() instanceof Item) {
682 LOG.info("requestDocumentLinks.getItemIdentifier " + ((Item) requestDocumentLinks.getContent()
683 .getContentObject())
684 .getItemIdentifier());
685
686
687
688 }
689
690 }
691 }
692 @Ignore
693 @Test
694
695
696
697 public void testIndexDocuments() throws Exception {
698
699 printInitialLog();
700 String inputFilePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
701 invokeIndexDocuments(inputFilePath);
702
703 inputFilePath = "/work/bib/dublin/DublinQ-Test1.xml";
704 invokeIndexDocuments(inputFilePath);
705
706 inputFilePath = "/work/bib/dublin/unqualified/DublinUnq-Test1.xml";
707 invokeIndexDocuments(inputFilePath);
708
709
710
711
712
713
714 printFinalLog();
715 }
716
717
718
719
720
721
722 private void invokeIndexDocuments(String inputFilePath) {
723 try {
724 printLogInfo("-----");
725 printLogInfo("Input file : " + inputFilePath);
726
727 URL resource = getClass().getResource(inputFilePath);
728 File file = new File(resource.toURI());
729 String requestContent = readFile(file);
730 RequestHandler requestHandler = new RequestHandler();
731 Request request = requestHandler.toObject(requestContent);
732 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
733
734 indexDocuments(requestDocumentList);
735 printLogInfo("-----");
736 } catch (Exception e) {
737 LOG.error("Exception:", e);
738 }
739 }
740
741
742
743
744
745
746 private void indexDocuments(List<RequestDocument> requestDocuments) throws Exception {
747 String methodName = getCallingMethod().toString();
748 printLogInfo("Testing " + methodName + "...");
749
750 int numInputDocs = requestDocuments.size();
751 printLogInfo("Num of input documents : " + numInputDocs);
752 String result = "";
753 for (RequestDocument requestDocument : requestDocuments) {
754 result += getIndexerService(requestDocument).indexDocument(requestDocument);
755 printLogInfo("result = " + result);
756 }
757
758 assertTrue(result.contains("success"));
759 long numOfIndexedDocs = getCountFromResult(result);
760 String message = "Number of indexed docs equals number of input docs.";
761 assertEquals("Assertion failed:" + message, numInputDocs, numOfIndexedDocs);
762
763 printLogInfo(message);
764 printLogInfo("Testing " + methodName + "...done");
765 }
766
767 private long getCountFromResult(String result) {
768 result = result.replaceAll("success","");
769 long count = 0;
770 String[] strings = result.split("-");
771 for(int i=0; i<strings.length ; i++) {
772 if(StringUtils.isNotEmpty( strings[i])) {
773 count = count + Integer.parseInt(strings[i]);
774 }
775 }
776 return count;
777 }
778
779 protected String getRecordField(String filedName, String fieldValue) throws Exception {
780 QueryResponse response = executeQuery(filedName + ":" + fieldValue);
781 String result = response.getResults().toString();
782 return result;
783 }
784
785 @Test
786 public void testIndexDocument() {
787 try {
788 printInitialLog();
789 String filePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
790 URL resource = getClass().getResource(filePath);
791 File file = new File(resource.toURI());
792 String requestContent = readFile(file);
793 RequestHandler requestHandler = new RequestHandler();
794 Request request = requestHandler.toObject(requestContent);
795 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
796 int i = 0;
797 for (RequestDocument requestDocument : requestDocumentList) {
798 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
799 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
800 assertTrue(result.contains("success"));
801 }
802 printFinalLog();
803 } catch (Exception e) {
804 LOG.error(e.getMessage() , e);
805 }
806 }
807
808 @Test
809 public void testdeleteDocumentsByUUIDList() {
810 try {
811
812 printInitialLog();
813 String filePath = "/work/bib/marc/marcOneRec-Test1.xml";
814 List<String> UUIDList = new ArrayList<String>();
815 URL resource = getClass().getResource(filePath);
816 File file = new File(resource.toURI());
817 String requestContent = readFile(file);
818 RequestHandler requestHandler = new RequestHandler();
819 Request request = requestHandler.toObject(requestContent);
820 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
821 int i = 0;
822 for (RequestDocument requestDocument : requestDocumentList) {
823 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
824 UUIDList.add(requestDocument.getUuid());
825 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
826 assertTrue(result.contains("success"));
827 }
828 System.out.println("uuid list size:" + UUIDList.size());
829 LOG.info("Record count before deletion:" + getRecordCount());
830 getIndexerService(requestDocumentList.get(0)).deleteDocuments(DocCategory.WORK.getCode(), UUIDList);
831 LOG.info("Record count after deletion:" + getRecordCount());
832
833 printFinalLog();
834 } catch (Exception e) {
835 LOG.error(e.getMessage() , e);
836 }
837 }
838
839 @Ignore
840 @Test
841 public void testIndexDocumentsForONIXPL() throws Exception {
842 try {
843 cleanUpData();
844 String filePath = "/work/license/onixpl/OLE-License-ONIXPL.xml";
845 URL resource = getClass().getResource(filePath);
846 File file = new File(resource.toURI());
847 String requestContent = readFile(file);
848 RequestHandler requestHandler = new RequestHandler();
849 Request request = requestHandler.toObject(requestContent);
850 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
851 int i = 0;
852 for (RequestDocument requestDocument : requestDocumentList) {
853 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
854 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
855 LOG.info("uuid-->" + requestDocument.getUuid());
856 LOG.info("result-->" + result);
857 String searchValue = getRecordField("Title_search", "Test Title");
858 LOG.info("search result-->" + searchValue);
859 assertTrue(result.contains("success"));
860 }
861 } catch (Exception e) {
862 LOG.error(e.getMessage() , e);
863 }
864
865
866 }
867
868 @Test
869 public void testIndexDocumentsForBinary() throws Exception {
870 try {
871 cleanUpData();
872 String filePath = "/work/license/binary/OLE-License-Binary.xml";
873 Request request = getRequest(filePath);
874 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
875 for (RequestDocument requestDocument : requestDocumentList) {
876 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
877 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
878 LOG.info("uuid-->" + requestDocument.getUuid());
879 LOG.info("result-->" + result);
880 String searchValue = getRecordField("FileName_search", "test.xslt");
881 LOG.info("search result-->" + searchValue);
882 assertTrue(result.contains("success"));
883 }
884 } catch (Exception e) {
885 LOG.error(e.getMessage() , e);
886 }
887 }
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922 @Ignore
923 @Test
924 public void cleanupDiscoveryData() throws Exception {
925 String inputFilePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
926 List<SolrInputDocument> SolrInputDocument = new ArrayList<org.apache.solr.common.SolrInputDocument>();
927 URL resource = getClass().getResource(inputFilePath);
928 File file = new File(resource.toURI());
929 String requestContent = readFile(file);
930 RequestHandler requestHandler = new RequestHandler();
931 Request request = requestHandler.toObject(requestContent);
932 List<RequestDocument> requestDocumentList = request.getRequestDocuments();
933 for (RequestDocument requestDocument : requestDocumentList) {
934 requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
935 String result = getIndexerService(requestDocument).indexDocument(requestDocument);
936 LOG.info("uuid-->" + requestDocument.getUuid());
937 System.out.println("Solr Document before delete -->" + getIndexerService(requestDocument).getSolrDocumentBySolrId(requestDocument.getUuid()));
938 System.out.println("result-->" + result);
939 assertTrue(result.contains("success"));
940 getIndexerService(requestDocument).cleanupDiscoveryData();
941 System.out.println("Solr Document after delete -->" + getIndexerService(requestDocument).getSolrDocumentBySolrId(requestDocument.getUuid()));
942 }
943
944
945 }
946
947 public Request getRequest(String filePath) throws Exception {
948 URL resource = getClass().getResource(filePath);
949 File file = new File(resource.toURI());
950 String requestContent = readFile(file);
951 RequestHandler requestHandler = new RequestHandler();
952 Request request = requestHandler.toObject(requestContent);
953 return request;
954 }
955
956 public void printInitialLog() throws Exception {
957 printLogInfo("======");
958 printLogInfo("Total record count before cleanup = " + getRecordCount());
959 cleanUpData();
960 printLogInfo("Total record count after cleanup = " + getRecordCount());
961 }
962
963 public void printFinalLog() throws Exception {
964 printLogInfo("Total record count after the test = " + getRecordCount());
965 printLogInfo("======");
966 }
967
968 private IndexerService getIndexerService(RequestDocument requestDocument) {
969 IndexerService indexerService = DocumentIndexerManagerFactory.getInstance().getDocumentIndexManager(requestDocument.getCategory(), requestDocument.getType(), requestDocument.getFormat());
970 return indexerService;
971 }
972
973 private IndexerService getIndexerService(String category, String type, String format) {
974 IndexerService indexerService = DocumentIndexerManagerFactory.getInstance().getDocumentIndexManager(category, type, format);
975 return indexerService;
976 }
977 }