001package org.kuali.ole.docstore.discovery.service;
002
003import org.apache.commons.io.FileUtils;
004import org.apache.commons.lang.StringUtils;
005import org.apache.solr.client.solrj.SolrQuery;
006import org.apache.solr.client.solrj.SolrServer;
007import org.apache.solr.client.solrj.response.QueryResponse;
008import org.apache.solr.common.SolrDocument;
009import org.apache.solr.common.SolrDocumentList;
010import org.apache.solr.common.SolrInputDocument;
011import org.junit.Before;
012import org.junit.Ignore;
013import org.junit.Test;
014import org.kuali.ole.docstore.common.document.content.instance.Instance;
015import org.kuali.ole.docstore.common.document.content.instance.InstanceCollection;
016import org.kuali.ole.docstore.common.document.content.instance.Item;
017import org.kuali.ole.docstore.common.document.content.instance.xstream.InstanceOlemlRecordProcessor;
018import org.kuali.ole.docstore.discovery.BaseTestCase;
019import org.kuali.ole.docstore.indexer.solr.DocumentIndexerManagerFactory;
020import org.kuali.ole.docstore.indexer.solr.IndexerService;
021import org.kuali.ole.docstore.indexer.solr.WorkBibDocumentIndexer;
022import org.kuali.ole.docstore.model.enums.DocCategory;
023import org.kuali.ole.docstore.model.enums.DocFormat;
024import org.kuali.ole.docstore.model.enums.DocType;
025import org.kuali.ole.docstore.model.xmlpojo.ingest.Content;
026import org.kuali.ole.docstore.model.xmlpojo.ingest.Request;
027import org.kuali.ole.docstore.model.xmlpojo.ingest.RequestDocument;
028import org.kuali.ole.docstore.common.document.content.instance.OleHoldings;
029import org.kuali.ole.docstore.model.xstream.ingest.RequestHandler;
030
031import java.io.File;
032import java.net.URL;
033import java.util.ArrayList;
034import java.util.Iterator;
035import java.util.List;
036
037import static org.junit.Assert.*;
038
039/**
040 * Unit tests for IndexerService class.
041 * User: tirumalesh.b
042 * Date: 22/11/11 Time: 5:23 PM
043 */
044public class IndexerService_UT extends BaseTestCase {
045
046    private IndexerService indexerService = null;
047    private int oleItemCount = 0;
048
049    public IndexerService_UT() {
050
051    }
052
053    @Before
054    public void setUp() throws Exception {
055        super.setUp();
056    }
057
058    /**
059     * Test the bulk indexing of instance records, which also links them with the already indexed Bib records.
060     */
061    @Ignore
062    @Test
063    public void testInstanceBulkIndex() {
064        String result = "";
065        try {
066            // Initial log.
067            printInitialLog();
068            List<String> ids = new ArrayList<String>();
069            ids.add("1234");
070            ids.add("14445");
071            ids.add("144446");
072            ids.add("144447");
073            ids.add("144448");
074            ids.add("144449");
075            ids.add("144450");
076            ids.add("144451");
077            ids.add("144452");
078            ids.add("144453");
079            ids.add("144454");
080            ids.add("144455");
081            ids.add("144456");
082            String filePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
083            URL resource = getClass().getResource(filePath);
084            File file = new File(resource.toURI());
085            String instanceRequestContent = readFile(file);
086            RequestHandler requestHandler = new RequestHandler();
087            Request request = requestHandler.toObject(instanceRequestContent);
088            List<RequestDocument> marcRequestDocumentList = request.getRequestDocuments();
089            LOG.info("requestDocuments size " + marcRequestDocumentList.size());
090            int i = 0;
091            for (RequestDocument requestDocument : marcRequestDocumentList) {
092                if (requestDocument.getUuid() == null)
093                    requestDocument.setUuid(ids.get(i));
094                i++;
095            }
096            result = getIndexerService(marcRequestDocumentList.get(0)).bulkIndexDocuments(marcRequestDocumentList, true);
097            for (RequestDocument doc : marcRequestDocumentList) {
098                LOG.info("Doc Marc : " + doc.getUuid());
099            }
100            LOG.info("RESULT: " + result);
101            if (!result.toLowerCase().contains("success"))
102                fail("Bulk Index Failed.");
103
104            filePath = "/work/instance/oleml/OLE-Instance-bulkIngest-IU-Set1-split.xml";
105            resource = getClass().getResource(filePath);
106            file = new File(resource.toURI());
107            instanceRequestContent = readFile(file);
108            requestHandler = new RequestHandler();
109            Request instanceRequest = requestHandler.toObject(instanceRequestContent);
110            List<RequestDocument> newInstanceRequestDocumentList = new ArrayList<RequestDocument>();
111            List<RequestDocument> instanceRequestDocumentList = instanceRequest.getRequestDocuments();
112            LOG.info("requestDocuments size " + instanceRequestDocumentList.size());
113            for (RequestDocument requestDocument : instanceRequestDocumentList) {
114                if (requestDocument.getUuid() == null)
115                    requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
116                if (requestDocument.getContent().getContentObject() == null) {
117                    InstanceOlemlRecordProcessor oleMlProcessor = new InstanceOlemlRecordProcessor();
118                    InstanceCollection instanceCollection = oleMlProcessor.fromXML(requestDocument.getContent()
119                            .getContent());
120                    for (Instance instance : instanceCollection.getInstance()) {
121                        if (instance.getInstanceIdentifier() == null)
122                            instance.setInstanceIdentifier(requestDocument.getUuid());
123                        if (instance.getOleHoldings().getHoldingsIdentifier() == null)
124                            instance.getOleHoldings().setHoldingsIdentifier(String.valueOf(Math.round(Math.random())));
125                        for (Item item : instance.getItems().getItem())
126                            if (item.getItemIdentifier() == null)
127                                item.setItemIdentifier(String.valueOf(Math.round(Math.random())));
128                    }
129                    requestDocument.getContent().setContentObject(instanceCollection);
130                    newInstanceRequestDocumentList.add(requestDocument);
131                }
132
133            }
134            result = getIndexerService(newInstanceRequestDocumentList.get(0)).bulkIndexDocuments(newInstanceRequestDocumentList, true);
135            LOG.info("RESULT: " + result);
136            if (!result.toLowerCase().contains("success"))
137                fail("Bulk Index Failed.");
138
139            for (RequestDocument bibDoc : marcRequestDocumentList) {
140                List<SolrDocument> bibSolrDocs = getIndexerService(bibDoc).getSolrDocumentBySolrId(bibDoc.getUuid());
141                if (bibSolrDocs == null || bibSolrDocs.size() <= 0)
142                    fail(" Bulk Indexing Failed.");
143                for (SolrDocument outDoc : bibSolrDocs) {
144                    if (outDoc.getFieldValues("instanceIdentifier") != null)
145                        for (Object instanceId : outDoc.getFieldValues("instanceIdentifier")) {
146                            List<SolrDocument> linkedInstanceDocs = getIndexerService(bibDoc).getSolrDocumentBySolrId(instanceId
147                                    .toString());
148                            if (linkedInstanceDocs.size() <= 0)
149                                fail(" Bulk Indexing Failed.");
150                            else
151                                for (SolrDocument instDoc : linkedInstanceDocs)
152                                    if (!instDoc.getFieldValues("bibIdentifier").contains(bibDoc.getUuid()))
153                                        fail(" Bulk Indexing Failed.");
154                        }
155                }
156            }
157            // Final log.
158            printFinalLog();
159
160        } catch (Exception e) {
161            LOG.error(e.getMessage() , e);
162            fail("Bulk Index Failed.");
163        }
164    }
165
166    @Test
167    public void testDublinBulkIndex() throws Exception {
168        try {
169            // Initial log.
170            printInitialLog();
171            String filePath = "/work/bib/dublin/DublinQ-Test1.xml";
172            List<RequestDocument> dublinRequestDocumentList = getRequestDocumentList(filePath);
173            String result = getIndexerService(dublinRequestDocumentList.get(0)).bulkIndexDocuments(dublinRequestDocumentList, true);
174            LOG.info("result:" + result);
175            // Final log.
176            printFinalLog();
177
178        } catch (Exception e) {
179            LOG.error(e.getMessage() , e);
180        }
181    }
182
183    @Test
184    public void testDublinUnqBulkIndex() throws Exception {
185        try {
186            // Initial log.
187            printInitialLog();
188            String filePath = "/work/bib/dublin/unqualified/DublinUnq-Test1.xml";
189            List<RequestDocument> dublinUnqRequestDocumentList = getRequestDocumentList(filePath);
190            System.out.println("size:" + dublinUnqRequestDocumentList.size());
191            String result = getIndexerService(dublinUnqRequestDocumentList.get(0)).bulkIndexDocuments(dublinUnqRequestDocumentList, true);
192            LOG.info("result:" + result);
193            // Final log.
194            printFinalLog();
195
196        } catch (Exception e) {
197            LOG.error(e.getMessage() , e);
198        }
199    }
200
201    public List<RequestDocument> getRequestDocumentList(String path) throws Exception {
202        URL resource = getClass().getResource(path);
203        File file = new File(resource.toURI());
204        String requestContent = readFile(file);
205        RequestHandler requestHandler = new RequestHandler();
206        Request request = requestHandler.toObject(requestContent);
207        List<RequestDocument> requestDocumentList = request.getRequestDocuments();
208        int i = 1000004;
209        for (RequestDocument requestDocument : requestDocumentList) {
210            if (requestDocument.getUuid() == null) {
211                requestDocument.setUuid("" + i + "");
212            }
213            i++;
214        }
215        return requestDocumentList;
216    }
217
218    @Test
219    public void testDeleteDocuments() throws Exception {
220        try {
221            // Initial log.
222            printInitialLog();
223            List<String> UUIDList = new ArrayList<String>();
224            // get uuids of the  record using author name
225            // List<String> uuids = getUUIDsByAuthor("Test_Author_HTC (David George), 1940-");
226            String inputFilePath = "/work/instance/oleml/Instance-Request.xml";
227            Request request = getRequest(inputFilePath);
228            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
229            for (RequestDocument requestDocument : requestDocumentList) {
230                requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
231                UUIDList.add(requestDocument.getUuid());
232                String result = getIndexerService(requestDocument).indexDocument(requestDocument);
233                LOG.info("result:" + result);
234
235
236            }
237
238            printLogInfo("Record count after indexing:" + getRecordCount());
239            getIndexerService(requestDocumentList.get(0)).deleteDocuments(DocCategory.WORK.getCode(), UUIDList);
240            printLogInfo("Record count after deleting:" + getRecordCount());
241
242            // Final log.
243            printFinalLog();
244
245        } catch (Exception e) {
246            LOG.info("Exception+" + e.getMessage(), e);
247            LOG.error(e.getMessage() , e);
248        }
249    }
250
251    @Test
252    public void testDeleteDocument() throws Exception {
253        try {
254            // Initial log.
255            printInitialLog();
256
257            String inputFilePath = "/work/instance/oleml/Instance-Request.xml";
258            Request request = getRequest(inputFilePath);
259            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
260            RequestDocument requestDocument = requestDocumentList.get(0);
261            requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
262            String result = getIndexerService(requestDocument).indexDocument(requestDocument);
263            LOG.info("result:" + result);
264            printLogInfo("Record count after indexing:" + getRecordCount());
265
266            getIndexerService(requestDocument).deleteDocument(DocCategory.WORK.getCode(), requestDocument.getUuid());
267
268            printLogInfo("Record count after deleting:" + getRecordCount());
269
270            // Final log.
271            printFinalLog();
272
273        } catch (Exception e) {
274            LOG.info("Exception+" + e.getMessage(), e);
275        }
276
277    }
278    @Ignore
279    @Test
280    public void testIndexDocumentsFromFilesBySolrDoc() throws Exception {         // *********************
281        // Initial log.
282        printLogInfo("======");
283        printLogInfo("Total record count before cleanup = " + getRecordCount());
284        cleanUpData();
285        printLogInfo("Total record count after cleanup = " + getRecordCount());
286
287        // Test Marc Records
288        LOG.info("testIndexDocumentsFromFileBySolrDoc() :  Marc Records Test");
289        String docCategory = DocCategory.WORK.getCode();
290        String docType = DocType.BIB.getDescription();
291        String docFormat = DocFormat.MARC.getCode();
292        String requestFile
293                = "/bib/bib/marc/marc-one-record-xstream.xml";  // "/work/instance/oleml/Instance-Request.xml";
294        invokeIndexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, requestFile);
295
296        // Test Dublin Records
297        LOG.info("testIndexDocumentsFromFileBySolrDoc() :  Dublin Records Test");
298        docFormat = DocFormat.DUBLIN_CORE.getCode();
299        requestFile = "/bib/bib/dublin/Bib-Bib-DublinQ-Test1.xml";
300        invokeIndexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, requestFile);
301
302        // Test Dublin UnQualified Records
303        LOG.info("testIndexDocumentsFromFileBySolrDoc() :  Dublin Unqualified Records Test");
304        requestFile = "/bib/bib/dublin/unqualified/Bib-Bib-DublinUnQ-Test1.xml";
305        docFormat = DocFormat.DUBLIN_UNQUALIFIED.getCode();
306        invokeIndexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, requestFile);
307
308        // Final log.
309        printLogInfo("Total record count after the test = " + getRecordCount());
310        printLogInfo("======");
311    }
312    @Ignore
313    @Test
314    /**
315     *  Same as testIndexDocumentsFromFileBySolrDoc, but user needs to provide dataDir=a/b/c.
316     *  For manual testing. Test data inserted by this method is not deleted automatically.
317     *  e.g. run with maven as follows:
318     *  $ole-discovery-core>mvn -Dtest=IndexerService_UT#testIndexDocumentsFromFileBy
319     SolrDocWithConfirm -DdataDir=.\src\test\resources\bib\bib\marc test
320     */
321    public void testIndexDocumentsFromDirBySolrDocWithConfirm() throws Exception {
322        // Initial log.
323        printLogInfo("======");
324        printLogInfo("Total record count before cleanup = " + getRecordCount());
325        cleanUpData();
326        printLogInfo("Total record count after cleanup = " + getRecordCount());
327        String dirPath = "/bib/bib/marc/request";
328        invokeIndexDocumentsFromDirBySolrDoc(dirPath);
329        // Final log.
330        printLogInfo("Total record count after the test = " + getRecordCount());
331        printLogInfo("======");
332    }
333
334    @Test
335    public void testIndexDocumentsWithInvalidDirectoryORFile() throws Exception {
336        try {
337            String docCategory = DocCategory.WORK.getCode();
338            String docType = DocType.BIB.getDescription();
339            String docFormat = DocFormat.MARC.getCode();
340            // Initial log.
341            printInitialLog();
342            String dirPath = "/bib/bib/marc/request/invalidDir";
343            getIndexerService(docCategory, docType, docFormat).indexDocumentsFromDirBySolrDoc(docCategory, docType, docFormat, dirPath);
344
345            dirPath = "/work/emptyDir";
346            getIndexerService(docCategory, docType, docFormat).indexDocumentsFromDirBySolrDoc(docCategory, docType, docFormat, dirPath);
347            // Final log.
348            printFinalLog();
349        } catch (Exception e) {
350            LOG.error(e.getMessage() , e);
351        }
352
353    }
354
355    public void invokeIndexDocumentsFromDirBySolrDoc(String path) {
356        try {
357            printLogInfo("-----");
358            printLogInfo("Input Dir/File : " + path);
359            String docCategory = DocCategory.WORK.getCode();
360            String docType = DocType.BIB.getDescription();
361            String docFormat = DocFormat.MARC.getCode();
362            URL resource = getClass().getResource(path);
363            File file = new File(resource.toURI());
364            String dataDir = file.getAbsolutePath();
365            if (null == dataDir) {
366                LOG.info("dataDir is null.");
367                System.out.println("dataDir is null.");
368                return;
369            }
370            String result = getIndexerService(docCategory, docType, docFormat).indexDocumentsFromDirBySolrDoc(docCategory, docType, docFormat, dataDir);
371            LOG.info("result=" + result);
372            System.out.println("result=" + result);
373            assertNotNull(result);
374            assertTrue(result.contains(IndexerService.SUCCESS));
375            printLogInfo("-----");
376        } catch (Exception e) {
377            LOG.error("Exception:", e);
378        }
379    }
380    @Ignore
381    @Test
382    public void testIndexDocumentFromStringBySolr() throws Exception {
383        // Initial log.
384        printInitialLog();
385        String requestFile = "/bib/bib/marc/marc-one-record-xstream.xml";
386        invokeIndexDocumentsFromStringBySolrDoc(requestFile);
387        // Final log.
388        printFinalLog();
389
390    }
391
392    public void invokeIndexDocumentsFromStringBySolrDoc(String requestFile) throws Exception {
393        try {
394            printLogInfo("-----");
395            printLogInfo("Input Dir/File : " + requestFile);
396            String docCategory = DocCategory.WORK.getCode();
397            String docType = DocType.BIB.getDescription();
398            String docFormat = DocFormat.MARC.getCode();
399            URL resource = getClass().getResource(requestFile);
400            File file = new File(resource.toURI());
401            String requestContent = FileUtils.readFileToString(file);
402            getIndexerService(docCategory, docType, docFormat).indexDocumentsFromStringBySolrDoc(docCategory, docType, docFormat, requestContent);
403            printLogInfo("-----");
404        } catch (Exception e) {
405            LOG.error("Exception:", e);
406        }
407
408    }
409
410    public void invokeIndexDocumentsFromFileBySolrDoc(String docCategory, String docType, String docFormat,
411                                                      String requestFile) throws Exception {
412        URL resource = getClass().getResource(requestFile);
413        File file = new File(resource.toURI());
414        String path = file.getAbsolutePath();
415        String result = getIndexerService(docCategory, docType, docFormat).indexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, path);
416        System.out.println("result=" + result);
417        LOG.info("result=" + result);
418        assertNotNull(result);
419        assertTrue(result.contains(IndexerService.SUCCESS));
420    }
421
422    @Test
423    /**
424     *  Same as deleteTestData, but user needs to provide confirmDelete=true.
425     *  For manual testing. Used to manually delete test data.
426     */
427    public void deleteTestDataWithConfirm() {
428        String confirmDelete = System.getProperty("confirmDelete");
429        if (confirmDelete != null) {
430            boolean confirm = Boolean.valueOf(confirmDelete);
431            if (!confirm) {
432                LOG.info("Delete is not confirmed through a system property.");
433                return;
434            }
435            deleteTestData();
436        }
437    }
438
439    @Test
440    public void testSpecialCharsInInput() {
441        assert (true);
442    }
443
444    protected void indexTestFile(String docCategory, String docType, String docFormat, String resFile)
445            throws Exception {
446        URL resource = getClass().getResource(resFile);
447        File file = new File(resource.toURI());
448        String filePath = file.getAbsolutePath();
449        String result = getIndexerService(docCategory, docType, docFormat).indexDocumentsFromFileBySolrDoc(docCategory, docType, docFormat, filePath);
450        LOG.info("result =" + result);
451        assertNotNull(result);
452        assertTrue(result.contains(IndexerService.SUCCESS));
453        String countStr = result.substring(result.indexOf("-") + 1);
454        int count = Integer.parseInt(countStr);
455        assertTrue(count > 0);
456    }
457
458    protected void deleteTestData() {
459        String docCategory = DocCategory.WORK.getCode();
460        // Test data is assumed to have id field value starting with "id_".
461        String result = WorkBibDocumentIndexer.getInstance().deleteDocument(docCategory, indexerService.ID_FIELD_PREFIX + "*");
462        LOG.info("result=" + result);
463        assertNotNull(result);
464        assertTrue(result.contains(IndexerService.SUCCESS));
465    }
466
467    protected List<String> getUUIDsByAuthor(String authorName) throws Exception {
468        List<String> uuidList = new ArrayList<String>();
469        SolrServer solr = SolrServerManager.getInstance().getSolrServer();
470        SolrQuery query = new SolrQuery();
471        query.setQuery("Author_display:" + authorName);
472        QueryResponse queryResponse = solr.query(query);
473        Iterator<SolrDocument> iter = queryResponse.getResults().iterator();
474        while (iter.hasNext()) {
475            SolrDocument resultDoc = iter.next();
476            String id = (String) resultDoc.getFieldValue("id");
477            uuidList.add(id);
478            LOG.info("uuidList=" + uuidList);
479        }
480        return uuidList;
481    }
482    @Ignore
483    @Test
484    public void testWorkInstanceOlemlIndexDocuments() throws Exception {
485        cleanUpData();
486        RequestDocument rd = null;
487        QueryResponse response;
488        String instanceRequestFile = "/work/instance/oleml/Instance-Request.xml";   // "/work/instance/oleml/Instance-Request.xml";
489        URL resource = getClass().getResource(instanceRequestFile);
490        File file = new File(resource.toURI());
491        String instanceRequestContent = readFile(file);
492        RequestHandler requestHandler = new RequestHandler();
493        org.kuali.ole.docstore.model.xmlpojo.ingest.Request request = requestHandler.toObject(instanceRequestContent);
494        List<RequestDocument> requestDocumentList = request.getRequestDocuments();
495        LOG.info("requestDocuments size " + requestDocumentList.size());
496        int i = 0;
497
498        for (RequestDocument requestDocument : requestDocumentList) {
499            modifyRequestDocument(requestDocument);
500            display(requestDocument);
501            getIndexerService(requestDocument).indexDocument(requestDocument);
502            response = searchBibRecord(DocCategory.WORK.getCode(), DocType.BIB.getDescription(),
503                    DocFormat.MARC.getCode(), "id", requestDocument.getUuid());
504            // LOG.info("searchBibRecord : response " + response);
505            SolrDocumentList solrDocumentList = response.getResults();
506
507            //assertEquals(1, solrDocumentList.getNumFound());
508            LOG.info("getNumFound marc " + solrDocumentList.getNumFound());
509
510
511            for (RequestDocument requestDocumentLinks : requestDocument.getLinkedRequestDocuments()) {
512                InstanceCollection ic = (InstanceCollection) requestDocumentLinks.getContent().getContentObject();
513                for (Instance oleInstance : ic.getInstance()) {
514                    response = searchInstanceRecord(requestDocumentLinks.getCategory(), requestDocumentLinks.getType(),
515                            requestDocumentLinks.getFormat(), "id",
516                            oleInstance.getInstanceIdentifier());
517                    solrDocumentList = response.getResults();
518                    LOG.info("getNumFound oleinstance " + solrDocumentList.getNumFound());
519                    //assertEquals(1, solrDocumentList.getNumFound());
520                    response = searchHoldingRecord(DocCategory.WORK.getCode(), "holding",
521                            DocFormat.OLEML.getCode(), "id",
522                            oleInstance.getOleHoldings().getHoldingsIdentifier());
523                    solrDocumentList = response.getResults();
524                    LOG.info("getNumFound oleholding " + solrDocumentList.getNumFound());
525                    //assertEquals(1, solrDocumentList.getNumFound());
526                    for (Item oleItem : oleInstance.getItems().getItem()) {
527                        response = searchItemsRecord(DocCategory.WORK.getCode(), "item",
528                                DocFormat.OLEML.getCode(), "id", oleItem.getItemIdentifier());
529                        solrDocumentList = response.getResults();
530                        LOG.info("getNumFound oleitem " + solrDocumentList.getNumFound());
531                       // assertEquals(1, solrDocumentList.getNumFound());
532                    }
533                }
534            }
535        }
536    }
537
538
539    public QueryResponse searchRecord(String docCat, String docType, String docFormat, String fieldName,
540                                      String fieldValue) throws Exception {
541        LOG.info("fieldNameIdentifier " + fieldName);
542        String identifier_args = "(" + fieldName + ":" + fieldValue + ")";
543        String docCategory_args = "(DocCategory" + ":" + docCat + ")";
544        String docType_args = "(DocType" + ":" + docType + ")";
545        String docFormat_args = "(DocFormat" + ":" + docFormat + ")";
546        String args = identifier_args + "AND" + docCategory_args + "AND" + docType_args + "AND" + docFormat_args;
547        //SolrServer solr = new CommonsHttpSolrServer(PropertyUtil.getPropertyUtil().getProperty("docSearchURL") + "bib");
548        SolrServer solr = SolrServerManager.getInstance().getSolrServer();
549        SolrQuery query = new SolrQuery();
550        LOG.info("args " + args);
551        query.setQuery(args);
552        //query.setQuery(docCategory_args);
553        //query.setQuery(docType_args);
554        //query.setQuery(docFormat_args);
555        //LOG.info("getQuery "+query.getQuery());
556
557
558        QueryResponse response = solr.query(query);
559        // LOG.info("response "+response);
560        return response;
561    }
562
563    public QueryResponse searchBibRecord(String docCat, String docType, String docFormat, String fieldNameIdentifier,
564                                         String fieldValueIdentifier) throws Exception {
565        return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
566    }
567
568    public QueryResponse searchInstanceRecord(String docCat, String docType, String docFormat,
569                                              String fieldNameIdentifier, String fieldValueIdentifier)
570            throws Exception {
571        return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
572    }
573
574    public QueryResponse searchHoldingRecord(String docCat, String docType, String docFormat,
575                                             String fieldNameIdentifier, String fieldValueIdentifier) throws Exception {
576        return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
577    }
578
579    public QueryResponse searchItemsRecord(String docCat, String docType, String docFormat, String fieldNameIdentifier,
580                                           String fieldValueIdentifier) throws Exception {
581        return searchRecord(docCat, docType, docFormat, fieldNameIdentifier, fieldValueIdentifier);
582
583    }
584
585    private static int count = 0;
586
587    public void modifyRequestDocument(RequestDocument requestDocument) {
588        count++;
589        LOG.info("count " + count);
590        oleItemCount = 0;
591        String marcID = String.valueOf(Math.round(Math.random()));
592        LOG.info("modifyRequestDocument : marcID " + marcID);
593        String oleHoldingID = String.valueOf(Math.round(Math.random()));
594
595        LOG.info("modifyRequestDocument : oleHoldingID " + oleHoldingID);
596        String oleItemID = "";
597        String instanceID = String.valueOf(Math.round(Math.random()));
598        LOG.info("modifyRequestDocument : instanceID " + instanceID);
599
600        requestDocument.setUuid(marcID);
601        InstanceOlemlRecordProcessor instanceOlemlRecordProcessor = new InstanceOlemlRecordProcessor();
602        List<RequestDocument> linkedRequestDocumentsModified = new ArrayList<RequestDocument>();
603        Content content = null;
604        for (RequestDocument req : requestDocument.getLinkedRequestDocuments()) {
605            InstanceCollection ic = instanceOlemlRecordProcessor.fromXML(req.getContent().getContent());
606            List<Instance> oleInstances = ic.getInstance();
607            req.getContent().setContentObject(ic);
608            req.setUuid(instanceID);
609            for (Instance oleInstance : oleInstances) {
610                oleInstance.setInstanceIdentifier(instanceID);
611//                oleInstance.setResourceIdentifier(marcID);
612                /*
613                                rd = (RequestDocument) req.clone();
614                                content1 = new Content();
615                                rd.setContent(content1);
616                */
617                OleHoldings holding = oleInstance.getOleHoldings();
618                holding.setHoldingsIdentifier(oleHoldingID);
619//                holding.setInstanceIdentifier(instanceID);
620                /*
621                                rd.getContent().setContentObject(holding);
622                                linkedRequestDocumentsModified.add(rd);
623                */
624                for (Item oleItem : oleInstance.getItems().getItem()) {
625                    oleItemCount++;
626                    oleItemID = String.valueOf(Math.round(Math.random()));
627                    //                    rd = (RequestDocument) req.clone();
628                    oleItem.setItemIdentifier(oleItemID);
629//                    oleItem.setInstanceIdentifier(instanceID);
630                    /*
631                                        content1 = new Content();
632                                        rd.setContent(content1);
633                                        rd.getContent().setContentObject(oleItem);
634                                        linkedRequestDocumentsModified.add(rd);
635                    */
636                }
637            }
638            //            requestDocument.setLinkedRequestDocuments(linkedRequestDocumentsModified);
639        }
640
641    }
642
643    public void display(RequestDocument requestDocument) {
644        LOG.info("in display");
645        LOG.info("requestDocument.getId " + requestDocument.getId());
646        LOG.info("requestDocument.getCategory " + requestDocument.getCategory());
647        LOG.info("requestDocument.getFormat " + requestDocument.getFormat());
648        LOG.info("requestDocument.getType " + requestDocument.getType());
649        //  LOG.info("requestDocument.getType " + requestDocument.getType());
650        List<RequestDocument> links = requestDocument.getLinkedRequestDocuments();
651        LOG.info("links.size " + links.size());
652        for (RequestDocument requestDocumentLinks : links) {
653            LOG.info("requestDocumentLinks.getId " + requestDocumentLinks.getId());
654            LOG.info("requestDocumentLinks.getUuid " + requestDocumentLinks.getUuid());
655            LOG.info("requestDocumentLinks.getCategory " + requestDocumentLinks.getCategory());
656            LOG.info("requestDocumentLinks.getFormat " + requestDocumentLinks.getFormat());
657            LOG.info("requestDocumentLinks.getType " + requestDocumentLinks.getType());
658            LOG.info("requestDocumentLinks getContentObject " + requestDocumentLinks.getContent().getContentObject());
659            InstanceCollection ic = (InstanceCollection) requestDocumentLinks.getContent().getContentObject();
660            for (Instance oleInstance : ic.getInstance()) {
661                LOG.info("getResourceIdentifier " + oleInstance.getResourceIdentifier());
662                LOG.info("getHoldingsIdentifier " + oleInstance.getOleHoldings().getHoldingsIdentifier());
663//               LOG.info("getInstanceIdentifier " + oleInstance.getHoldings().getInstanceIdentifier());
664                for (Item oleItem : oleInstance.getItems().getItem()) {
665                    LOG.info("getItemIdentifier " + oleItem.getItemIdentifier());
666//                    LOG.info("getInstanceIdentifier " + oleItem.getInstanceIdentifier());
667
668                }
669
670            }
671
672            if (requestDocumentLinks.getContent().getContentObject() instanceof OleHoldings) {
673                LOG.info("requestDocumentLinks.getHoldingsIdentifier " + ((OleHoldings) requestDocumentLinks.getContent()
674                        .getContentObject())
675                        .getHoldingsIdentifier());
676//               LOG.info(
677//                        "requestDocumentLinks.Holding getInstanceIdentifier " + ((OleHolding) requestDocumentLinks
678//                                .getContent().getContentObject()).getInstanceIdentifier());
679
680            }
681            if (requestDocumentLinks.getContent().getContentObject() instanceof Item) {
682                LOG.info("requestDocumentLinks.getItemIdentifier " + ((Item) requestDocumentLinks.getContent()
683                        .getContentObject())
684                        .getItemIdentifier());
685//               LOG.info("requestDocumentLinks.item getInstanceIdentifier " + ((OleItem) requestDocumentLinks
686//                        .getContent().getContentObject()).getInstanceIdentifier());
687
688            }
689
690        }
691    }
692    @Ignore
693    @Test
694    /**
695     * This method manages the testing of {@link IndexerService#indexDocuments(List<RequestDocument> requestDocuments)}
696     */
697    public void testIndexDocuments() throws Exception {
698        // Initial log.
699        printInitialLog();
700        String inputFilePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
701        invokeIndexDocuments(inputFilePath);
702
703        inputFilePath = "/work/bib/dublin/DublinQ-Test1.xml";
704        invokeIndexDocuments(inputFilePath);
705
706        inputFilePath = "/work/bib/dublin/unqualified/DublinUnq-Test1.xml";
707        invokeIndexDocuments(inputFilePath);
708
709        // inputFilePath = "/work/instance/oleml/instanceOneRec-Test1.xml";
710        //invokeIndexDocuments(inputFilePath);
711        // Test with more input files.
712
713        // Final log.
714        printFinalLog();
715    }
716
717    /**
718     * This method prepares the input and calls the test for {@link IndexerService#indexDocuments(List<RequestDocument>)}
719     *
720     * @param inputFilePath
721     */
722    private void invokeIndexDocuments(String inputFilePath) {
723        try {
724            printLogInfo("-----");
725            printLogInfo("Input file : " + inputFilePath);
726            // Preparation for test.
727            URL resource = getClass().getResource(inputFilePath);
728            File file = new File(resource.toURI());
729            String requestContent = readFile(file);
730            RequestHandler requestHandler = new RequestHandler();
731            Request request = requestHandler.toObject(requestContent);
732            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
733
734            indexDocuments(requestDocumentList);
735            printLogInfo("-----");
736        } catch (Exception e) {
737             LOG.error("Exception:", e);
738        }
739    }
740
741    /**
742     * This method actually tests {@link IndexerService#indexDocuments(List<RequestDocument>)}
743     *
744     * @param requestDocuments
745     */
746    private void indexDocuments(List<RequestDocument> requestDocuments) throws Exception {
747        String methodName = getCallingMethod().toString();
748        printLogInfo("Testing " + methodName + "...");
749
750        int numInputDocs = requestDocuments.size();
751        printLogInfo("Num of input documents : " + numInputDocs);
752        String result = "";
753        for (RequestDocument requestDocument : requestDocuments) {
754            result += getIndexerService(requestDocument).indexDocument(requestDocument);
755            printLogInfo("result = " + result);
756        }
757        // Assertions.
758        assertTrue(result.contains("success"));
759        long numOfIndexedDocs = getCountFromResult(result);
760        String message = "Number of indexed docs equals number of input docs.";
761        assertEquals("Assertion failed:" + message, numInputDocs, numOfIndexedDocs);
762
763        printLogInfo(message);
764        printLogInfo("Testing " + methodName + "...done");
765    }
766
767    private long getCountFromResult(String result) {
768        result = result.replaceAll("success","");
769        long count = 0;
770        String[] strings = result.split("-");
771        for(int i=0; i<strings.length ; i++) {
772            if(StringUtils.isNotEmpty( strings[i]))  {
773                count = count + Integer.parseInt(strings[i]);
774            }
775        }
776        return count;
777    }
778
779    protected String getRecordField(String filedName, String fieldValue) throws Exception {
780        QueryResponse response = executeQuery(filedName + ":" + fieldValue);
781        String result = response.getResults().toString();
782        return result;
783    }
784
785    @Test
786    public void testIndexDocument() {
787        try {
788            printInitialLog();
789            String filePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
790            URL resource = getClass().getResource(filePath);
791            File file = new File(resource.toURI());
792            String requestContent = readFile(file);
793            RequestHandler requestHandler = new RequestHandler();
794            Request request = requestHandler.toObject(requestContent);
795            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
796            int i = 0;
797            for (RequestDocument requestDocument : requestDocumentList) {
798                requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
799                String result = getIndexerService(requestDocument).indexDocument(requestDocument);
800                assertTrue(result.contains("success"));
801            }
802            printFinalLog();
803        } catch (Exception e) {
804            LOG.error(e.getMessage() , e);
805        }
806    }
807
808    @Test
809    public void testdeleteDocumentsByUUIDList() {
810        try {
811            // Initial log.
812            printInitialLog();
813            String filePath = "/work/bib/marc/marcOneRec-Test1.xml";
814            List<String> UUIDList = new ArrayList<String>();
815            URL resource = getClass().getResource(filePath);
816            File file = new File(resource.toURI());
817            String requestContent = readFile(file);
818            RequestHandler requestHandler = new RequestHandler();
819            Request request = requestHandler.toObject(requestContent);
820            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
821            int i = 0;
822            for (RequestDocument requestDocument : requestDocumentList) {
823                requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
824                UUIDList.add(requestDocument.getUuid());
825                String result = getIndexerService(requestDocument).indexDocument(requestDocument);
826                assertTrue(result.contains("success"));
827            }
828            System.out.println("uuid list size:" + UUIDList.size());
829            LOG.info("Record count before deletion:" + getRecordCount());
830            getIndexerService(requestDocumentList.get(0)).deleteDocuments(DocCategory.WORK.getCode(), UUIDList);
831            LOG.info("Record count after deletion:" + getRecordCount());
832            // Final log.
833            printFinalLog();
834        } catch (Exception e) {
835            LOG.error(e.getMessage() , e);
836        }
837    }
838
839    @Ignore
840    @Test
841    public void testIndexDocumentsForONIXPL() throws Exception {
842        try {
843            cleanUpData();
844            String filePath = "/work/license/onixpl/OLE-License-ONIXPL.xml";
845            URL resource = getClass().getResource(filePath);
846            File file = new File(resource.toURI());
847            String requestContent = readFile(file);
848            RequestHandler requestHandler = new RequestHandler();
849            Request request = requestHandler.toObject(requestContent);
850            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
851            int i = 0;
852            for (RequestDocument requestDocument : requestDocumentList) {
853                requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
854                String result = getIndexerService(requestDocument).indexDocument(requestDocument);
855                LOG.info("uuid-->" + requestDocument.getUuid());
856                LOG.info("result-->" + result);
857                String searchValue = getRecordField("Title_search", "Test Title");
858                LOG.info("search result-->" + searchValue);
859                assertTrue(result.contains("success"));
860            }
861        } catch (Exception e) {
862            LOG.error(e.getMessage() , e);
863        }
864
865
866    }
867
868    @Test
869    public void testIndexDocumentsForBinary() throws Exception {
870        try {
871            cleanUpData();
872            String filePath = "/work/license/binary/OLE-License-Binary.xml";
873            Request request = getRequest(filePath);
874            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
875            for (RequestDocument requestDocument : requestDocumentList) {
876                requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
877                String result = getIndexerService(requestDocument).indexDocument(requestDocument);
878                LOG.info("uuid-->" + requestDocument.getUuid());
879                LOG.info("result-->" + result);
880                String searchValue = getRecordField("FileName_search", "test.xslt");
881                LOG.info("search result-->" + searchValue);
882                assertTrue(result.contains("success"));
883            }
884        } catch (Exception e) {
885            LOG.error(e.getMessage() , e);
886        }
887    }
888
889//      @Test
890//    public void testIndexWithSpecialChars() {
891//        List<String> uuids = null;
892//        try {
893//            cleanUpData();
894//            String filePath = "/work/bib/marc/marcOneRec-Test1.xml";
895//            URL resource = getClass().getResource(filePath);
896//            File file = new File(resource.toURI());
897//            String input = readFile(file);
898//            StringBuffer stringBuffer = new StringBuffer();
899//            String regex = "BIAC journal";
900//            String replace = "BIAC journal H.X1< 9+&5#%,&*(2)\\[\\^\\!\\}";
901//            Pattern pattern = Pattern.compile(regex);
902//            Matcher matcher = pattern.matcher(input);
903//            while (matcher.find()) {
904//                matcher.appendReplacement(stringBuffer, replace);
905//            }
906//            matcher.appendTail(stringBuffer);
907//            String inputFile = stringBuffer.toString();
908//            System.out.println(inputFile);
909//            RequestHandler requestHandler = new RequestHandler();
910//            Request request = requestHandler.toObject(inputFile);
911//            List<RequestDocument> requestDocumentList = request.getRequestDocuments();
912//            indexerService.indexDocuments(requestDocumentList);
913//            uuids = getUUIDsByAuthor(replace);
914//
915//        } catch (Exception e) {
916//            LOG.error("Exception:", e);
917//        }
918//        assertNotNull(uuids);
919//        assertTrue(uuids.size() > 0);
920//    }
921
922    @Ignore
923    @Test
924    public void cleanupDiscoveryData() throws Exception {
925        String inputFilePath = "/work/bib/marc/OLE-Bib-bulkIngest-IU-Set1-split.xml";
926        List<SolrInputDocument> SolrInputDocument = new ArrayList<org.apache.solr.common.SolrInputDocument>();
927        URL resource = getClass().getResource(inputFilePath);
928        File file = new File(resource.toURI());
929        String requestContent = readFile(file);
930        RequestHandler requestHandler = new RequestHandler();
931        Request request = requestHandler.toObject(requestContent);
932        List<RequestDocument> requestDocumentList = request.getRequestDocuments();
933        for (RequestDocument requestDocument : requestDocumentList) {
934            requestDocument.setUuid(String.valueOf(Math.round(Math.random())));
935            String result = getIndexerService(requestDocument).indexDocument(requestDocument);
936            LOG.info("uuid-->" + requestDocument.getUuid());
937            System.out.println("Solr Document before delete -->" + getIndexerService(requestDocument).getSolrDocumentBySolrId(requestDocument.getUuid()));
938            System.out.println("result-->" + result);
939            assertTrue(result.contains("success"));
940            getIndexerService(requestDocument).cleanupDiscoveryData();
941            System.out.println("Solr Document after delete -->" + getIndexerService(requestDocument).getSolrDocumentBySolrId(requestDocument.getUuid()));
942        }
943
944
945    }
946
947    public Request getRequest(String filePath) throws Exception {
948        URL resource = getClass().getResource(filePath);
949        File file = new File(resource.toURI());
950        String requestContent = readFile(file);
951        RequestHandler requestHandler = new RequestHandler();
952        Request request = requestHandler.toObject(requestContent);
953        return request;
954    }
955
956    public void printInitialLog() throws Exception {
957        printLogInfo("======");
958        printLogInfo("Total record count before cleanup = " + getRecordCount());
959        cleanUpData();
960        printLogInfo("Total record count after cleanup = " + getRecordCount());
961    }
962
963    public void printFinalLog() throws Exception {
964        printLogInfo("Total record count after the test = " + getRecordCount());
965        printLogInfo("======");
966    }
967
968    private IndexerService getIndexerService(RequestDocument requestDocument) {
969        IndexerService indexerService = DocumentIndexerManagerFactory.getInstance().getDocumentIndexManager(requestDocument.getCategory(), requestDocument.getType(), requestDocument.getFormat());
970        return indexerService;
971    }
972
973    private IndexerService getIndexerService(String category, String type, String format) {
974        IndexerService indexerService = DocumentIndexerManagerFactory.getInstance().getDocumentIndexManager(category, type, format);
975        return indexerService;
976    }
977}