1 package org.kuali.ole.batch.ingest;
2
3 import org.apache.commons.lang3.StringUtils;
4 import org.kuali.ole.DataCarrierService;
5 import org.kuali.ole.OLEConstants;
6 import org.kuali.ole.batch.bo.OLEBatchBibImportDataObjects;
7 import org.kuali.ole.batch.bo.OLEBatchBibImportStatistics;
8 import org.kuali.ole.batch.bo.OLEBatchProcessJobDetailsBo;
9 import org.kuali.ole.batch.bo.OLEBatchProcessProfileBo;
10 import org.kuali.ole.batch.document.OLEBatchProcessDefinitionDocument;
11 import org.kuali.ole.batch.helper.BatchBibImportHelper;
12 import org.kuali.ole.batch.helper.OLEBatchProcessDataHelper;
13 import org.kuali.ole.batch.impl.AbstractBatchProcess;
14 import org.kuali.ole.batch.service.BatchProcessBibImportService;
15 import org.kuali.ole.docstore.common.document.content.bib.marc.BibMarcRecord;
16 import org.kuali.ole.docstore.common.document.content.bib.marc.BibMarcRecords;
17 import org.kuali.ole.docstore.common.document.content.bib.marc.OrderBibMarcRecord;
18 import org.kuali.ole.docstore.common.document.content.bib.marc.xstream.BibMarcRecordProcessor;
19 import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader;
20 import org.kuali.rice.krad.service.KRADServiceLocator;
21 import org.slf4j.Logger;
22 import org.slf4j.LoggerFactory;
23
24 import java.util.ArrayList;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28
29
30
31
32
33
34
35
36 public class BatchProcessBibImport extends AbstractBatchProcess {
37
38 public BatchProcessBibImport() {
39 }
40
41 public BatchProcessBibImport(OLEBatchProcessDefinitionDocument processDef, OLEBatchProcessJobDetailsBo job) {
42 this.processDef = processDef;
43 this.job = job;
44 }
45
46 private static final Logger LOG = LoggerFactory.getLogger(BatchProcessBibImport.class);
47 OLEBatchBibImportStatistics bibImportStatistics = new OLEBatchBibImportStatistics();
48
49 private OLEBatchProcessProfileBo oleBatchProcessProfileBo;
50
51 private String user;
52
53 private BatchProcessBibImportService batchProcessBibImportService;
54
55 private OLEBatchProcessDataHelper oleBatchProcessDataHelper;
56 DataCarrierService dataCarrierService = GlobalResourceLoader.getService(OLEConstants.DATA_CARRIER_SERVICE);
57
58
59 @Override
60 protected void getNextBatch() throws Exception {
61 if (processDef.getChunkSize() < bibImportStatistics.getBibMarcRecordList().size()) {
62 if (bibImportStatistics.getChunkCount() + processDef.getChunkSize() < bibImportStatistics.getBibMarcRecordList().size()) {
63 bibImportStatistics.setBibImportChunkRecordsList(bibImportStatistics.getBibMarcRecordList().subList(bibImportStatistics.getChunkCount(), bibImportStatistics.getChunkCount() + processDef.getChunkSize()));
64 processBatch(bibImportStatistics.getBibImportChunkRecordsList());
65 bibImportStatistics.addChunkCount(processDef.getChunkSize());
66 } else {
67 bibImportStatistics.setBibImportChunkRecordsList(bibImportStatistics.getBibMarcRecordList().subList(bibImportStatistics.getChunkCount(), bibImportStatistics.getBibMarcRecordList().size()));
68 processBatch(bibImportStatistics.getBibImportChunkRecordsList());
69 bibImportStatistics.setChunkCount(bibImportStatistics.getBibMarcRecordList().size());
70 deleteBatchFile();
71 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
72
73 }
74 job.setJobstatistics(bibImportStatistics);
75 }
76 }
77
78 @Override
79 protected void processBatch() throws Exception {
80 try {
81 user = processDef.getUser();
82 if (processDef.getChunkSize() > bibImportStatistics.getBibMarcRecordList().size()) {
83 processBatch(bibImportStatistics.getBibMarcRecordList());
84 job.setJobstatistics(bibImportStatistics);
85 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
86
87 if (StringUtils.isNotEmpty(bibImportStatistics.getErrorBuilder().toString()))
88 createBatchErrorAttachmentFile(bibImportStatistics.getErrorBuilder().toString());
89 }
90 } catch (Exception e) {
91 job.setStatusDesc(OLEConstants.OLEBatchProcess.BIB_IMPORT_FAILURE);
92 LOG.error(String.valueOf(e));
93
94 if (StringUtils.isNotEmpty(bibImportStatistics.getErrorBuilder().toString()))
95 createBatchErrorAttachmentFile(bibImportStatistics.getErrorBuilder().toString());
96
97 }
98 }
99
100
101 @Override
102 protected void prepareForRead() throws Exception {
103 String marcFileContent = getBatchProcessFileContent();
104 bibImportStatistics.setBibMarcRecordList(getBibImportRecords(marcFileContent));
105 job.setIntailJob(bibImportStatistics);
106 oleBatchProcessProfileBo = getBatchProcessProfile(processDef.getBatchProcessProfileId());
107 }
108
109 @Override
110 protected void prepareForWrite() throws Exception {
111
112 }
113
114
115
116
117
118
119
120
121 private List<BibMarcRecord> getBibImportRecords(String marcFileContent) throws Exception {
122 String marcXMLContent = null;
123 try {
124 marcXMLContent = getBatchProcessBibImportService().preProcessMarc(marcFileContent);
125 } catch (Exception ex) {
126 ex.getMessage();
127 List<String> reasonForFailure = new ArrayList<>();
128 reasonForFailure.add("Unable to parse the marc file. Allowed format is UTF-8");
129 reasonForFailure.add("======================================================");
130 reasonForFailure.add(ex.getMessage());
131 dataCarrierService.addData("reasonForBibImportFailure", reasonForFailure);
132 }
133 BibMarcRecords bibRecords = new BibMarcRecordProcessor().fromXML(marcXMLContent);
134 return bibRecords.getRecords();
135 }
136
137
138
139
140
141
142
143 private OLEBatchProcessProfileBo getBatchProcessProfile(String batchProcessProfileId) {
144 Map<String, String> profileIdMap = new HashMap<String, String>();
145 profileIdMap.put(OLEConstants.OLEBatchProcess.BATCH_PROCESS_PROFILE_ID, batchProcessProfileId);
146 return KRADServiceLocator.getBusinessObjectService().findByPrimaryKey(OLEBatchProcessProfileBo.class, profileIdMap);
147 }
148
149
150
151
152
153
154
155 public void processBatch(List<BibMarcRecord> bibMarcRecordList) throws Exception {
156 bibImportStatistics.setTotalCount(bibMarcRecordList.size());
157 BatchBibImportHelper batchBibImportHelper = new BatchBibImportHelper();
158 OLEBatchBibImportDataObjects oleBatchBibImportDataObjects = batchBibImportHelper.processBatch(bibMarcRecordList, oleBatchProcessProfileBo, bibImportStatistics,user);
159 saveAndBuildResult(bibMarcRecordList, oleBatchBibImportDataObjects);
160 }
161
162
163
164
165
166
167
168 private void saveAndBuildResult(List<BibMarcRecord> bibMarcRecordList, OLEBatchBibImportDataObjects oleBatchBibImportDataObjects) throws Exception {
169
170 List<BibMarcRecord> mismatchRecordList = getBatchProcessBibImportService().saveBatch(bibMarcRecordList, oleBatchBibImportDataObjects, bibImportStatistics);
171
172 bibImportStatistics.setInstanceStatistics(mismatchRecordList);
173
174
175 createMismatchedFiles();
176
177 bibImportStatistics.addSuccessRecord(bibImportStatistics.getTotalCount() - bibImportStatistics.getMismatchRecordList().size());
178
179 }
180
181 private void createMismatchedFiles() throws Exception {
182
183 if (bibImportStatistics.getInvalidLeaderField() != null && bibImportStatistics.getInvalidLeaderField().size() > 0) {
184 List reasonForFailure = (List) dataCarrierService.getData("reasonForFailure");
185 if (reasonForFailure != null) {
186 reasonForFailure.addAll(bibImportStatistics.getInvalidLeaderField());
187 dataCarrierService.addData("reasonForFailure", reasonForFailure);
188 }
189 }
190
191
192 if (bibImportStatistics.getMismatchRecordList().size() > 0) {
193 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getMismatchRecordList())));
194 createBatchFailureFile(bibImportStatistics.getMisMatchMarcRecords().toString());
195 }
196
197 if (bibImportStatistics.getRecordsCreatedWithOutLink().size() > 0) {
198 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getRecordsCreatedWithOutLink())));
199 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.RECORDS_CREATED_WITHOUT_LINK);
200 }
201
202
203 if (bibImportStatistics.getRecordsCreatedWithMoreThanOneLink().size() > 0) {
204 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getRecordsCreatedWithMoreThanOneLink())));
205 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.RECORDS_CREATED_WITH_MORE_THAN_ONE_LINK);
206 }
207
208 if (bibImportStatistics.getMoreThanOneHoldingsMatched().size() > 0) {
209 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getMoreThanOneHoldingsMatched())));
210 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.HOLDINGS_MATCHED_MORE_THAN_ONE);
211 }
212
213 if (bibImportStatistics.getMoreThanOneItemMatched().size() > 0) {
214 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getMoreThanOneItemMatched())));
215 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.ITEMS_MATCHED_MORE_THAN_ONE);
216 }
217 }
218
219
220 private BatchProcessBibImportService getBatchProcessBibImportService() {
221 if (batchProcessBibImportService == null)
222 batchProcessBibImportService = GlobalResourceLoader.getService("batchProcessBibImportServiceImpl");
223 return batchProcessBibImportService;
224 }
225
226 public OLEBatchProcessDataHelper getOleBatchProcessDataHelper() {
227 if (oleBatchProcessDataHelper == null) oleBatchProcessDataHelper = OLEBatchProcessDataHelper.getInstance();
228 return oleBatchProcessDataHelper;
229 }
230
231 @Override
232 protected void createBatchFailureFile(String misMatchMarcRecords) throws Exception {
233 getOleBatchProcessDataHelper().createBatchBibImportFailureFile(misMatchMarcRecords, processDef.getBatchProcessType(), job.getJobId() + "_FailureRecord" + "_" + job.getUploadFileName(), job.getJobId());
234 }
235
236 protected void createBatchMismatchFile(String misMatchMarcRecords, String recordName) throws Exception {
237 getOleBatchProcessDataHelper().createBatchBibImportFailureFile(misMatchMarcRecords, processDef.getBatchProcessType(), job.getJobId() + recordName + job.getUploadFileName(), job.getJobId());
238 }
239
240 public OLEBatchProcessProfileBo getOleBatchProcessProfileBo() {
241 return oleBatchProcessProfileBo;
242 }
243
244 public void setOleBatchProcessProfileBo(OLEBatchProcessProfileBo oleBatchProcessProfileBo) {
245 this.oleBatchProcessProfileBo = oleBatchProcessProfileBo;
246 }
247
248
249 public List<OrderBibMarcRecord> processBatchOrder(List<OrderBibMarcRecord> orderBibMarcRecords) throws Exception {
250 bibImportStatistics.setTotalCount(orderBibMarcRecords.size());
251 BatchBibImportHelper batchBibImportHelper = new BatchBibImportHelper();
252 OLEBatchBibImportDataObjects oleBatchBibImportDataObjects = batchBibImportHelper.processOrderBatch(orderBibMarcRecords, oleBatchProcessProfileBo, bibImportStatistics,user);
253 List<OrderBibMarcRecord> orderBibMarcRecordList = getBatchProcessBibImportService().saveOderBatch(orderBibMarcRecords, oleBatchBibImportDataObjects, bibImportStatistics);
254 createMismatchedFiles();
255 bibImportStatistics.addSuccessRecord(bibImportStatistics.getTotalCount() - bibImportStatistics.getMismatchRecordList().size());
256 return orderBibMarcRecordList;
257 }
258
259 }