1 package org.kuali.ole.batch.ingest;
2
3 import org.apache.commons.lang3.StringUtils;
4 import org.kuali.ole.DataCarrierService;
5 import org.kuali.ole.OLEConstants;
6 import org.kuali.ole.batch.bo.OLEBatchBibImportDataObjects;
7 import org.kuali.ole.batch.bo.OLEBatchBibImportStatistics;
8 import org.kuali.ole.batch.bo.OLEBatchProcessJobDetailsBo;
9 import org.kuali.ole.batch.bo.OLEBatchProcessProfileBo;
10 import org.kuali.ole.batch.document.OLEBatchProcessDefinitionDocument;
11 import org.kuali.ole.batch.helper.BatchBibImportHelper;
12 import org.kuali.ole.batch.helper.OLEBatchProcessDataHelper;
13 import org.kuali.ole.batch.impl.AbstractBatchProcess;
14 import org.kuali.ole.batch.service.BatchProcessBibImportService;
15 import org.kuali.ole.docstore.common.document.content.bib.marc.BibMarcRecord;
16 import org.kuali.ole.docstore.common.document.content.bib.marc.BibMarcRecords;
17 import org.kuali.ole.docstore.common.document.content.bib.marc.OrderBibMarcRecord;
18 import org.kuali.ole.docstore.common.document.content.bib.marc.xstream.BibMarcRecordProcessor;
19 import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader;
20 import org.kuali.rice.krad.service.KRADServiceLocator;
21 import org.slf4j.Logger;
22 import org.slf4j.LoggerFactory;
23
24 import java.util.ArrayList;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28
29
30
31
32
33
34
35
36 public class BatchProcessBibImport extends AbstractBatchProcess {
37
38 public BatchProcessBibImport() {
39 }
40
41 public BatchProcessBibImport(OLEBatchProcessDefinitionDocument processDef, OLEBatchProcessJobDetailsBo job) {
42 this.processDef = processDef;
43 this.job = job;
44 }
45
46 protected BatchBibImportHelper batchBibImportHelper = getBatchBibImportHelper();
47
48 protected BatchBibImportHelper getBatchBibImportHelper() {
49 if(batchBibImportHelper == null) {
50 batchBibImportHelper = new BatchBibImportHelper();
51 }
52 return batchBibImportHelper;
53 }
54
55 private static final Logger LOG = LoggerFactory.getLogger(BatchProcessBibImport.class);
56 OLEBatchBibImportStatistics bibImportStatistics = new OLEBatchBibImportStatistics();
57
58 protected OLEBatchProcessProfileBo oleBatchProcessProfileBo;
59
60 protected String user;
61
62 private BatchProcessBibImportService batchProcessBibImportService;
63
64 private OLEBatchProcessDataHelper oleBatchProcessDataHelper;
65 DataCarrierService dataCarrierService = GlobalResourceLoader.getService(OLEConstants.DATA_CARRIER_SERVICE);
66
67
68 @Override
69 protected void getNextBatch() throws Exception {
70 if (processDef.getChunkSize() < bibImportStatistics.getBibMarcRecordList().size()) {
71 if (bibImportStatistics.getChunkCount() + processDef.getChunkSize() < bibImportStatistics.getBibMarcRecordList().size()) {
72 bibImportStatistics.setBibImportChunkRecordsList(bibImportStatistics.getBibMarcRecordList().subList(bibImportStatistics.getChunkCount(), bibImportStatistics.getChunkCount() + processDef.getChunkSize()));
73 processBatch(bibImportStatistics.getBibImportChunkRecordsList());
74 bibImportStatistics.addChunkCount(processDef.getChunkSize());
75 } else {
76 bibImportStatistics.setBibImportChunkRecordsList(bibImportStatistics.getBibMarcRecordList().subList(bibImportStatistics.getChunkCount(), bibImportStatistics.getBibMarcRecordList().size()));
77 processBatch(bibImportStatistics.getBibImportChunkRecordsList());
78 bibImportStatistics.setChunkCount(bibImportStatistics.getBibMarcRecordList().size());
79 deleteBatchFile();
80 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
81
82 }
83 job.setJobstatistics(bibImportStatistics);
84 }
85 }
86
87 @Override
88 protected void processBatch() throws Exception {
89 try {
90 user = processDef.getUser();
91 if (processDef.getChunkSize() > bibImportStatistics.getBibMarcRecordList().size()) {
92 processBatch(bibImportStatistics.getBibMarcRecordList());
93 job.setJobstatistics(bibImportStatistics);
94 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
95
96 if (StringUtils.isNotEmpty(bibImportStatistics.getErrorBuilder().toString()))
97 createBatchErrorAttachmentFile(bibImportStatistics.getErrorBuilder().toString());
98 }
99 } catch (Exception e) {
100 job.setStatusDesc(OLEConstants.OLEBatchProcess.BIB_IMPORT_FAILURE);
101 LOG.error(String.valueOf(e));
102 e.printStackTrace();
103
104 if (StringUtils.isNotEmpty(bibImportStatistics.getErrorBuilder().toString()))
105 createBatchErrorAttachmentFile(bibImportStatistics.getErrorBuilder().toString());
106
107 }
108 }
109
110
111 @Override
112 protected void prepareForRead() throws Exception {
113 String marcFileContent = getBatchProcessFileContent();
114 bibImportStatistics.setBibMarcRecordList(getBibImportRecords(marcFileContent));
115 job.setIntailJob(bibImportStatistics);
116 oleBatchProcessProfileBo = getBatchProcessProfile(processDef.getBatchProcessProfileId());
117 }
118
119 @Override
120 protected void prepareForWrite() throws Exception {
121
122 }
123
124
125
126
127
128
129
130
131 private List<BibMarcRecord> getBibImportRecords(String marcFileContent) throws Exception {
132 String marcXMLContent = null;
133 try {
134 marcXMLContent = getBatchProcessBibImportService().preProcessMarc(marcFileContent);
135 } catch (Exception ex) {
136 ex.getMessage();
137 List<String> reasonForFailure = new ArrayList<>();
138 reasonForFailure.add("Unable to parse the marc file. Allowed format is UTF-8");
139 reasonForFailure.add("======================================================");
140 reasonForFailure.add(ex.getMessage());
141 dataCarrierService.addData("reasonForBibImportFailure", reasonForFailure);
142 }
143 BibMarcRecords bibRecords = new BibMarcRecordProcessor().fromXML(marcXMLContent);
144 return bibRecords.getRecords();
145 }
146
147
148
149
150
151
152
153 private OLEBatchProcessProfileBo getBatchProcessProfile(String batchProcessProfileId) {
154 Map<String, String> profileIdMap = new HashMap<String, String>();
155 profileIdMap.put(OLEConstants.OLEBatchProcess.BATCH_PROCESS_PROFILE_ID, batchProcessProfileId);
156 return KRADServiceLocator.getBusinessObjectService().findByPrimaryKey(OLEBatchProcessProfileBo.class, profileIdMap);
157 }
158
159
160
161
162
163
164
165 public void processBatch(List<BibMarcRecord> bibMarcRecordList) throws Exception {
166 bibImportStatistics.setTotalCount(bibMarcRecordList.size());
167 OLEBatchBibImportDataObjects oleBatchBibImportDataObjects = getBatchBibImportHelper().processBatch(bibMarcRecordList, oleBatchProcessProfileBo, bibImportStatistics,user);
168 saveAndBuildResult(bibMarcRecordList, oleBatchBibImportDataObjects);
169 }
170
171
172
173
174
175
176
177 protected void saveAndBuildResult(List<BibMarcRecord> bibMarcRecordList, OLEBatchBibImportDataObjects oleBatchBibImportDataObjects) throws Exception {
178
179 List<BibMarcRecord> mismatchRecordList = getBatchProcessBibImportService().saveBatch(bibMarcRecordList, oleBatchBibImportDataObjects, bibImportStatistics);
180
181 bibImportStatistics.setInstanceStatistics(mismatchRecordList);
182
183
184 createMismatchedFiles();
185
186 bibImportStatistics.addSuccessRecord(bibImportStatistics.getTotalCount() - bibImportStatistics.getMismatchRecordList().size());
187
188 }
189
190 private void createMismatchedFiles() throws Exception {
191
192 if (bibImportStatistics.getInvalidLeaderField() != null && bibImportStatistics.getInvalidLeaderField().size() > 0) {
193 List reasonForFailure = (List) dataCarrierService.getData("reasonForFailure");
194 if (reasonForFailure != null) {
195 reasonForFailure.addAll(bibImportStatistics.getInvalidLeaderField());
196 dataCarrierService.addData("reasonForFailure", reasonForFailure);
197 }
198 }
199
200
201 if (bibImportStatistics.getMismatchRecordList().size() > 0) {
202 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getMismatchRecordList())));
203 createBatchFailureFile(bibImportStatistics.getMisMatchMarcRecords().toString());
204 }
205
206 if (bibImportStatistics.getRecordsCreatedWithOutLink().size() > 0) {
207 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getRecordsCreatedWithOutLink())));
208 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.RECORDS_CREATED_WITHOUT_LINK);
209 }
210
211
212 if (bibImportStatistics.getRecordsCreatedWithMoreThanOneLink().size() > 0) {
213 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getRecordsCreatedWithMoreThanOneLink())));
214 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.RECORDS_CREATED_WITH_MORE_THAN_ONE_LINK);
215 }
216
217 if (bibImportStatistics.getMoreThanOneHoldingsMatched().size() > 0) {
218 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getMoreThanOneHoldingsMatched())));
219 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.HOLDINGS_MATCHED_MORE_THAN_ONE);
220 }
221
222 if (bibImportStatistics.getMoreThanOneItemMatched().size() > 0) {
223 bibImportStatistics.setMisMatchMarcRecords(new StringBuffer(new BibMarcRecordProcessor().generateXML(bibImportStatistics.getMoreThanOneItemMatched())));
224 createBatchMismatchFile(bibImportStatistics.getMisMatchMarcRecords().toString(), OLEConstants.OLEBatchProcess.ITEMS_MATCHED_MORE_THAN_ONE);
225 }
226 }
227
228
229 private BatchProcessBibImportService getBatchProcessBibImportService() {
230 if (batchProcessBibImportService == null)
231 batchProcessBibImportService = GlobalResourceLoader.getService("batchProcessBibImportServiceImpl");
232 return batchProcessBibImportService;
233 }
234
235 public OLEBatchProcessDataHelper getOleBatchProcessDataHelper() {
236 if (oleBatchProcessDataHelper == null) oleBatchProcessDataHelper = OLEBatchProcessDataHelper.getInstance();
237 return oleBatchProcessDataHelper;
238 }
239
240 @Override
241 protected void createBatchFailureFile(String misMatchMarcRecords) throws Exception {
242 getOleBatchProcessDataHelper().createBatchBibImportFailureFile(misMatchMarcRecords, processDef.getBatchProcessType(), job.getJobId() + "_FailureRecord" + "_" + job.getUploadFileName(), job.getJobId());
243 }
244
245 protected void createBatchMismatchFile(String misMatchMarcRecords, String recordName) throws Exception {
246 getOleBatchProcessDataHelper().createBatchBibImportFailureFile(misMatchMarcRecords, processDef.getBatchProcessType(), job.getJobId() + recordName + job.getUploadFileName(), job.getJobId());
247 }
248
249 public OLEBatchProcessProfileBo getOleBatchProcessProfileBo() {
250 return oleBatchProcessProfileBo;
251 }
252
253 public void setOleBatchProcessProfileBo(OLEBatchProcessProfileBo oleBatchProcessProfileBo) {
254 this.oleBatchProcessProfileBo = oleBatchProcessProfileBo;
255 }
256
257
258 public List<OrderBibMarcRecord> processBatchOrder(List<OrderBibMarcRecord> orderBibMarcRecords) throws Exception {
259 bibImportStatistics.setTotalCount(orderBibMarcRecords.size());
260 BatchBibImportHelper batchBibImportHelper = new BatchBibImportHelper();
261 OLEBatchBibImportDataObjects oleBatchBibImportDataObjects = batchBibImportHelper.processOrderBatch(orderBibMarcRecords, oleBatchProcessProfileBo, bibImportStatistics,user);
262 List<OrderBibMarcRecord> orderBibMarcRecordList = getBatchProcessBibImportService().saveOderBatch(orderBibMarcRecords, oleBatchBibImportDataObjects, bibImportStatistics);
263 createMismatchedFiles();
264 bibImportStatistics.addSuccessRecord(bibImportStatistics.getTotalCount() - bibImportStatistics.getMismatchRecordList().size());
265 return orderBibMarcRecordList;
266 }
267
268 }