1 package org.kuali.ole.batch.ingest;
2
3 import org.apache.log4j.Logger;
4 import org.kuali.ole.OLEConstants;
5 import org.kuali.ole.batch.bo.OLEBatchProcessProfileMatchPoint;
6 import org.kuali.ole.batch.impl.AbstractBatchProcess;
7 import org.kuali.ole.batch.service.BatchProcessDeleteService;
8 import org.kuali.ole.batch.util.BatchBibImportUtil;
9 import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader;
10 import org.marc4j.*;
11 import org.marc4j.marc.Record;
12 import org.marc4j.marc.impl.ControlFieldImpl;
13 import org.marc4j.marc.impl.DataFieldImpl;
14
15 import java.io.*;
16 import java.util.*;
17
18
19
20
21
22
23
24
25 public class BatchProcessDeleteData extends AbstractBatchProcess {
26
27 private static final Logger LOG = Logger.getLogger(BatchProcessDeleteData.class);
28 private BatchProcessDeleteService batchProcessDeleteService;
29
30 private List docBibIdsList;
31 private List deleteChunkBibIdsList = new ArrayList(0);
32 private int chunkCount = 0;
33 private int totalCount = 0;
34 private int failureCount = 0;
35 private StringBuffer misMatchMarcRecords = new StringBuffer("");
36 private StringBuffer matchMarcRecords = new StringBuffer("");
37 private StringBuffer failureReport = new StringBuffer("");
38 private String fileType = "mrc";
39
40 private BatchProcessDeleteService getBatchProcessDeleteService() {
41 if (batchProcessDeleteService == null)
42 batchProcessDeleteService = GlobalResourceLoader.getService("batchDeleteServiceImpl");
43 return batchProcessDeleteService;
44 }
45
46 @Override
47 protected void prepareForRead() throws Exception {
48
49 String fileContent = getBatchProcessFileContent();
50 job.setNoOfSuccessRecords("0");
51 job.setNoOfFailureRecords("0");
52 job.setNoOfRecordsProcessed("0");
53 if (job.getUploadFileName() != null && job.getUploadFileName().contains(".txt")) {
54 fileType = "txt";
55 }
56 docBibIdsList = getBatchDeleteBibIdsList(fileContent);
57 job.setTotalNoOfRecords(totalCount + "");
58
59 }
60
61 @Override
62 protected void prepareForWrite() throws Exception {
63 if (docBibIdsList != null && docBibIdsList.size() > 0) {
64 if (processDef.getChunkSize() == 1000) {
65 String profileField = getProfileField();
66 if (profileField != null) {
67 int sucRecordCount = getBatchProcessDeleteService().performBatchDelete(docBibIdsList, profileField);
68 deleteBatchFile();
69 createBatchFailureFile();
70 job.setNoOfRecordsProcessed(totalCount + "");
71 job.setNoOfSuccessRecords((totalCount - failureCount) + "");
72 job.setNoOfFailureRecords(failureCount + "");
73 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
74 }
75 }
76 } else {
77 job.setNoOfRecordsProcessed(totalCount + "");
78 job.setNoOfSuccessRecords((totalCount - failureCount) + "");
79 job.setNoOfFailureRecords(failureCount + "");
80 deleteBatchFile();
81 createBatchFailureFile();
82 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
83 }
84
85
86 }
87
88 @Override
89 protected void getNextBatch() throws Exception {
90 if (chunkCount < docBibIdsList.size()) {
91 if (chunkCount + processDef.getChunkSize() < docBibIdsList.size()) {
92 deleteChunkBibIdsList = docBibIdsList.subList(chunkCount, chunkCount + processDef.getChunkSize());
93 chunkCount += processDef.getChunkSize();
94 } else {
95 deleteChunkBibIdsList = docBibIdsList.subList(chunkCount, docBibIdsList.size());
96 performBatchDelete();
97 chunkCount += processDef.getChunkSize();
98 deleteBatchFile();
99 createBatchFailureFile();
100 job.setStatus(OLEConstants.OLEBatchProcess.JOB_STATUS_COMPLETED);
101 }
102
103 }
104
105 }
106
107 @Override
108 protected void processBatch() throws Exception {
109
110 if (deleteChunkBibIdsList != null && deleteChunkBibIdsList.size() > 0) {
111
112 performBatchDelete();
113
114 }
115 }
116
117
118
119
120
121
122 private String getProfileField() {
123 if (processDef.getBatchProcessProfileBo() != null) {
124 List<OLEBatchProcessProfileMatchPoint> bibMatchPointList = BatchBibImportUtil.buildMatchPointListByDataType(processDef.getBatchProcessProfileBo().getOleBatchProcessProfileMatchPointList(), org.kuali.ole.docstore.common.document.content.enums.DocType.BIB.getCode());
125 if (bibMatchPointList != null && bibMatchPointList.size() > 0) {
126 String profileField = bibMatchPointList.get(0).getMatchPoint();
127 return profileField;
128 }
129 }
130 return null;
131 }
132
133
134
135
136
137
138
139 public List getBatchDeleteBibIdsList(String fileContent) throws Exception {
140
141 String profileField = getProfileField();
142 List matchBibIdsList = new ArrayList(0);
143 totalCount = 0;
144 if (profileField != null) {
145 String profileDataFiled = profileField.substring(0, 3);
146 char profileSubField = ' ';
147 if(profileField!=null && profileField.contains("$")) {
148 profileSubField = profileField.substring(profileField.lastIndexOf('$') + 1).toCharArray()[0];
149 } else if (profileField != null && profileField.length() > 3) {
150 profileSubField = profileField.substring(3).toCharArray()[0];
151 }
152 String profileFieldQuery = profileDataFiled;
153 if (profileField != null && profileField.length() > 3) {
154 profileFieldQuery = profileFieldQuery + profileSubField;
155 profileFieldQuery = profileFieldQuery.trim();
156 }
157 try {
158
159 if ("mrc".equals(fileType)) {
160 ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(fileContent.getBytes());
161 ByteArrayOutputStream failureByteArrayOutputStream = new ByteArrayOutputStream();
162 ByteArrayOutputStream successByteArrayOutputStream = new ByteArrayOutputStream();
163 MarcReader reader = new MarcStreamReader(byteArrayInputStream);
164 MarcWriter failureWriter = new MarcStreamWriter(failureByteArrayOutputStream);
165 MarcWriter successWriter = new MarcStreamWriter(successByteArrayOutputStream);
166 while (reader.hasNext()) {
167 Record record = reader.next();
168 totalCount++;
169 boolean recExistFlag = true;
170 for (DataFieldImpl dataFieldImpl : (List<DataFieldImpl>) record.getDataFields()) {
171 String searchMrcFieldData = "";
172 if (profileDataFiled.equals(dataFieldImpl.getTag())) {
173 searchMrcFieldData = dataFieldImpl.getSubfield(profileSubField).getData().replaceAll(":", "");
174 Map batchDeleteMap = getBatchProcessDeleteService().getBibIdsForBatchDelete(searchMrcFieldData, profileFieldQuery);
175 List bibIdList = (List) batchDeleteMap.get(OLEConstants.BIB_SEARCH);
176 if (bibIdList == null || (bibIdList != null && bibIdList.size() > 1)) {
177 failureCount++;
178 failureWriter.write(record);
179 } else {
180 matchBibIdsList.addAll(bibIdList);
181 successWriter.write(record);
182 }
183 String failureInfo = (String) batchDeleteMap.get(OLEConstants.OLEBatchProcess.FAILURE_INFO);
184 if (failureInfo != null) {
185 failureReport.append(searchMrcFieldData + " (" + profileField + ") :: " + failureInfo + "\n");
186 }
187 recExistFlag = false;
188 break;
189 }
190 }
191
192 if (OLEConstants.OLEBatchProcess.CONTROL_FIELD_001.equals(profileDataFiled)) {
193 for (ControlFieldImpl controlField : (List<ControlFieldImpl>) record.getControlFields()) {
194 if (!OLEConstants.OLEBatchProcess.CONTROL_FIELD_001.equalsIgnoreCase(controlField.getTag()))
195 continue;
196 Map batchDeleteMap = getBatchProcessDeleteService().getBibIdsForBatchDelete(controlField.getData(), profileFieldQuery);
197 List bibIdList = (List) batchDeleteMap.get(OLEConstants.BIB_SEARCH);
198 if (bibIdList == null || (bibIdList != null && bibIdList.size() > 1)) {
199 failureCount++;
200 failureWriter.write(record);
201 } else {
202 matchBibIdsList.addAll(bibIdList);
203 successWriter.write(record);
204 }
205 String failureInfo = (String) batchDeleteMap.get(OLEConstants.OLEBatchProcess.FAILURE_INFO);
206 if (failureInfo != null) {
207 failureReport.append(controlField.getData() + " (" + profileField + ") :: " + failureInfo + "\n");
208 }
209 recExistFlag = false;
210 break;
211
212
213 }
214 }
215
216
217 if (recExistFlag) {
218 failureCount++;
219 failureWriter.write(record);
220 }
221
222 }
223 misMatchMarcRecords.append(failureByteArrayOutputStream.toString());
224 matchMarcRecords.append(successByteArrayOutputStream.toString());
225 failureWriter.close();
226 successWriter.close();
227 }
228
229 else if ("txt".equals(fileType)) {
230 List txtFileDataList = new ArrayList();
231 BufferedReader br = new BufferedReader(new StringReader(fileContent));
232 StringBuilder sb = new StringBuilder();
233 String line = br.readLine();
234 while (line != null) {
235 totalCount++;
236 if (!txtFileDataList.contains(line)) {
237 txtFileDataList.add(line);
238 Map batchDeleteMap = getBatchProcessDeleteService().getBibIdsForBatchDelete(line, profileFieldQuery);
239 List bibIdList = (List) batchDeleteMap.get("bibIdentifier");
240 if (bibIdList == null || (bibIdList != null && bibIdList.size() > 1)) {
241 failureCount++;
242 misMatchMarcRecords.append(line.toString() + "\n");
243 } else {
244 matchBibIdsList.addAll(bibIdList);
245 matchMarcRecords.append(line.toString() + "\n");
246 }
247 String failureInfo = (String) batchDeleteMap.get("failureInfo");
248 if (failureInfo != null) {
249 failureReport.append(line + " (" + profileField + ") :: " + failureInfo + "\n");
250 }
251 } else {
252 failureCount++;
253 misMatchMarcRecords.append(line.toString() + "\n");
254 }
255 line = br.readLine();
256 }
257 }
258 } catch (Exception e) {
259 LOG.error("exception : " + e.getMessage());
260 }
261 }
262 return matchBibIdsList;
263 }
264
265
266
267
268
269 public void performBatchDelete() throws Exception {
270 String profileField = getProfileField();
271 if (profileField != null) {
272
273 int sucRecordCount = getBatchProcessDeleteService().performBatchDelete(deleteChunkBibIdsList, profileField);
274
275 job.setNoOfSuccessRecords((Integer.parseInt(job.getNoOfSuccessRecords()) + sucRecordCount) + "");
276 job.setNoOfFailureRecords(totalCount - (Integer.parseInt(job.getNoOfSuccessRecords())) + "");
277 job.setNoOfRecordsProcessed((Integer.parseInt(job.getNoOfRecordsProcessed()) + deleteChunkBibIdsList.size()) + "");
278 }
279 }
280
281
282
283
284
285 public void createBatchFailureFile() throws Exception {
286
287 if (!"".equals(misMatchMarcRecords.toString())) {
288 createBatchFailureFile(misMatchMarcRecords.toString());
289 }
290 if (!"".equals(matchMarcRecords.toString())) {
291 createBatchSuccessFile(matchMarcRecords.toString());
292 }
293 if (!"".equals(failureReport.toString())) {
294 createBatchDeleteFailureReportFile(failureReport.toString());
295 }
296
297 }
298
299
300 }