001 package org.kuali.common.impex.data.service.impl;
002
003 import java.io.File;
004 import java.io.IOException;
005 import java.io.OutputStream;
006 import java.util.List;
007
008 import org.apache.commons.lang3.StringUtils;
009 import org.kuali.common.impex.data.service.DumpDataContext;
010 import org.kuali.common.impex.model.Column;
011 import org.kuali.common.impex.model.util.ModelUtils;
012 import org.kuali.common.util.CollectionUtils;
013 import org.kuali.common.util.FormatUtils;
014 import org.kuali.common.util.LocationUtils;
015 import org.kuali.common.util.Str;
016 import org.slf4j.Logger;
017 import org.slf4j.LoggerFactory;
018
019 public class DataHandler {
020
021 private static final Logger logger = LoggerFactory.getLogger(DataHandler.class);
022
023 protected static final String FS = File.separator;
024 protected static final String LF = "\n";
025 protected static final String DOUBLE_QUOTE = "\"";
026 protected static final String COMMA = ",";
027
028 public static final String MPX_EXTENSION = ".mpx";
029
030 public static final String CARRIAGE_RETURN = "\r";
031
032 public static final String MPX_NULL = "${mpx.null}";
033 public static final String MPX_CARRIAGE_RETURN = "${mpx.cr}";
034 public static final String MPX_LINEFEED = "${mpx.lf}";
035 public static final String MPX_QUOTE = "${mpx.quote}";
036
037 public static final String MPX_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ";
038
039 public static File getFileForTable(DumpDataContext context, String tableName) {
040 String filename = getFilename(context.getWorkingDir(), tableName);
041 return new File(filename);
042 }
043
044 protected static String getFilename(File workingDir, String tableName) {
045 // To keep file names consistent, always capitalize the name of the table before returning a file name
046 tableName = StringUtils.upperCase(tableName);
047 return LocationUtils.getCanonicalPath(workingDir) + FS + tableName + MPX_EXTENSION;
048 }
049
050 public static void startData(DumpProgress progress) throws IOException {
051 List<Column> columns = progress.getColumns();
052 String encoding = progress.getContext().getEncoding();
053 String header = ModelUtils.getCsvColumnNames(columns) + LF;
054 OutputStream out = progress.getOutputStream();
055 out.write(header.getBytes(encoding));
056 }
057
058 public static void doData(DumpProgress progress) throws IOException {
059 String encoding = progress.getContext().getEncoding();
060 formatMpx(progress.getCurrentData());
061 writeRows(progress.getCurrentData(), encoding, progress.getOutputStream());
062 }
063
064 protected static void writeRows(List<List<String>> rows, String encoding, OutputStream out) throws IOException {
065 for (List<String> row : rows) {
066 String line = getLine(row);
067 byte[] bytes = line.getBytes(encoding);
068 out.write(bytes);
069 }
070 }
071
072 /**
073 * Comma separated, all values enclosed in double quotes, always terminated by a linefeed
074 */
075 protected static String getLine(List<String> row) {
076 StringBuilder sb = new StringBuilder();
077 for (int column = 0; column < row.size(); column++) {
078 if (column != 0) {
079 sb.append(COMMA);
080 }
081 String columnValue = row.get(column);
082 sb.append(Str.quote(columnValue));
083 }
084 sb.append(LF);
085 return sb.toString();
086 }
087
088 public static void finishData(DumpProgress exportProgress) throws IOException {
089
090 // Since we only dump from memory to disk at specific intervals, there is likely to be
091 // a few trailing rows that need to get dumped
092 if (!CollectionUtils.isEmpty(exportProgress.getCurrentData())) {
093 String encoding = exportProgress.getContext().getEncoding();
094 formatMpx(exportProgress.getCurrentData());
095 writeRows(exportProgress.getCurrentData(), encoding, exportProgress.getOutputStream());
096 }
097
098 // Extract our table tracker
099 TableTracker tracker = exportProgress.getTableTracker();
100
101 // If the table had at least one row of data there will be a .mpx file for this table
102 // Log a message displaying some stats about what got dumped to disk
103 if (tracker.getTotalRowCount().getValue() > 0) {
104 long threadId = Thread.currentThread().getId();
105 String tableName = exportProgress.getTableContext().getTable().getName();
106 String trc = FormatUtils.getCount(tracker.getTotalRowCount().getValue());
107 String tds = FormatUtils.getSize(tracker.getTotalDataSize().getValue());
108 Object[] args = { threadId, tableName, trc, tds };
109 logger.debug("[{}] - Dumped [{}] Total Rows: {} Total Size: {}", args);
110 }
111 }
112
113 protected static void formatMpx(List<List<String>> rows) {
114 for (List<String> row : rows) {
115 for (int i = 0; i < row.size(); i++) {
116 row.set(i, formatMpx(row.get(i)));
117 }
118 }
119 }
120
121 public static String formatMpx(String s) {
122 if (s == null) {
123 return MPX_NULL;
124 }
125 String converted = StringUtils.replace(s, CARRIAGE_RETURN, MPX_CARRIAGE_RETURN);
126 converted = StringUtils.replace(converted, LF, MPX_LINEFEED);
127 converted = StringUtils.replace(converted, DOUBLE_QUOTE, MPX_QUOTE);
128 return converted;
129 }
130
131 }