1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.apache.torque.mojo;
17
18 import java.io.ByteArrayInputStream;
19 import java.io.File;
20 import java.io.IOException;
21 import java.io.InputStream;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.List;
25
26 import org.apache.commons.io.FileUtils;
27 import org.apache.commons.io.IOUtils;
28 import org.apache.commons.lang.StringUtils;
29 import org.apache.maven.plugin.MojoExecutionException;
30 import org.apache.maven.plugin.MojoFailureException;
31 import org.apache.torque.util.CloverETLColumn;
32 import org.apache.torque.util.CloverETLTable;
33
34
35
36
37
38
39 public class ConvertCloverETLMojo extends BaseMojo {
40
41
42
43
44
45
46
47 File sourceDir;
48
49
50
51
52
53
54
55 File outputDir;
56
57
58
59
60
61
62
63 String delimiter;
64
65
66
67
68
69
70
71 String schemaFilename;
72
73
74
75
76
77
78
79 String databaseDTDFilename;
80
81
82
83
84
85
86
87 String dataDTDFilename;
88
89 @Override
90 protected void executeMojo() throws MojoExecutionException, MojoFailureException {
91 getLog().info("Examining " + sourceDir.getAbsolutePath());
92 handleSchema();
93 handleData();
94 }
95
96 protected void handleSchema() {
97 try {
98 handleDataDTD();
99 File newSchemaFile = new File(outputDir + "/" + schemaFilename);
100 File oldSchemaFile = new File(sourceDir + "/" + schemaFilename);
101 getLog().info("Creating " + newSchemaFile.getCanonicalPath());
102 FileUtils.copyFile(oldSchemaFile, newSchemaFile);
103 File newDatabaseDTDFile = new File(outputDir + "/" + databaseDTDFilename);
104 File oldDatabaseDTDFile = new File(sourceDir + "/" + databaseDTDFilename);
105 getLog().info("Creating " + newDatabaseDTDFile.getCanonicalPath());
106 FileUtils.copyFile(oldDatabaseDTDFile, newDatabaseDTDFile);
107 } catch (IOException e) {
108 throw new IllegalStateException("Unexpected IO error", e);
109 }
110 }
111
112 protected String[] parseAll(String s, String open, String close) {
113 String[] tokens = StringUtils.substringsBetween(s, open, close);
114 if (tokens == null) {
115 return null;
116 }
117 for (int i = 0; i < tokens.length; i++) {
118 tokens[i] = open + tokens[i] + close;
119 }
120 return tokens;
121
122 }
123
124 protected void handleDataDTD() throws IOException {
125 File schemaFile = new File(sourceDir + "/" + schemaFilename);
126 String contents = FileUtils.readFileToString(schemaFile);
127 String[] tables = getTables(contents);
128 getLog().info("Located " + tables.length + " schema tables");
129 List<CloverETLTable> realTables = new ArrayList<CloverETLTable>();
130 for (String table : tables) {
131 CloverETLTable realTable = getDataDTDTable(table);
132 realTables.add(realTable);
133 }
134 String content = getDataDTDContent(realTables);
135 File dataDTDFile = new File(outputDir + "/" + dataDTDFilename);
136 getLog().info("Creating " + dataDTDFile.getCanonicalPath());
137 FileUtils.writeStringToFile(dataDTDFile, content);
138 }
139
140 protected String getDataDTDContent(List<CloverETLTable> tables) {
141 StringBuilder sb = new StringBuilder();
142 sb.append(getProlog(tables));
143 for (CloverETLTable table : tables) {
144 sb.append(getTableDTDContent(table));
145 }
146 return sb.toString();
147 }
148
149 protected String getTableDTDContent(CloverETLTable table) {
150 StringBuilder sb = new StringBuilder();
151 sb.append("<!ELEMENT " + table.getName() + " EMPTY>\n");
152 sb.append("<!ATTLIST " + table.getName() + "\n");
153 sb.append(getColumnDTDContent(table.getEtlColumns()));
154 sb.append(">\n");
155 return sb.toString();
156 }
157
158 protected String getColumnDTDContent(List<CloverETLColumn> columns) {
159 StringBuilder sb = new StringBuilder();
160 for (CloverETLColumn column : columns) {
161 sb.append(" ");
162 sb.append(column.getName());
163 sb.append(" ");
164 sb.append("CDATA");
165 sb.append(" ");
166 sb.append(column.isRequired() ? "#IMPLIED" : "#REQUIRED");
167 sb.append("\n");
168 }
169 return sb.toString();
170 }
171
172 protected String getProlog(List<CloverETLTable> tables) {
173 StringBuilder sb = new StringBuilder();
174 sb.append("<!ELEMENT dataset (\n");
175 for (int i = 0; i < tables.size(); i++) {
176 CloverETLTable table = tables.get(i);
177 sb.append(" " + table.getName());
178 if (i < tables.size() - 1) {
179 sb.append("|\n");
180 } else {
181 sb.append(")*>\n");
182 }
183 }
184 return sb.toString();
185 }
186
187 protected CloverETLTable getDataDTDTable(String s) {
188 String tablename = StringUtils.substringBetween(s, "<table name=\"", "\"");
189 String[] columns = parseAll(s, "<column ", "/>");
190 List<CloverETLColumn> realColumns = new ArrayList<CloverETLColumn>();
191 for (String column : columns) {
192 realColumns.add(getCloverETLColumn(column));
193 }
194
195 CloverETLTable table = new CloverETLTable();
196 table.setName(tablename);
197 table.setEtlColumns(realColumns);
198 return table;
199 }
200
201 protected CloverETLColumn getCloverETLColumn(String s) {
202 String columnName = StringUtils.substringBetween(s, "name=\"", "\"");
203 boolean required = s.contains("required=\"true\"");
204 CloverETLColumn cec = new CloverETLColumn();
205 cec.setName(columnName);
206 cec.setRequired(required);
207 return cec;
208 }
209
210 protected String[] getTables(String contents) {
211 String begin = "<table name=\"";
212 String close = "</table>";
213 return parseAll(contents, begin, close);
214 }
215
216 protected String parse(String s, String begin, String close) {
217 String between = StringUtils.substringBetween(s, begin, close);
218 return begin + between + close;
219 }
220
221 protected void handleData() {
222 File dataDir = new File(sourceDir + "/data");
223 File[] files = dataDir.listFiles();
224 if (files == null) {
225 getLog().info("Converting 0 Clover ETL data files");
226 } else {
227 getLog().info("Converting " + files.length + " Clover ETL data files");
228 Arrays.sort(files);
229 for (File file : files) {
230 convertFile(file);
231 }
232 }
233 }
234
235 protected void convertFile(File file) {
236 try {
237 CloverETLTable table = getTable(file);
238 String xml = getXml(table);
239 File outputFile = new File(outputDir + "/" + table.getName() + ".xml");
240 getLog().info("Creating " + outputFile.getCanonicalPath());
241 FileUtils.writeStringToFile(outputFile, xml);
242 } catch (IOException e) {
243 throw new IllegalStateException("Unexpected IO error", e);
244 }
245 }
246
247 protected CloverETLTable getTable(File file) throws IOException {
248 String filename = file.getName().toUpperCase();
249 int pos = filename.indexOf(".");
250 String tablename = filename.substring(0, pos);
251
252 String content = FileUtils.readFileToString(file);
253 String headerLine = content.substring(0, content.indexOf("\n"));
254 String[] columns = StringUtils.splitByWholeSeparatorPreserveAllTokens(headerLine, delimiter);
255
256 for (int i = 0; i < columns.length; i++) {
257 columns[i] = columns[i].toUpperCase();
258 }
259
260 List<String[]> rows = getRows(content, columns, file);
261
262 CloverETLTable table = new CloverETLTable();
263 table.setName(tablename);
264 table.setColumns(Arrays.asList(columns));
265 table.setRows(rows);
266 return table;
267 }
268
269 @SuppressWarnings("unchecked")
270 protected List<String> readLines(String s) {
271 try {
272 InputStream in = new ByteArrayInputStream(s.getBytes());
273 return IOUtils.readLines(in);
274 } catch (IOException e) {
275 throw new IllegalStateException("Unexpected IO error", e);
276 }
277
278 }
279
280 protected List<String[]> getRows(String content, String[] columns, File file) {
281 List<String> lines = readLines(content);
282 List<String[]> rows = new ArrayList<String[]>();
283 for (int i = 1; i < lines.size(); i++) {
284 String line = lines.get(i);
285 while (!line.endsWith(delimiter)) {
286 i = i + 1;
287 line = line + "\n" + lines.get(i);
288 }
289 String[] row = StringUtils.splitByWholeSeparatorPreserveAllTokens(line, delimiter);
290 if (row.length != columns.length) {
291 throw new IllegalStateException("Column count doesn't match. [" + file.getAbsolutePath() + ",row " + i + "] columns=" + columns.length + " row=" + row.length);
292 }
293 for (int j = 0; j < row.length; j++) {
294 row[j] = escape(row[j]);
295 }
296 rows.add(row);
297 }
298 return rows;
299 }
300
301 protected String escape(String s) {
302 return s.replace("&", "&").replace("<", "<").replace("\"", """).replace("\n", "
").replace("\r", "
");
303 }
304
305 protected String getXml(CloverETLTable table) {
306 StringBuilder sb = new StringBuilder();
307 sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
308 sb.append("<!-- Auto-generated by the Maven Impex Plugin -->\n");
309 sb.append("<!DOCTYPE dataset SYSTEM \"data.dtd\">\n");
310 sb.append("<dataset>\n");
311 List<String[]> rows = table.getRows();
312 List<String> columns = table.getColumns();
313 for (int i = 0; i < rows.size(); i++) {
314 sb.append(" <" + table.getName());
315 String[] row = rows.get(i);
316 for (int j = 0; j < columns.size(); j++) {
317 String column = columns.get(j);
318 String value = row[j];
319 if (!StringUtils.isEmpty(value)) {
320 sb.append(" " + column + "=" + '"' + value + '"');
321 }
322 }
323 sb.append(" />\n");
324 }
325 sb.append("</dataset>\n");
326 return sb.toString();
327 }
328
329 public File getSourceDir() {
330 return sourceDir;
331 }
332
333 public void setSourceDir(File sourceDir) {
334 this.sourceDir = sourceDir;
335 }
336
337 public File getOutputDir() {
338 return outputDir;
339 }
340
341 public void setOutputDir(File outputDir) {
342 this.outputDir = outputDir;
343 }
344
345 public String getDelimiter() {
346 return delimiter;
347 }
348
349 public void setDelimiter(String delimiter) {
350 this.delimiter = delimiter;
351 }
352
353 public String getSchemaFilename() {
354 return schemaFilename;
355 }
356
357 public void setSchemaFilename(String schemaFilename) {
358 this.schemaFilename = schemaFilename;
359 }
360
361 public String getDatabaseDTDFilename() {
362 return databaseDTDFilename;
363 }
364
365 public void setDatabaseDTDFilename(String databaseDTDFilename) {
366 this.databaseDTDFilename = databaseDTDFilename;
367 }
368
369 public String getDataDTDFilename() {
370 return dataDTDFilename;
371 }
372
373 public void setDataDTDFilename(String dataDTDFilename) {
374 this.dataDTDFilename = dataDTDFilename;
375 }
376 }