001 /**
002 * Copyright 2004-2013 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.apache.torque.mojo;
017
018 import java.io.ByteArrayInputStream;
019 import java.io.File;
020 import java.io.IOException;
021 import java.io.InputStream;
022 import java.util.ArrayList;
023 import java.util.Arrays;
024 import java.util.List;
025
026 import org.apache.commons.io.FileUtils;
027 import org.apache.commons.io.IOUtils;
028 import org.apache.commons.lang.StringUtils;
029 import org.apache.maven.plugin.MojoExecutionException;
030 import org.apache.maven.plugin.MojoFailureException;
031 import org.apache.torque.util.CloverETLColumn;
032 import org.apache.torque.util.CloverETLTable;
033
034 /**
035 * This project converts Clover ETL data generated by Kuali's Ant based tooling into a format that is consumable by the Maven Impex plugin
036 *
037 * @goal convertcloveretl
038 */
039 public class ConvertCloverETLMojo extends BaseMojo {
040
041 /**
042 * The source directory where Clover ETL data was generated
043 *
044 * @parameter expression="${impex.sourceDirectory}" default-value="${project.basedir}/src/main/resources"
045 * @required
046 */
047 File sourceDir;
048
049 /**
050 * The output directory where maven-impex-plugin data will get generated
051 *
052 * @parameter expression="${impex.outputDir}" default-value="${project.build.directory}/impex"
053 * @required
054 */
055 File outputDir;
056
057 /**
058 * The delimiter used by the Clover ETL data
059 *
060 * @parameter expression="${impex.delimiter}" default-value="|"
061 * @required
062 */
063 String delimiter;
064
065 /**
066 * The schema.xml file for Clover ETL
067 *
068 * @parameter expression="${impex.schemaFilename}" default-value="schema.xml"
069 * @required
070 */
071 String schemaFilename;
072
073 /**
074 * The name of the file containing the DTD for database schemas
075 *
076 * @parameter expression="${impex.databaseDTDFilename}" default-value="database.dtd"
077 * @required
078 */
079 String databaseDTDFilename;
080
081 /**
082 * The name of the file containing the DTD for this database schema
083 *
084 * @parameter expression="${impex.dataDTDFilename}" default-value="data.dtd"
085 * @required
086 */
087 String dataDTDFilename;
088
089 @Override
090 protected void executeMojo() throws MojoExecutionException, MojoFailureException {
091 getLog().info("Examining " + sourceDir.getAbsolutePath());
092 handleSchema();
093 handleData();
094 }
095
096 protected void handleSchema() {
097 try {
098 handleDataDTD();
099 File newSchemaFile = new File(outputDir + "/" + schemaFilename);
100 File oldSchemaFile = new File(sourceDir + "/" + schemaFilename);
101 getLog().info("Creating " + newSchemaFile.getCanonicalPath());
102 FileUtils.copyFile(oldSchemaFile, newSchemaFile);
103 File newDatabaseDTDFile = new File(outputDir + "/" + databaseDTDFilename);
104 File oldDatabaseDTDFile = new File(sourceDir + "/" + databaseDTDFilename);
105 getLog().info("Creating " + newDatabaseDTDFile.getCanonicalPath());
106 FileUtils.copyFile(oldDatabaseDTDFile, newDatabaseDTDFile);
107 } catch (IOException e) {
108 throw new IllegalStateException("Unexpected IO error", e);
109 }
110 }
111
112 protected String[] parseAll(String s, String open, String close) {
113 String[] tokens = StringUtils.substringsBetween(s, open, close);
114 if (tokens == null) {
115 return null;
116 }
117 for (int i = 0; i < tokens.length; i++) {
118 tokens[i] = open + tokens[i] + close;
119 }
120 return tokens;
121
122 }
123
124 protected void handleDataDTD() throws IOException {
125 File schemaFile = new File(sourceDir + "/" + schemaFilename);
126 String contents = FileUtils.readFileToString(schemaFile);
127 String[] tables = getTables(contents);
128 getLog().info("Located " + tables.length + " schema tables");
129 List<CloverETLTable> realTables = new ArrayList<CloverETLTable>();
130 for (String table : tables) {
131 CloverETLTable realTable = getDataDTDTable(table);
132 realTables.add(realTable);
133 }
134 String content = getDataDTDContent(realTables);
135 File dataDTDFile = new File(outputDir + "/" + dataDTDFilename);
136 getLog().info("Creating " + dataDTDFile.getCanonicalPath());
137 FileUtils.writeStringToFile(dataDTDFile, content);
138 }
139
140 protected String getDataDTDContent(List<CloverETLTable> tables) {
141 StringBuilder sb = new StringBuilder();
142 sb.append(getProlog(tables));
143 for (CloverETLTable table : tables) {
144 sb.append(getTableDTDContent(table));
145 }
146 return sb.toString();
147 }
148
149 protected String getTableDTDContent(CloverETLTable table) {
150 StringBuilder sb = new StringBuilder();
151 sb.append("<!ELEMENT " + table.getName() + " EMPTY>\n");
152 sb.append("<!ATTLIST " + table.getName() + "\n");
153 sb.append(getColumnDTDContent(table.getEtlColumns()));
154 sb.append(">\n");
155 return sb.toString();
156 }
157
158 protected String getColumnDTDContent(List<CloverETLColumn> columns) {
159 StringBuilder sb = new StringBuilder();
160 for (CloverETLColumn column : columns) {
161 sb.append(" ");
162 sb.append(column.getName());
163 sb.append(" ");
164 sb.append("CDATA");
165 sb.append(" ");
166 sb.append(column.isRequired() ? "#IMPLIED" : "#REQUIRED");
167 sb.append("\n");
168 }
169 return sb.toString();
170 }
171
172 protected String getProlog(List<CloverETLTable> tables) {
173 StringBuilder sb = new StringBuilder();
174 sb.append("<!ELEMENT dataset (\n");
175 for (int i = 0; i < tables.size(); i++) {
176 CloverETLTable table = tables.get(i);
177 sb.append(" " + table.getName());
178 if (i < tables.size() - 1) {
179 sb.append("|\n");
180 } else {
181 sb.append(")*>\n");
182 }
183 }
184 return sb.toString();
185 }
186
187 protected CloverETLTable getDataDTDTable(String s) {
188 String tablename = StringUtils.substringBetween(s, "<table name=\"", "\"");
189 String[] columns = parseAll(s, "<column ", "/>");
190 List<CloverETLColumn> realColumns = new ArrayList<CloverETLColumn>();
191 for (String column : columns) {
192 realColumns.add(getCloverETLColumn(column));
193 }
194
195 CloverETLTable table = new CloverETLTable();
196 table.setName(tablename);
197 table.setEtlColumns(realColumns);
198 return table;
199 }
200
201 protected CloverETLColumn getCloverETLColumn(String s) {
202 String columnName = StringUtils.substringBetween(s, "name=\"", "\"");
203 boolean required = s.contains("required=\"true\"");
204 CloverETLColumn cec = new CloverETLColumn();
205 cec.setName(columnName);
206 cec.setRequired(required);
207 return cec;
208 }
209
210 protected String[] getTables(String contents) {
211 String begin = "<table name=\"";
212 String close = "</table>";
213 return parseAll(contents, begin, close);
214 }
215
216 protected String parse(String s, String begin, String close) {
217 String between = StringUtils.substringBetween(s, begin, close);
218 return begin + between + close;
219 }
220
221 protected void handleData() {
222 File dataDir = new File(sourceDir + "/data");
223 File[] files = dataDir.listFiles();
224 if (files == null) {
225 getLog().info("Converting 0 Clover ETL data files");
226 } else {
227 getLog().info("Converting " + files.length + " Clover ETL data files");
228 Arrays.sort(files);
229 for (File file : files) {
230 convertFile(file);
231 }
232 }
233 }
234
235 protected void convertFile(File file) {
236 try {
237 CloverETLTable table = getTable(file);
238 String xml = getXml(table);
239 File outputFile = new File(outputDir + "/" + table.getName() + ".xml");
240 getLog().info("Creating " + outputFile.getCanonicalPath());
241 FileUtils.writeStringToFile(outputFile, xml);
242 } catch (IOException e) {
243 throw new IllegalStateException("Unexpected IO error", e);
244 }
245 }
246
247 protected CloverETLTable getTable(File file) throws IOException {
248 String filename = file.getName().toUpperCase();
249 int pos = filename.indexOf(".");
250 String tablename = filename.substring(0, pos);
251
252 String content = FileUtils.readFileToString(file);
253 String headerLine = content.substring(0, content.indexOf("\n"));
254 String[] columns = StringUtils.splitByWholeSeparatorPreserveAllTokens(headerLine, delimiter);
255
256 for (int i = 0; i < columns.length; i++) {
257 columns[i] = columns[i].toUpperCase();
258 }
259
260 List<String[]> rows = getRows(content, columns, file);
261
262 CloverETLTable table = new CloverETLTable();
263 table.setName(tablename);
264 table.setColumns(Arrays.asList(columns));
265 table.setRows(rows);
266 return table;
267 }
268
269 @SuppressWarnings("unchecked")
270 protected List<String> readLines(String s) {
271 try {
272 InputStream in = new ByteArrayInputStream(s.getBytes());
273 return IOUtils.readLines(in);
274 } catch (IOException e) {
275 throw new IllegalStateException("Unexpected IO error", e);
276 }
277
278 }
279
280 protected List<String[]> getRows(String content, String[] columns, File file) {
281 List<String> lines = readLines(content);
282 List<String[]> rows = new ArrayList<String[]>();
283 for (int i = 1; i < lines.size(); i++) {
284 String line = lines.get(i);
285 while (!line.endsWith(delimiter)) {
286 i = i + 1;
287 line = line + "\n" + lines.get(i);
288 }
289 String[] row = StringUtils.splitByWholeSeparatorPreserveAllTokens(line, delimiter);
290 if (row.length != columns.length) {
291 throw new IllegalStateException("Column count doesn't match. [" + file.getAbsolutePath() + ",row " + i + "] columns=" + columns.length + " row=" + row.length);
292 }
293 for (int j = 0; j < row.length; j++) {
294 row[j] = escape(row[j]);
295 }
296 rows.add(row);
297 }
298 return rows;
299 }
300
301 protected String escape(String s) {
302 return s.replace("&", "&").replace("<", "<").replace("\"", """).replace("\n", "
").replace("\r", "
");
303 }
304
305 protected String getXml(CloverETLTable table) {
306 StringBuilder sb = new StringBuilder();
307 sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
308 sb.append("<!-- Auto-generated by the Maven Impex Plugin -->\n");
309 sb.append("<!DOCTYPE dataset SYSTEM \"data.dtd\">\n");
310 sb.append("<dataset>\n");
311 List<String[]> rows = table.getRows();
312 List<String> columns = table.getColumns();
313 for (int i = 0; i < rows.size(); i++) {
314 sb.append(" <" + table.getName());
315 String[] row = rows.get(i);
316 for (int j = 0; j < columns.size(); j++) {
317 String column = columns.get(j);
318 String value = row[j];
319 if (!StringUtils.isEmpty(value)) {
320 sb.append(" " + column + "=" + '"' + value + '"');
321 }
322 }
323 sb.append(" />\n");
324 }
325 sb.append("</dataset>\n");
326 return sb.toString();
327 }
328
329 public File getSourceDir() {
330 return sourceDir;
331 }
332
333 public void setSourceDir(File sourceDir) {
334 this.sourceDir = sourceDir;
335 }
336
337 public File getOutputDir() {
338 return outputDir;
339 }
340
341 public void setOutputDir(File outputDir) {
342 this.outputDir = outputDir;
343 }
344
345 public String getDelimiter() {
346 return delimiter;
347 }
348
349 public void setDelimiter(String delimiter) {
350 this.delimiter = delimiter;
351 }
352
353 public String getSchemaFilename() {
354 return schemaFilename;
355 }
356
357 public void setSchemaFilename(String schemaFilename) {
358 this.schemaFilename = schemaFilename;
359 }
360
361 public String getDatabaseDTDFilename() {
362 return databaseDTDFilename;
363 }
364
365 public void setDatabaseDTDFilename(String databaseDTDFilename) {
366 this.databaseDTDFilename = databaseDTDFilename;
367 }
368
369 public String getDataDTDFilename() {
370 return dataDTDFilename;
371 }
372
373 public void setDataDTDFilename(String dataDTDFilename) {
374 this.dataDTDFilename = dataDTDFilename;
375 }
376 }