|  1 |     | 
     | 
  |  2 |     | 
     | 
  |  3 |     | 
     | 
  |  4 |     | 
     | 
  |  5 |     | 
     | 
  |  6 |     | 
     | 
  |  7 |     | 
     | 
  |  8 |     | 
     | 
  |  9 |     | 
     | 
  |  10 |     | 
     | 
  |  11 |     | 
     | 
  |  12 |     | 
     | 
  |  13 |     | 
     | 
  |  14 |     | 
     | 
  |  15 |     | 
     | 
  |  16 |     | 
   package org.kuali.rice.kew.batch;  | 
  |  17 |     | 
     | 
  |  18 |     | 
   import java.io.BufferedReader;  | 
  |  19 |     | 
   import java.io.File;  | 
  |  20 |     | 
   import java.io.FileReader;  | 
  |  21 |     | 
   import java.io.FileWriter;  | 
  |  22 |     | 
   import java.io.IOException;  | 
  |  23 |     | 
   import java.text.Format;  | 
  |  24 |     | 
   import java.text.SimpleDateFormat;  | 
  |  25 |     | 
   import java.util.ArrayList;  | 
  |  26 |     | 
   import java.util.Calendar;  | 
  |  27 |     | 
   import java.util.Collection;  | 
  |  28 |     | 
   import java.util.Date;  | 
  |  29 |     | 
   import java.util.Iterator;  | 
  |  30 |     | 
   import java.util.List;  | 
  |  31 |     | 
     | 
  |  32 |     | 
   import org.kuali.rice.core.api.CoreApiServiceLocator;  | 
  |  33 |     | 
   import org.kuali.rice.core.api.impex.xml.DirectoryXmlDocCollection;  | 
  |  34 |     | 
   import org.kuali.rice.core.api.impex.xml.FileXmlDocCollection;  | 
  |  35 |     | 
   import org.kuali.rice.core.api.impex.xml.XmlDocCollection;  | 
  |  36 |     | 
   import org.kuali.rice.core.api.impex.xml.XmlIngesterService;  | 
  |  37 |     | 
   import org.kuali.rice.core.api.impex.xml.ZipXmlDocCollection;  | 
  |  38 |     | 
     | 
  |  39 |     | 
     | 
  |  40 |     | 
     | 
  |  41 |     | 
     | 
  |  42 |     | 
     | 
  |  43 |     | 
     | 
  |  44 |     | 
     | 
  |  45 |     | 
     | 
  |  46 |     | 
     | 
  |  47 |     | 
     | 
  |  48 |     | 
     | 
  |  49 |     | 
     | 
  |  50 |     | 
     | 
  |  51 |     | 
     | 
  |  52 |     | 
     | 
  |  53 |     | 
     | 
  |  54 |     | 
     | 
  |  55 |     | 
     | 
  |  56 |     | 
     | 
  |  57 |     | 
     | 
  |  58 |    0 |    public class XmlPollerServiceImpl implements XmlPollerService { | 
  |  59 |     | 
     | 
  |  60 |    0 |        private static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger  | 
  |  61 |     | 
               .getLogger(XmlPollerServiceImpl.class);  | 
  |  62 |    0 |        private static final Format DIR_FORMAT = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS"); | 
  |  63 |     | 
     | 
  |  64 |     | 
         | 
  |  65 |     | 
     | 
  |  66 |     | 
     | 
  |  67 |    0 |        private int pollIntervalSecs = 5 * 60;   | 
  |  68 |     | 
         | 
  |  69 |     | 
     | 
  |  70 |     | 
     | 
  |  71 |    0 |        private int initialDelaySecs = 30;   | 
  |  72 |     | 
         | 
  |  73 |     | 
     | 
  |  74 |     | 
     | 
  |  75 |     | 
       private String xmlPendingLocation;  | 
  |  76 |     | 
         | 
  |  77 |     | 
     | 
  |  78 |     | 
     | 
  |  79 |     | 
       private String xmlCompletedLocation;  | 
  |  80 |     | 
         | 
  |  81 |     | 
     | 
  |  82 |     | 
     | 
  |  83 |     | 
       private String xmlProblemLocation;  | 
  |  84 |     | 
         | 
  |  85 |     | 
       private String xmlParentDirectory;  | 
  |  86 |     | 
       private static final String PENDING_MOVE_FAILED_ARCHIVE_FILE = "movesfailed";  | 
  |  87 |     | 
       private static final String NEW_LINE = "\n";  | 
  |  88 |     | 
     | 
  |  89 |     | 
       public void run() { | 
  |  90 |     | 
             | 
  |  91 |     | 
             | 
  |  92 |     | 
             | 
  |  93 |     | 
             | 
  |  94 |    0 |            LOG.debug("checking for xml data files..."); | 
  |  95 |    0 |            File[] files = getXmlPendingDir().listFiles();  | 
  |  96 |    0 |            if (files == null || files.length == 0) { | 
  |  97 |    0 |                    return;  | 
  |  98 |     | 
           }  | 
  |  99 |    0 |            LOG.info("Found " + files.length + " files to ingest."); | 
  |  100 |    0 |            List<XmlDocCollection> collections = new ArrayList<XmlDocCollection>();  | 
  |  101 |    0 |            for (File file : files)  | 
  |  102 |     | 
           { | 
  |  103 |    0 |                if (file.isDirectory())  | 
  |  104 |     | 
               { | 
  |  105 |    0 |                    collections.add(new DirectoryXmlDocCollection(file));  | 
  |  106 |    0 |                } else if (file.getName().equals(PENDING_MOVE_FAILED_ARCHIVE_FILE))  | 
  |  107 |     | 
               { | 
  |  108 |     | 
                     | 
  |  109 |    0 |                    continue;  | 
  |  110 |    0 |                } else if (file.getName().toLowerCase().endsWith(".zip")) | 
  |  111 |     | 
               { | 
  |  112 |     | 
                   try  | 
  |  113 |     | 
                   { | 
  |  114 |    0 |                        collections.add(new ZipXmlDocCollection(file));  | 
  |  115 |    0 |                    } catch (IOException ioe)  | 
  |  116 |     | 
                   { | 
  |  117 |    0 |                        LOG.error("Unable to load file: " + file); | 
  |  118 |    0 |                    }  | 
  |  119 |    0 |                } else if (file.getName().endsWith(".xml")) | 
  |  120 |     | 
               { | 
  |  121 |    0 |                    collections.add(new FileXmlDocCollection(file));  | 
  |  122 |     | 
               } else  | 
  |  123 |     | 
               { | 
  |  124 |    0 |                    LOG.warn("Ignoring extraneous file in xml pending directory: " + file); | 
  |  125 |     | 
               }  | 
  |  126 |     | 
           }  | 
  |  127 |     | 
     | 
  |  128 |     | 
             | 
  |  129 |    0 |            Iterator collectionsIt = collections.iterator();  | 
  |  130 |    0 |            Collection<XmlDocCollection> culled = new ArrayList<XmlDocCollection>();  | 
  |  131 |    0 |            while (collectionsIt.hasNext()) { | 
  |  132 |    0 |                XmlDocCollection container = (XmlDocCollection) collectionsIt.next();  | 
  |  133 |     | 
                 | 
  |  134 |    0 |                if (inPendingMoveFailedArchive(container.getFile())) { | 
  |  135 |    0 |                    LOG.info("Ignoring previously processed resource: " + container); | 
  |  136 |    0 |                    culled.add(container);  | 
  |  137 |     | 
               }  | 
  |  138 |    0 |            }  | 
  |  139 |    0 |            collections.removeAll(culled);  | 
  |  140 |     | 
     | 
  |  141 |    0 |            if (collections.size() == 0) { | 
  |  142 |    0 |                LOG.debug("No valid new resources found to ingest"); | 
  |  143 |    0 |                return;  | 
  |  144 |     | 
           }  | 
  |  145 |     | 
     | 
  |  146 |    0 |            Date LOAD_TIME = Calendar.getInstance().getTime();  | 
  |  147 |     | 
             | 
  |  148 |    0 |            File completeDir = new File(getXmlCompleteDir(), DIR_FORMAT.format(LOAD_TIME));  | 
  |  149 |    0 |            File failedDir = new File(getXmlProblemDir(), DIR_FORMAT.format(LOAD_TIME));  | 
  |  150 |     | 
     | 
  |  151 |     | 
             | 
  |  152 |    0 |            Collection failed = null;  | 
  |  153 |     | 
           try { | 
  |  154 |    0 |                failed = CoreApiServiceLocator.getXmlIngesterService().ingest(collections);  | 
  |  155 |    0 |            } catch (Exception e) { | 
  |  156 |    0 |                LOG.error("Error ingesting data", e); | 
  |  157 |     | 
                 | 
  |  158 |    0 |            }  | 
  |  159 |     | 
         | 
  |  160 |     | 
             | 
  |  161 |    0 |            LOG.info("Moving files..."); | 
  |  162 |    0 |            collectionsIt = collections.iterator();  | 
  |  163 |    0 |            while (collectionsIt.hasNext()) { | 
  |  164 |    0 |                XmlDocCollection container = (XmlDocCollection) collectionsIt.next();  | 
  |  165 |    0 |                LOG.debug("container: " + container); | 
  |  166 |     | 
               try { | 
  |  167 |     | 
                     | 
  |  168 |     | 
                     | 
  |  169 |    0 |                    container.close();  | 
  |  170 |    0 |                } catch (IOException ioe) { | 
  |  171 |    0 |                    LOG.warn("Error closing " + container, ioe); | 
  |  172 |    0 |                }  | 
  |  173 |    0 |                if (failed.contains(container)) { | 
  |  174 |     | 
                     | 
  |  175 |     | 
                     | 
  |  176 |    0 |                    if (container.getFile() != null) { | 
  |  177 |    0 |                        LOG.error("Moving " + container.getFile() + " to problem dir."); | 
  |  178 |    0 |                        if ((!failedDir.isDirectory() && !failedDir.mkdirs())  | 
  |  179 |     | 
                           || !moveFile(failedDir, container.getFile())) { | 
  |  180 |    0 |                            LOG.error("Could not move: " + container.getFile()); | 
  |  181 |    0 |                            recordUnmovablePendingFile(container.getFile(), LOAD_TIME);           | 
  |  182 |     | 
                       }  | 
  |  183 |     | 
                   }  | 
  |  184 |     | 
               } else { | 
  |  185 |    0 |                    if (container.getFile() != null) { | 
  |  186 |    0 |                        LOG.info("Moving " + container.getFile() + " to loaded dir."); | 
  |  187 |    0 |                        if((!completeDir.isDirectory() && !completeDir.mkdirs())  | 
  |  188 |     | 
                           || !moveFile(completeDir, container.getFile())){ | 
  |  189 |    0 |                            LOG.error("Could not move: " + container.getFile()); | 
  |  190 |    0 |                            recordUnmovablePendingFile(container.getFile(), LOAD_TIME);           | 
  |  191 |     | 
                       }  | 
  |  192 |     | 
                   }  | 
  |  193 |     | 
               }  | 
  |  194 |    0 |            }  | 
  |  195 |    0 |        }  | 
  |  196 |     | 
     | 
  |  197 |     | 
       private boolean inPendingMoveFailedArchive(File xmlDataFile){ | 
  |  198 |    0 |            if (xmlDataFile == null) return false;  | 
  |  199 |    0 |            BufferedReader inFile = null;  | 
  |  200 |    0 |            File movesFailedFile = new File(getXmlPendingDir(), PENDING_MOVE_FAILED_ARCHIVE_FILE);  | 
  |  201 |    0 |            if (!movesFailedFile.isFile()) return false;  | 
  |  202 |     | 
           try { | 
  |  203 |    0 |                inFile = new BufferedReader(new FileReader(movesFailedFile));  | 
  |  204 |     | 
               String line;  | 
  |  205 |     | 
                 | 
  |  206 |    0 |                while((line = inFile.readLine()) != null){ | 
  |  207 |    0 |                    String trimmedLine = line.trim();  | 
  |  208 |    0 |                    if(trimmedLine.equals(xmlDataFile.getName()) ||  | 
  |  209 |     | 
                      trimmedLine.startsWith(xmlDataFile.getName() + "=")) {  | 
  |  210 |    0 |                        return true;  | 
  |  211 |     | 
                   }  | 
  |  212 |    0 |                }  | 
  |  213 |    0 |            } catch (IOException e){ | 
  |  214 |    0 |                LOG.warn("Error reading file " + movesFailedFile); | 
  |  215 |     | 
                 | 
  |  216 |     | 
           } finally { | 
  |  217 |    0 |                if (inFile != null) try { | 
  |  218 |    0 |                    inFile.close();  | 
  |  219 |    0 |                } catch (Exception e) { | 
  |  220 |    0 |                    LOG.warn("Error closing buffered reader for " + movesFailedFile); | 
  |  221 |    0 |                }  | 
  |  222 |     | 
           }  | 
  |  223 |     | 
           | 
  |  224 |    0 |            return false;  | 
  |  225 |     | 
       }  | 
  |  226 |     | 
     | 
  |  227 |     | 
       private boolean recordUnmovablePendingFile(File unMovablePendingFile, Date dateLoaded){ | 
  |  228 |    0 |            boolean recorded = false;  | 
  |  229 |    0 |            FileWriter archiveFile = null;  | 
  |  230 |     | 
           try{ | 
  |  231 |    0 |                archiveFile = new FileWriter(new File(getXmlPendingDir(), PENDING_MOVE_FAILED_ARCHIVE_FILE), true);    | 
  |  232 |    0 |                archiveFile.write(unMovablePendingFile.getName() + "=" + dateLoaded.getTime() + NEW_LINE);  | 
  |  233 |    0 |                recorded = true;  | 
  |  234 |    0 |            } catch (IOException e){ | 
  |  235 |    0 |                LOG.error("Unable to record unmovable pending file " + unMovablePendingFile.getName() + "in the archive file " + PENDING_MOVE_FAILED_ARCHIVE_FILE); | 
  |  236 |     | 
           } finally { | 
  |  237 |    0 |                if (archiveFile != null) { | 
  |  238 |     | 
                   try { | 
  |  239 |    0 |                        archiveFile.close();  | 
  |  240 |    0 |                    } catch (IOException ioe) { | 
  |  241 |    0 |                        LOG.error("Error closing unmovable pending file", ioe); | 
  |  242 |    0 |                    }  | 
  |  243 |     | 
               }  | 
  |  244 |     | 
           }  | 
  |  245 |    0 |            return recorded;         | 
  |  246 |     | 
       }  | 
  |  247 |     | 
     | 
  |  248 |     | 
       private boolean moveFile(File toDirectory, File fileToMove){ | 
  |  249 |    0 |            boolean moved = true;  | 
  |  250 |    0 |            if (!fileToMove.renameTo(new File(toDirectory.getPath(), fileToMove.getName()))){ | 
  |  251 |    0 |                LOG.error("Unable to move file " + fileToMove.getName() + " to directory " + toDirectory.getPath()); | 
  |  252 |    0 |                moved = false;  | 
  |  253 |     | 
           }  | 
  |  254 |    0 |            return moved;  | 
  |  255 |     | 
       }  | 
  |  256 |     | 
     | 
  |  257 |     | 
       private File getXmlPendingDir() { | 
  |  258 |    0 |            return new File(getXmlPendingLocation());  | 
  |  259 |     | 
       }  | 
  |  260 |     | 
     | 
  |  261 |     | 
       private File getXmlCompleteDir() { | 
  |  262 |    0 |            return new File(getXmlCompletedLocation());  | 
  |  263 |     | 
       }  | 
  |  264 |     | 
     | 
  |  265 |     | 
       private File getXmlProblemDir() { | 
  |  266 |    0 |            return new File(getXmlProblemLocation());  | 
  |  267 |     | 
       }  | 
  |  268 |     | 
     | 
  |  269 |     | 
       public String getXmlCompletedLocation() { | 
  |  270 |    0 |            return xmlCompletedLocation;  | 
  |  271 |     | 
       }  | 
  |  272 |     | 
     | 
  |  273 |     | 
       public void setXmlCompletedLocation(String xmlCompletedLocation) { | 
  |  274 |    0 |            this.xmlCompletedLocation = xmlCompletedLocation;  | 
  |  275 |    0 |        }  | 
  |  276 |     | 
     | 
  |  277 |     | 
       public String getXmlPendingLocation() { | 
  |  278 |    0 |            return xmlPendingLocation;  | 
  |  279 |     | 
       }  | 
  |  280 |     | 
     | 
  |  281 |     | 
         | 
  |  282 |     | 
     | 
  |  283 |     | 
     | 
  |  284 |     | 
     | 
  |  285 |     | 
     | 
  |  286 |     | 
       public void setXmlPendingLocation(String xmlPendingLocation) { | 
  |  287 |    0 |            this.xmlPendingLocation = xmlPendingLocation;  | 
  |  288 |    0 |        }  | 
  |  289 |     | 
     | 
  |  290 |     | 
       public String getXmlProblemLocation() { | 
  |  291 |    0 |            return xmlProblemLocation;  | 
  |  292 |     | 
       }  | 
  |  293 |     | 
     | 
  |  294 |     | 
       public void setXmlProblemLocation(String xmlProblemLocation) { | 
  |  295 |    0 |            this.xmlProblemLocation = xmlProblemLocation;  | 
  |  296 |    0 |        }  | 
  |  297 |     | 
       public String getXmlParentDirectory() { | 
  |  298 |    0 |            return xmlParentDirectory;  | 
  |  299 |     | 
       }  | 
  |  300 |     | 
       public void setXmlParentDirectory(String xmlDataParentDirectory) { | 
  |  301 |    0 |            this.xmlParentDirectory = xmlDataParentDirectory;  | 
  |  302 |    0 |        }  | 
  |  303 |     | 
     | 
  |  304 |     | 
         | 
  |  305 |     | 
     | 
  |  306 |     | 
     | 
  |  307 |     | 
     | 
  |  308 |     | 
       public void setPollIntervalSecs(int seconds) { | 
  |  309 |    0 |            this.pollIntervalSecs = seconds;  | 
  |  310 |    0 |        }  | 
  |  311 |     | 
     | 
  |  312 |     | 
         | 
  |  313 |     | 
     | 
  |  314 |     | 
     | 
  |  315 |     | 
     | 
  |  316 |     | 
       public int getPollIntervalSecs() { | 
  |  317 |    0 |            return this.pollIntervalSecs;  | 
  |  318 |     | 
       }  | 
  |  319 |     | 
     | 
  |  320 |     | 
         | 
  |  321 |     | 
     | 
  |  322 |     | 
     | 
  |  323 |     | 
     | 
  |  324 |     | 
       public void setInitialDelaySecs(int seconds) { | 
  |  325 |    0 |            this.initialDelaySecs = seconds;  | 
  |  326 |    0 |        }  | 
  |  327 |     | 
     | 
  |  328 |     | 
         | 
  |  329 |     | 
     | 
  |  330 |     | 
     | 
  |  331 |     | 
     | 
  |  332 |     | 
       public int getInitialDelaySecs() { | 
  |  333 |    0 |            return this.initialDelaySecs;  | 
  |  334 |     | 
       }  | 
  |  335 |     | 
   }  |