Coverage Report - org.kuali.rice.kew.batch.XmlPollerServiceImpl
 
Classes in this File Line Coverage Branch Coverage Complexity
XmlPollerServiceImpl
0%
0/123
0%
0/56
3.105
 
 1  
 /**
 2  
  * Copyright 2005-2011 The Kuali Foundation
 3  
  *
 4  
  * Licensed under the Educational Community License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  * http://www.opensource.org/licenses/ecl2.php
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  */
 16  
 package org.kuali.rice.kew.batch;
 17  
 
 18  
 import java.io.BufferedReader;
 19  
 import java.io.File;
 20  
 import java.io.FileReader;
 21  
 import java.io.FileWriter;
 22  
 import java.io.IOException;
 23  
 import java.text.Format;
 24  
 import java.text.SimpleDateFormat;
 25  
 import java.util.ArrayList;
 26  
 import java.util.Calendar;
 27  
 import java.util.Collection;
 28  
 import java.util.Date;
 29  
 import java.util.Iterator;
 30  
 import java.util.List;
 31  
 
 32  
 import org.kuali.rice.core.api.CoreApiServiceLocator;
 33  
 import org.kuali.rice.core.api.impex.xml.DirectoryXmlDocCollection;
 34  
 import org.kuali.rice.core.api.impex.xml.FileXmlDocCollection;
 35  
 import org.kuali.rice.core.api.impex.xml.XmlDocCollection;
 36  
 import org.kuali.rice.core.api.impex.xml.XmlIngesterService;
 37  
 import org.kuali.rice.core.api.impex.xml.ZipXmlDocCollection;
 38  
 
 39  
 
 40  
 /**
 41  
  * Utility class responsible for polling and ingesting XML data files
 42  
  * containing various forms of workflow engine data (e.g. document types
 43  
  * and rules).
 44  
  * Loaded files and problem files are placed into a subdirectory of a
 45  
  * configured 'loaded' and 'problem' directory, respectively.
 46  
  * "Problem-ness" is determined by inspecting a 'processed' flag on each <code>XmlDoc</code>
 47  
  * in each collection.  If not all <code>XmlDoc</code>s are marked 'processed' an
 48  
  * error is assumed, and the collection file (e.g. for a Zip, the Zip file) is moved
 49  
  * to the 'problem' directory.
 50  
  * As such, it is the <b><code>XmlIngesterService</code>'s responsibility</b> to mark
 51  
  * any unknown or otherwise innocuous non-failure non-processed files, as 'processed'.
 52  
  * A different mechanism can be developed if this proves to be a problem, but for now
 53  
  * it is simple enough for the <code>XmlIngesterService</code> to determine this.
 54  
  * @see org.kuali.rice.kew.batch.XmlPollerService
 55  
  * @see org.kuali.rice.kew.batch.XmlIngesterServiceImpl
 56  
  * @author Kuali Rice Team (rice.collab@kuali.org)
 57  
  */
 58  0
 public class XmlPollerServiceImpl implements XmlPollerService {
 59  
 
 60  0
     private static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger
 61  
             .getLogger(XmlPollerServiceImpl.class);
 62  0
     private static final Format DIR_FORMAT = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS");
 63  
 
 64  
     /**
 65  
      * Specifies the polling interval that should be used with this task.
 66  
      */
 67  0
     private int pollIntervalSecs = 5 * 60; // default to 5 minutes
 68  
     /**
 69  
      * Specifies the initial delay the poller should wait before starting to poll
 70  
      */
 71  0
     private int initialDelaySecs = 30; // default to 30 seconds
 72  
     /**
 73  
      * Location in which to find XML files to load.
 74  
      */
 75  
     private String xmlPendingLocation;
 76  
     /**
 77  
      * Location in which to place successfully loaded XML files.
 78  
      */
 79  
     private String xmlCompletedLocation;
 80  
     /**
 81  
      * Location in which to place XML files which have failed to load.
 82  
      */
 83  
     private String xmlProblemLocation;
 84  
     
 85  
     private String xmlParentDirectory;
 86  
     private static final String PENDING_MOVE_FAILED_ARCHIVE_FILE = "movesfailed";
 87  
     private static final String NEW_LINE = "\n";
 88  
 
 89  
     public void run() {
 90  
         // if(!directoriesWritable()){
 91  
         //     LOG.error("Error writing to xml data directories. Stopping xmlLoader ...");
 92  
         //     this.cancel();
 93  
         // }
 94  0
         LOG.debug("checking for xml data files...");
 95  0
         File[] files = getXmlPendingDir().listFiles();
 96  0
         if (files == null || files.length == 0) {
 97  0
                 return;
 98  
         }
 99  0
         LOG.info("Found " + files.length + " files to ingest.");
 100  0
         List<XmlDocCollection> collections = new ArrayList<XmlDocCollection>();
 101  0
         for (File file : files)
 102  
         {
 103  0
             if (file.isDirectory())
 104  
             {
 105  0
                 collections.add(new DirectoryXmlDocCollection(file));
 106  0
             } else if (file.getName().equals(PENDING_MOVE_FAILED_ARCHIVE_FILE))
 107  
             {
 108  
                 // the movesfailed file...ignore this
 109  0
                 continue;
 110  0
             } else if (file.getName().toLowerCase().endsWith(".zip"))
 111  
             {
 112  
                 try
 113  
                 {
 114  0
                     collections.add(new ZipXmlDocCollection(file));
 115  0
                 } catch (IOException ioe)
 116  
                 {
 117  0
                     LOG.error("Unable to load file: " + file);
 118  0
                 }
 119  0
             } else if (file.getName().endsWith(".xml"))
 120  
             {
 121  0
                 collections.add(new FileXmlDocCollection(file));
 122  
             } else
 123  
             {
 124  0
                 LOG.warn("Ignoring extraneous file in xml pending directory: " + file);
 125  
             }
 126  
         }
 127  
 
 128  
         // Cull any resources which were already processed and whose moves failed
 129  0
         Iterator collectionsIt = collections.iterator();
 130  0
         Collection<XmlDocCollection> culled = new ArrayList<XmlDocCollection>();
 131  0
         while (collectionsIt.hasNext()) {
 132  0
             XmlDocCollection container = (XmlDocCollection) collectionsIt.next();
 133  
             // if a move has already failed for this archive, ignore it
 134  0
             if (inPendingMoveFailedArchive(container.getFile())) {
 135  0
                 LOG.info("Ignoring previously processed resource: " + container);
 136  0
                 culled.add(container);
 137  
             }
 138  0
         }
 139  0
         collections.removeAll(culled);
 140  
 
 141  0
         if (collections.size() == 0) {
 142  0
             LOG.debug("No valid new resources found to ingest");
 143  0
             return;
 144  
         }
 145  
 
 146  0
         Date LOAD_TIME = Calendar.getInstance().getTime();
 147  
         // synchronization around date format should not be an issue as this code is single-threaded
 148  0
         File completeDir = new File(getXmlCompleteDir(), DIR_FORMAT.format(LOAD_TIME));
 149  0
         File failedDir = new File(getXmlProblemDir(), DIR_FORMAT.format(LOAD_TIME));
 150  
 
 151  
         // now ingest the containers
 152  0
         Collection failed = null;
 153  
         try {
 154  0
             failed = CoreApiServiceLocator.getXmlIngesterService().ingest(collections);
 155  0
         } catch (Exception e) {
 156  0
             LOG.error("Error ingesting data", e);
 157  
             //throw new RuntimeException(e);
 158  0
         }
 159  
     
 160  
         // now iterate through all containers again, and move containers to approprate dir
 161  0
         LOG.info("Moving files...");
 162  0
         collectionsIt = collections.iterator();
 163  0
         while (collectionsIt.hasNext()) {
 164  0
             XmlDocCollection container = (XmlDocCollection) collectionsIt.next();
 165  0
             LOG.debug("container: " + container);
 166  
             try {
 167  
                 // "close" the container
 168  
                 // this only matters for ZipFiles for now
 169  0
                 container.close();
 170  0
             } catch (IOException ioe) {
 171  0
                 LOG.warn("Error closing " + container, ioe);
 172  0
             }
 173  0
             if (failed.contains(container)) {
 174  
                 // some docs must have failed, move the whole
 175  
                 // container to the failed dir
 176  0
                 if (container.getFile() != null) {
 177  0
                     LOG.error("Moving " + container.getFile() + " to problem dir.");
 178  0
                     if ((!failedDir.isDirectory() && !failedDir.mkdirs())
 179  
                         || !moveFile(failedDir, container.getFile())) {
 180  0
                         LOG.error("Could not move: " + container.getFile());
 181  0
                         recordUnmovablePendingFile(container.getFile(), LOAD_TIME);         
 182  
                     }
 183  
                 }
 184  
             } else {
 185  0
                 if (container.getFile() != null) {
 186  0
                     LOG.info("Moving " + container.getFile() + " to loaded dir.");
 187  0
                     if((!completeDir.isDirectory() && !completeDir.mkdirs())
 188  
                         || !moveFile(completeDir, container.getFile())){
 189  0
                         LOG.error("Could not move: " + container.getFile());
 190  0
                         recordUnmovablePendingFile(container.getFile(), LOAD_TIME);         
 191  
                     }
 192  
                 }
 193  
             }
 194  0
         }
 195  0
     }
 196  
 
 197  
     private boolean inPendingMoveFailedArchive(File xmlDataFile){
 198  0
         if (xmlDataFile == null) return false;
 199  0
         BufferedReader inFile = null;
 200  0
         File movesFailedFile = new File(getXmlPendingDir(), PENDING_MOVE_FAILED_ARCHIVE_FILE);
 201  0
         if (!movesFailedFile.isFile()) return false;
 202  
         try {
 203  0
             inFile = new BufferedReader(new FileReader(movesFailedFile));
 204  
             String line;
 205  
             
 206  0
             while((line = inFile.readLine()) != null){
 207  0
                 String trimmedLine = line.trim();
 208  0
                 if(trimmedLine.equals(xmlDataFile.getName()) ||
 209  
                    trimmedLine.startsWith(xmlDataFile.getName() + "=")) { 
 210  0
                     return true;
 211  
                 }
 212  0
             }
 213  0
         } catch (IOException e){
 214  0
             LOG.warn("Error reading file " + movesFailedFile);
 215  
             //TODO try reading the pending file or stop?
 216  
         } finally {
 217  0
             if (inFile != null) try {
 218  0
                 inFile.close();
 219  0
             } catch (Exception e) {
 220  0
                 LOG.warn("Error closing buffered reader for " + movesFailedFile);
 221  0
             }
 222  
         }
 223  
       
 224  0
         return false;
 225  
     }
 226  
 
 227  
     private boolean recordUnmovablePendingFile(File unMovablePendingFile, Date dateLoaded){
 228  0
         boolean recorded = false;
 229  0
         FileWriter archiveFile = null;
 230  
         try{
 231  0
             archiveFile = new FileWriter(new File(getXmlPendingDir(), PENDING_MOVE_FAILED_ARCHIVE_FILE), true);  
 232  0
             archiveFile.write(unMovablePendingFile.getName() + "=" + dateLoaded.getTime() + NEW_LINE);
 233  0
             recorded = true;
 234  0
         } catch (IOException e){
 235  0
             LOG.error("Unable to record unmovable pending file " + unMovablePendingFile.getName() + "in the archive file " + PENDING_MOVE_FAILED_ARCHIVE_FILE);
 236  
         } finally {
 237  0
             if (archiveFile != null) {
 238  
                 try {
 239  0
                     archiveFile.close();
 240  0
                 } catch (IOException ioe) {
 241  0
                     LOG.error("Error closing unmovable pending file", ioe);
 242  0
                 }
 243  
             }
 244  
         }
 245  0
         return recorded;       
 246  
     }
 247  
 
 248  
     private boolean moveFile(File toDirectory, File fileToMove){
 249  0
         boolean moved = true;
 250  0
         if (!fileToMove.renameTo(new File(toDirectory.getPath(), fileToMove.getName()))){
 251  0
             LOG.error("Unable to move file " + fileToMove.getName() + " to directory " + toDirectory.getPath());
 252  0
             moved = false;
 253  
         }
 254  0
         return moved;
 255  
     }
 256  
 
 257  
     private File getXmlPendingDir() {
 258  0
         return new File(getXmlPendingLocation());
 259  
     }
 260  
 
 261  
     private File getXmlCompleteDir() {
 262  0
         return new File(getXmlCompletedLocation());
 263  
     }
 264  
 
 265  
     private File getXmlProblemDir() {
 266  0
         return new File(getXmlProblemLocation());
 267  
     }
 268  
 
 269  
     public String getXmlCompletedLocation() {
 270  0
         return xmlCompletedLocation;
 271  
     }
 272  
 
 273  
     public void setXmlCompletedLocation(String xmlCompletedLocation) {
 274  0
         this.xmlCompletedLocation = xmlCompletedLocation;
 275  0
     }
 276  
 
 277  
     public String getXmlPendingLocation() {
 278  0
         return xmlPendingLocation;
 279  
     }
 280  
 
 281  
     /*public boolean validate(File uploadedFile) {
 282  
         XmlDataLoaderFileFilter filter = new XmlDataLoaderFileFilter();
 283  
         return filter.accept(uploadedFile);
 284  
     }*/
 285  
 
 286  
     public void setXmlPendingLocation(String xmlPendingLocation) {
 287  0
         this.xmlPendingLocation = xmlPendingLocation;
 288  0
     }
 289  
 
 290  
     public String getXmlProblemLocation() {
 291  0
         return xmlProblemLocation;
 292  
     }
 293  
 
 294  
     public void setXmlProblemLocation(String xmlProblemLocation) {
 295  0
         this.xmlProblemLocation = xmlProblemLocation;
 296  0
     }
 297  
     public String getXmlParentDirectory() {
 298  0
         return xmlParentDirectory;
 299  
     }
 300  
     public void setXmlParentDirectory(String xmlDataParentDirectory) {
 301  0
         this.xmlParentDirectory = xmlDataParentDirectory;
 302  0
     }
 303  
 
 304  
     /**
 305  
      * Sets the polling interval time in seconds
 306  
      * @param seconds the polling interval time in seconds
 307  
      */
 308  
     public void setPollIntervalSecs(int seconds) {
 309  0
         this.pollIntervalSecs = seconds;
 310  0
     }
 311  
 
 312  
     /**
 313  
      * Gets the polling interval time in seconds
 314  
      * @return the polling interval time in seconds
 315  
      */
 316  
     public int getPollIntervalSecs() {
 317  0
         return this.pollIntervalSecs;
 318  
     }
 319  
 
 320  
     /**
 321  
      * Sets the initial delay time in seconds
 322  
      * @param seconds the initial delay time in seconds
 323  
      */
 324  
     public void setInitialDelaySecs(int seconds) {
 325  0
         this.initialDelaySecs = seconds;
 326  0
     }
 327  
 
 328  
     /**
 329  
      * Gets the initial delay time in seconds
 330  
      * @return the initial delay time in seconds
 331  
      */
 332  
     public int getInitialDelaySecs() {
 333  0
         return this.initialDelaySecs;
 334  
     }
 335  
 }