001/*
002 * Copyright 2007 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.kuali.ole.coa.service.impl;
017
018import java.io.BufferedReader;
019import java.io.IOException;
020import java.io.InputStream;
021import java.io.InputStreamReader;
022import java.util.Calendar;
023import java.util.Collection;
024import java.util.Comparator;
025import java.util.List;
026import java.util.Map;
027import java.util.SortedMap;
028import java.util.TreeMap;
029
030import org.apache.commons.lang.StringUtils;
031import org.apache.commons.net.ftp.FTPClient;
032import org.apache.commons.net.ftp.FTPReply;
033import org.kuali.ole.coa.batch.CfdaBatchStep;
034import org.kuali.ole.coa.businessobject.CFDA;
035import org.kuali.ole.coa.businessobject.CfdaUpdateResults;
036import org.kuali.ole.coa.service.CfdaService;
037import org.kuali.ole.sys.OLEConstants;
038import org.kuali.ole.sys.context.SpringContext;
039import org.kuali.rice.core.api.datetime.DateTimeService;
040import org.kuali.rice.coreservice.framework.parameter.ParameterService;
041import org.kuali.rice.krad.service.BusinessObjectService;
042
043import au.com.bytecode.opencsv.CSVReader;
044
045public class CfdaServiceImpl implements CfdaService {
046    private static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(CfdaServiceImpl.class);
047
048    protected BusinessObjectService businessObjectService;
049    protected static Comparator cfdaComparator;
050
051    static {
052        cfdaComparator = new Comparator() {
053            @Override
054            public int compare(Object o1, Object o2) {
055                String lhs = (String) o1;
056                String rhs = (String) o2;
057                return lhs.compareTo(rhs);
058            }
059        };
060    }
061
062    /**
063     * @return
064     * @throws IOException
065     */
066    public SortedMap<String, CFDA> getGovCodes() throws IOException {
067        Calendar calendar = SpringContext.getBean(DateTimeService.class).getCurrentCalendar();
068        SortedMap<String, CFDA> govMap = new TreeMap<String, CFDA>();
069
070        // ftp://ftp.cfda.gov/programs09187.csv
071        String govURL = SpringContext.getBean(ParameterService.class).getParameterValueAsString(CfdaBatchStep.class, OLEConstants.SOURCE_URL_PARAMETER);
072        String fileName = StringUtils.substringAfterLast(govURL, "/");
073        govURL = StringUtils.substringBeforeLast(govURL, "/");
074        if (StringUtils.contains(govURL, "ftp://")) {
075            govURL = StringUtils.remove(govURL, "ftp://");
076        }
077
078        // need to pull off the '20' in 2009
079        String year = "" + calendar.get(Calendar.YEAR);
080        year = year.substring(2, 4);
081        fileName = fileName + year;
082
083        // the last 3 numbers in the file name are the day of the year, but the files are from "yesterday"
084        fileName = fileName + String.format("%03d", calendar.get(Calendar.DAY_OF_YEAR) - 1);
085        fileName = fileName + ".csv";
086
087        LOG.info("Getting government file: " + fileName + " for update");
088
089        InputStream inputStream = null;
090        FTPClient ftp = new FTPClient();
091        try {
092            ftp.connect(govURL);
093            int reply = ftp.getReplyCode();
094
095            if (!FTPReply.isPositiveCompletion(reply)) {
096                LOG.error("FTP connection to server not established.");
097                throw new IOException("FTP connection to server not established.");
098            }
099
100            boolean loggedIn = ftp.login("anonymous", "");
101            if (!loggedIn) {
102                LOG.error("Could not login as anonymous.");
103                throw new IOException("Could not login as anonymous.");
104            }
105
106            LOG.info("Successfully connected and logged in");
107            ftp.enterLocalPassiveMode();
108            inputStream = ftp.retrieveFileStream(fileName);
109            if (inputStream != null) {
110                LOG.info("reading input stream");
111                InputStreamReader screenReader = new InputStreamReader(inputStream);
112                BufferedReader screen = new BufferedReader(screenReader);
113
114                CSVReader csvReader = new CSVReader(screenReader, ',', '"', 1);
115                List<String[]> lines = csvReader.readAll();
116                for (String[] line : lines) {
117                    String title = line[0];
118                    String number = line[1];
119
120                    CFDA cfda = new CFDA();
121                    cfda.setCfdaNumber(number);
122                    cfda.setCfdaProgramTitleName(title);
123
124                    govMap.put(number, cfda);
125                }
126            }
127
128            ftp.logout();
129            ftp.disconnect();
130        }
131        finally {
132            if (ftp.isConnected()) {
133                ftp.disconnect();
134            }
135        }
136
137        return govMap;
138    }
139
140    /**
141     * @return
142     * @throws IOException
143     */
144    public SortedMap<String, CFDA> getKfsCodes() throws IOException {
145        Collection allCodes = businessObjectService.findAll(CFDA.class);
146
147        SortedMap<String, CFDA> kfsMapAll = new TreeMap<String, CFDA>(cfdaComparator);
148        for (Object o : allCodes) {
149            CFDA c = (CFDA) o;
150            kfsMapAll.put(c.getCfdaNumber(), c);
151        }
152        return kfsMapAll;
153    }
154
155    /**
156     *
157     */
158    @Override
159    public CfdaUpdateResults update() throws IOException {
160
161        CfdaUpdateResults results = new CfdaUpdateResults();
162        Map<String, CFDA> govMap = null;
163
164        try {
165            govMap = getGovCodes();
166        }
167        catch (IOException ioe) {
168            LOG.error("Error connecting to URL resource: " + ioe.getMessage(), ioe);
169            StringBuilder builder = new StringBuilder();
170            builder.append("No updates took place.\n");
171            builder.append(ioe.getMessage());
172            results.setMessage(builder.toString());
173            return results;
174        }
175        Map<String, CFDA> kfsMap = getKfsCodes();
176
177        results.setNumberOfRecordsInKfsDatabase(kfsMap.keySet().size());
178        results.setNumberOfRecordsRetrievedFromWebSite(govMap.keySet().size());
179
180        for (Object key : kfsMap.keySet()) {
181
182            CFDA cfdaKfs = kfsMap.get(key);
183            CFDA cfdaGov = govMap.get(key);
184
185            if (cfdaKfs.getCfdaMaintenanceTypeId().startsWith("M")) {
186                // Leave it alone. It's maintained manually.
187                results.setNumberOfRecordsNotUpdatedBecauseManual(1 + results.getNumberOfRecordsNotUpdatedBecauseManual());
188            }
189            else if (cfdaKfs.getCfdaMaintenanceTypeId().startsWith("A")) {
190
191                if (cfdaGov == null) {
192                    if (cfdaKfs.isActive()) {
193                        cfdaKfs.setActive(false);
194                        businessObjectService.save(cfdaKfs);
195                        results.setNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite(results.getNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite() + 1);
196                    }
197                    else {
198                        // Leave it alone for historical purposes
199                        results.setNumberOfRecrodsNotUpdatedForHistoricalPurposes(results.getNumberOfRecrodsNotUpdatedForHistoricalPurposes() + 1);
200                    }
201                }
202                else {
203                    if (cfdaKfs.isActive()) {
204                        results.setNumberOfRecordsUpdatedBecauseAutomatic(results.getNumberOfRecordsUpdatedBecauseAutomatic() + 1);
205                    }
206                    else {
207                        cfdaKfs.setActive(true);
208                        results.setNumberOfRecordsReActivated(results.getNumberOfRecordsReActivated() + 1);
209                    }
210
211                    cfdaKfs.setCfdaProgramTitleName(cfdaGov.getCfdaProgramTitleName());
212                    businessObjectService.save(cfdaKfs);
213                }
214            }
215
216            // Remove it from the govMap so we know what codes from the govMap don't already exist in KFS.
217            govMap.remove(key);
218        }
219
220        // What's left in govMap now is just the codes that don't exist in KFS
221        for (String key : govMap.keySet()) {
222            CFDA cfdaGov = govMap.get(key);
223            cfdaGov.setCfdaMaintenanceTypeId("AUTOMATIC");
224            cfdaGov.setActive(true);
225            businessObjectService.save(cfdaGov);
226            results.setNumberOfRecordsNewlyAddedFromWebSite(results.getNumberOfRecordsNewlyAddedFromWebSite() + 1);
227        }
228
229        return results;
230    }
231
232    public void setBusinessObjectService(BusinessObjectService businessObjectService) {
233        this.businessObjectService = businessObjectService;
234    }
235
236    @Override
237    public CFDA getByPrimaryId(String cfdaNumber) {
238        if (StringUtils.isBlank(cfdaNumber)) {
239            return null;
240        }
241        return businessObjectService.findBySinglePrimaryKey(CFDA.class, cfdaNumber.trim());
242    }
243
244}