001 /**
002 * Copyright 2008-2013 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.codehaus.mojo.wagon.shared;
017
018 /*
019 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
020 * agreements. See the NOTICE file distributed with this work for additional information regarding
021 * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
022 * "License"); you may not use this file except in compliance with the License. You may obtain a
023 * copy of the License at
024 *
025 * http://www.apache.org/licenses/LICENSE-2.0
026 *
027 * Unless required by applicable law or agreed to in writing, software distributed under the License
028 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
029 * or implied. See the License for the specific language governing permissions and limitations under
030 * the License.
031 */
032
033 import static com.google.common.base.Preconditions.checkState;
034 import static com.google.common.collect.Lists.newArrayList;
035 import static com.google.common.collect.Maps.newTreeMap;
036 import static java.lang.String.format;
037 import static java.lang.System.currentTimeMillis;
038 import static java.util.Collections.shuffle;
039 import static org.apache.commons.io.FileUtils.touch;
040 import static org.codehaus.plexus.util.StringUtils.isBlank;
041 import static org.kuali.common.util.FormatUtils.getCount;
042 import static org.kuali.common.util.FormatUtils.getRate;
043 import static org.kuali.common.util.FormatUtils.getSize;
044 import static org.kuali.common.util.FormatUtils.getTime;
045 import static org.kuali.common.util.base.Exceptions.illegalState;
046 import static org.kuali.common.util.execute.impl.ConcurrentExecutables.executeConcurrently;
047
048 import java.io.File;
049 import java.io.IOException;
050 import java.util.List;
051 import java.util.SortedMap;
052
053 import org.apache.maven.plugin.logging.Log;
054 import org.apache.maven.wagon.Wagon;
055 import org.apache.maven.wagon.WagonException;
056 import org.kuali.common.util.Counter;
057 import org.kuali.common.util.LongCounter;
058 import org.kuali.common.util.execute.Executable;
059 import org.kuali.common.util.file.CanonicalFile;
060 import org.kuali.common.util.inform.PercentCompleteInformer;
061
062 import com.google.common.collect.Lists;
063
064 /**
065 * @plexus.component role="org.codehaus.mojo.wagon.shared.WagonDownload" role-hint="default"
066 */
067
068 public class DefaultWagonDownload implements WagonDownload {
069
070 @Override
071 public List<String> getFileList(Wagon wagon, WagonFileSet fileSet, Log logger) throws WagonException {
072 logger.info("Scanning repository - " + wagon.getRepository().getUrl());
073
074 PercentCompleteInformer informer = new PercentCompleteInformer(250);
075 WagonDirectoryScanner dirScan = new WagonDirectoryScanner();
076 dirScan.setLogger(logger);
077 dirScan.setWagon(wagon);
078 dirScan.setExcludes(fileSet.getExcludes());
079 dirScan.setIncludes(fileSet.getIncludes());
080 dirScan.setCaseSensitive(fileSet.isCaseSensitive());
081 dirScan.setDirectory(fileSet.getDirectory());
082 dirScan.setInformer(informer);
083 if (fileSet.isUseDefaultExcludes()) {
084 dirScan.addDefaultExcludes();
085 }
086
087 long start = currentTimeMillis();
088 informer.start();
089 dirScan.scan();
090 long directoriesScanned = informer.getProgress();
091 informer.stop();
092 logger.info(format("dirs scanned -> %s [%s]", directoriesScanned, getTime(currentTimeMillis() - start)));
093 logger.info(format("files located -> %s", dirScan.getFilesIncluded().size()));
094
095 return dirScan.getFilesIncluded();
096 }
097
098 @Override
099 public void download(Wagon wagon, WagonFileSet remoteFileSet, Log logger, boolean skipExisting) throws WagonException {
100
101 List<String> fileList = getFileList(wagon, remoteFileSet, logger);
102
103 String url = wagon.getRepository().getUrl();
104 url = url.endsWith("/") ? url : url + "/";
105
106 if (fileList.size() == 0) {
107 logger.info("Nothing to download.");
108 return;
109 }
110
111 SortedMap<String, CanonicalFile> skipped = newTreeMap();
112 SortedMap<String, CanonicalFile> downloads = newTreeMap();
113 for (String remoteFile : fileList) {
114 CanonicalFile destination = new CanonicalFile(remoteFileSet.getDownloadDirectory() + "/" + remoteFile);
115 if (!isBlank(remoteFileSet.getDirectory())) {
116 remoteFile = remoteFileSet.getDirectory() + "/" + remoteFile;
117 }
118 if (skipExisting && destination.exists()) {
119 skipped.put(remoteFile, destination);
120 } else {
121 downloads.put(remoteFile, destination);
122 }
123 }
124
125 if (skipped.size() > 0) {
126 logger.info(format("skipping %s files that already exist on the local file system", skipped.size()));
127 }
128 logger.info(format("downloading %s files ", downloads.size()));
129 List<Executable> executables = newArrayList();
130 long total = Math.max(100, downloads.size() / 10);
131 PercentCompleteInformer informer = new PercentCompleteInformer(total);
132 Counter counter = new Counter();
133 LongCounter bytesCounter = new LongCounter();
134 long start = currentTimeMillis();
135 for (String remoteFile : downloads.keySet()) {
136 CanonicalFile destination = downloads.get(remoteFile);
137 WagonDownloadExecutable executable = WagonDownloadExecutable.builder().withDestination(destination).withRemoteFile(remoteFile).withWagon(wagon).withCounter(counter)
138 .withTotal(downloads.size()).withStart(start).withBytesCounter(bytesCounter).withInformer(informer).build();
139 executables.add(executable);
140 }
141 shuffle(executables);
142 informer.start();
143 executeConcurrently(executables, 10);
144 informer.stop();
145 long elapsed = currentTimeMillis() - start;
146 checkState(counter.getValue() == downloads.size(), "download counter is %s but should be %s", counter.getValue(), downloads.size());
147 List<File> files = Lists.<File> newArrayList(downloads.values());
148 long bytes = getBytes(files);
149 Object[] args = { getCount(files.size()), getSize(bytes), getRate(elapsed, bytes), getTime(elapsed) };
150 logger.info(format("files: %s size: %s rate: %s elapsed: %s", args));
151 }
152
153 protected long getBytes(List<File> files) {
154 long bytes = 0;
155 for (File file : files) {
156 bytes += file.length();
157 }
158 return bytes;
159 }
160
161 protected void touchQuietly(File file) {
162 try {
163 touch(file);
164 } catch (IOException e) {
165 throw illegalState(e);
166 }
167 }
168
169 /**
170 *
171 * @param wagon
172 * - a Wagon instance
173 * @param resource
174 * - Remote resource to check
175 * @throws WagonException
176 */
177 @Override
178 public boolean exists(Wagon wagon, String resource) throws WagonException {
179 return wagon.resourceExists(resource);
180 }
181
182 }