1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codehaus.mojo.wagon.shared;
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33 import static com.google.common.base.Preconditions.checkState;
34 import static com.google.common.collect.Lists.newArrayList;
35 import static com.google.common.collect.Maps.newTreeMap;
36 import static java.lang.String.format;
37 import static java.lang.System.currentTimeMillis;
38 import static java.util.Collections.shuffle;
39 import static org.apache.commons.io.FileUtils.touch;
40 import static org.codehaus.plexus.util.StringUtils.isBlank;
41 import static org.kuali.common.util.FormatUtils.getCount;
42 import static org.kuali.common.util.FormatUtils.getRate;
43 import static org.kuali.common.util.FormatUtils.getSize;
44 import static org.kuali.common.util.FormatUtils.getTime;
45 import static org.kuali.common.util.base.Exceptions.illegalState;
46 import static org.kuali.common.util.execute.impl.ConcurrentExecutables.executeConcurrently;
47
48 import java.io.File;
49 import java.io.IOException;
50 import java.util.List;
51 import java.util.SortedMap;
52
53 import org.apache.maven.plugin.logging.Log;
54 import org.apache.maven.wagon.Wagon;
55 import org.apache.maven.wagon.WagonException;
56 import org.kuali.common.util.Counter;
57 import org.kuali.common.util.LongCounter;
58 import org.kuali.common.util.execute.Executable;
59 import org.kuali.common.util.file.CanonicalFile;
60 import org.kuali.common.util.inform.PercentCompleteInformer;
61
62 import com.google.common.collect.Lists;
63
64
65
66
67
68 public class DefaultWagonDownload implements WagonDownload {
69
70 @Override
71 public List<String> getFileList(Wagon wagon, WagonFileSet fileSet, Log logger) throws WagonException {
72 logger.info("Scanning repository - " + wagon.getRepository().getUrl());
73
74 PercentCompleteInformer informer = new PercentCompleteInformer(250);
75 WagonDirectoryScanner dirScan = new WagonDirectoryScanner();
76 dirScan.setLogger(logger);
77 dirScan.setWagon(wagon);
78 dirScan.setExcludes(fileSet.getExcludes());
79 dirScan.setIncludes(fileSet.getIncludes());
80 dirScan.setCaseSensitive(fileSet.isCaseSensitive());
81 dirScan.setDirectory(fileSet.getDirectory());
82 dirScan.setInformer(informer);
83 if (fileSet.isUseDefaultExcludes()) {
84 dirScan.addDefaultExcludes();
85 }
86
87 long start = currentTimeMillis();
88 informer.start();
89 dirScan.scan();
90 long directoriesScanned = informer.getProgress();
91 informer.stop();
92 logger.info(format("dirs scanned -> %s [%s]", directoriesScanned, getTime(currentTimeMillis() - start)));
93 logger.info(format("files located -> %s", dirScan.getFilesIncluded().size()));
94
95 return dirScan.getFilesIncluded();
96 }
97
98 @Override
99 public void download(Wagon wagon, WagonFileSet remoteFileSet, Log logger, boolean skipExisting) throws WagonException {
100
101 List<String> fileList = getFileList(wagon, remoteFileSet, logger);
102
103 String url = wagon.getRepository().getUrl();
104 url = url.endsWith("/") ? url : url + "/";
105
106 if (fileList.size() == 0) {
107 logger.info("Nothing to download.");
108 return;
109 }
110
111 SortedMap<String, CanonicalFile> skipped = newTreeMap();
112 SortedMap<String, CanonicalFile> downloads = newTreeMap();
113 for (String remoteFile : fileList) {
114 CanonicalFile destination = new CanonicalFile(remoteFileSet.getDownloadDirectory() + "/" + remoteFile);
115 if (!isBlank(remoteFileSet.getDirectory())) {
116 remoteFile = remoteFileSet.getDirectory() + "/" + remoteFile;
117 }
118 if (skipExisting && destination.exists()) {
119 skipped.put(remoteFile, destination);
120 } else {
121 downloads.put(remoteFile, destination);
122 }
123 }
124
125 if (skipped.size() > 0) {
126 logger.info(format("skipping %s files that already exist on the local file system", skipped.size()));
127 }
128 logger.info(format("downloading %s files ", downloads.size()));
129 List<Executable> executables = newArrayList();
130 long total = Math.max(100, downloads.size() / 10);
131 PercentCompleteInformer informer = new PercentCompleteInformer(total);
132 Counter counter = new Counter();
133 LongCounter bytesCounter = new LongCounter();
134 long start = currentTimeMillis();
135 for (String remoteFile : downloads.keySet()) {
136 CanonicalFile destination = downloads.get(remoteFile);
137 WagonDownloadExecutable executable = WagonDownloadExecutable.builder().withDestination(destination).withRemoteFile(remoteFile).withWagon(wagon).withCounter(counter)
138 .withTotal(downloads.size()).withStart(start).withBytesCounter(bytesCounter).withInformer(informer).build();
139 executables.add(executable);
140 }
141 shuffle(executables);
142 informer.start();
143 executeConcurrently(executables, 10);
144 informer.stop();
145 long elapsed = currentTimeMillis() - start;
146 checkState(counter.getValue() == downloads.size(), "download counter is %s but should be %s", counter.getValue(), downloads.size());
147 List<File> files = Lists.<File> newArrayList(downloads.values());
148 long bytes = getBytes(files);
149 Object[] args = { getCount(files.size()), getSize(bytes), getRate(elapsed, bytes), getTime(elapsed) };
150 logger.info(format("files: %s size: %s rate: %s elapsed: %s", args));
151 }
152
153 protected long getBytes(List<File> files) {
154 long bytes = 0;
155 for (File file : files) {
156 bytes += file.length();
157 }
158 return bytes;
159 }
160
161 protected void touchQuietly(File file) {
162 try {
163 touch(file);
164 } catch (IOException e) {
165 throw illegalState(e);
166 }
167 }
168
169
170
171
172
173
174
175
176
177 @Override
178 public boolean exists(Wagon wagon, String resource) throws WagonException {
179 return wagon.resourceExists(resource);
180 }
181
182 }