View Javadoc
1   /**
2    * Copyright 2008-2013 The Kuali Foundation
3    *
4    * Licensed under the Educational Community License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    * http://www.opensource.org/licenses/ecl2.php
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package org.codehaus.mojo.wagon.shared;
17  
18  /*
19   * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
20   * agreements. See the NOTICE file distributed with this work for additional information regarding
21   * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
22   * "License"); you may not use this file except in compliance with the License. You may obtain a
23   * copy of the License at
24   *
25   * http://www.apache.org/licenses/LICENSE-2.0
26   *
27   * Unless required by applicable law or agreed to in writing, software distributed under the License
28   * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
29   * or implied. See the License for the specific language governing permissions and limitations under
30   * the License.
31   */
32  
33  import static com.google.common.base.Preconditions.checkState;
34  import static com.google.common.collect.Lists.newArrayList;
35  import static com.google.common.collect.Maps.newTreeMap;
36  import static java.lang.String.format;
37  import static java.lang.System.currentTimeMillis;
38  import static java.util.Collections.shuffle;
39  import static org.apache.commons.io.FileUtils.touch;
40  import static org.codehaus.plexus.util.StringUtils.isBlank;
41  import static org.kuali.common.util.FormatUtils.getCount;
42  import static org.kuali.common.util.FormatUtils.getRate;
43  import static org.kuali.common.util.FormatUtils.getSize;
44  import static org.kuali.common.util.FormatUtils.getTime;
45  import static org.kuali.common.util.base.Exceptions.illegalState;
46  import static org.kuali.common.util.execute.impl.ConcurrentExecutables.executeConcurrently;
47  
48  import java.io.File;
49  import java.io.IOException;
50  import java.util.List;
51  import java.util.SortedMap;
52  
53  import org.apache.maven.plugin.logging.Log;
54  import org.apache.maven.wagon.Wagon;
55  import org.apache.maven.wagon.WagonException;
56  import org.kuali.common.util.Counter;
57  import org.kuali.common.util.LongCounter;
58  import org.kuali.common.util.execute.Executable;
59  import org.kuali.common.util.file.CanonicalFile;
60  import org.kuali.common.util.inform.PercentCompleteInformer;
61  
62  import com.google.common.collect.Lists;
63  
64  /**
65   * @plexus.component role="org.codehaus.mojo.wagon.shared.WagonDownload" role-hint="default"
66   */
67  
68  public class DefaultWagonDownload implements WagonDownload {
69  
70  	@Override
71  	public List<String> getFileList(Wagon wagon, WagonFileSet fileSet, Log logger) throws WagonException {
72  		logger.info("Scanning repository - " + wagon.getRepository().getUrl());
73  
74  		PercentCompleteInformer informer = new PercentCompleteInformer(250);
75  		WagonDirectoryScanner dirScan = new WagonDirectoryScanner();
76  		dirScan.setLogger(logger);
77  		dirScan.setWagon(wagon);
78  		dirScan.setExcludes(fileSet.getExcludes());
79  		dirScan.setIncludes(fileSet.getIncludes());
80  		dirScan.setCaseSensitive(fileSet.isCaseSensitive());
81  		dirScan.setDirectory(fileSet.getDirectory());
82  		dirScan.setInformer(informer);
83  		if (fileSet.isUseDefaultExcludes()) {
84  			dirScan.addDefaultExcludes();
85  		}
86  
87  		long start = currentTimeMillis();
88  		informer.start();
89  		dirScan.scan();
90  		long directoriesScanned = informer.getProgress();
91  		informer.stop();
92  		logger.info(format("dirs scanned  -> %s [%s]", directoriesScanned, getTime(currentTimeMillis() - start)));
93  		logger.info(format("files located -> %s", dirScan.getFilesIncluded().size()));
94  
95  		return dirScan.getFilesIncluded();
96  	}
97  
98  	@Override
99  	public void download(Wagon wagon, WagonFileSet remoteFileSet, Log logger, boolean skipExisting) throws WagonException {
100 
101 		List<String> fileList = getFileList(wagon, remoteFileSet, logger);
102 
103 		String url = wagon.getRepository().getUrl();
104 		url = url.endsWith("/") ? url : url + "/";
105 
106 		if (fileList.size() == 0) {
107 			logger.info("Nothing to download.");
108 			return;
109 		}
110 
111 		SortedMap<String, CanonicalFile> skipped = newTreeMap();
112 		SortedMap<String, CanonicalFile> downloads = newTreeMap();
113 		for (String remoteFile : fileList) {
114 			CanonicalFile destination = new CanonicalFile(remoteFileSet.getDownloadDirectory() + "/" + remoteFile);
115 			if (!isBlank(remoteFileSet.getDirectory())) {
116 				remoteFile = remoteFileSet.getDirectory() + "/" + remoteFile;
117 			}
118 			if (skipExisting && destination.exists()) {
119 				skipped.put(remoteFile, destination);
120 			} else {
121 				downloads.put(remoteFile, destination);
122 			}
123 		}
124 
125 		if (skipped.size() > 0) {
126 			logger.info(format("skipping %s files that already exist on the local file system", skipped.size()));
127 		}
128 		logger.info(format("downloading %s files ", downloads.size()));
129 		List<Executable> executables = newArrayList();
130 		long total = Math.max(100, downloads.size() / 10);
131 		PercentCompleteInformer informer = new PercentCompleteInformer(total);
132 		Counter counter = new Counter();
133 		LongCounter bytesCounter = new LongCounter();
134 		long start = currentTimeMillis();
135 		for (String remoteFile : downloads.keySet()) {
136 			CanonicalFile destination = downloads.get(remoteFile);
137 			WagonDownloadExecutable executable = WagonDownloadExecutable.builder().withDestination(destination).withRemoteFile(remoteFile).withWagon(wagon).withCounter(counter)
138 					.withTotal(downloads.size()).withStart(start).withBytesCounter(bytesCounter).withInformer(informer).build();
139 			executables.add(executable);
140 		}
141 		shuffle(executables);
142 		informer.start();
143 		executeConcurrently(executables, 10);
144 		informer.stop();
145 		long elapsed = currentTimeMillis() - start;
146 		checkState(counter.getValue() == downloads.size(), "download counter is %s but should be %s", counter.getValue(), downloads.size());
147 		List<File> files = Lists.<File> newArrayList(downloads.values());
148 		long bytes = getBytes(files);
149 		Object[] args = { getCount(files.size()), getSize(bytes), getRate(elapsed, bytes), getTime(elapsed) };
150 		logger.info(format("files: %s  size: %s  rate: %s  elapsed: %s", args));
151 	}
152 
153 	protected long getBytes(List<File> files) {
154 		long bytes = 0;
155 		for (File file : files) {
156 			bytes += file.length();
157 		}
158 		return bytes;
159 	}
160 
161 	protected void touchQuietly(File file) {
162 		try {
163 			touch(file);
164 		} catch (IOException e) {
165 			throw illegalState(e);
166 		}
167 	}
168 
169 	/**
170 	 * 
171 	 * @param wagon
172 	 *            - a Wagon instance
173 	 * @param resource
174 	 *            - Remote resource to check
175 	 * @throws WagonException
176 	 */
177 	@Override
178 	public boolean exists(Wagon wagon, String resource) throws WagonException {
179 		return wagon.resourceExists(resource);
180 	}
181 
182 }