1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.kuali.common.util;
17
18 import java.io.File;
19 import java.io.IOException;
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.Collections;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Set;
26
27 import org.apache.commons.io.FileUtils;
28 import org.apache.commons.lang3.StringUtils;
29 import org.kuali.common.util.base.Threads;
30 import org.kuali.common.util.execute.CopyFileRequest;
31 import org.kuali.common.util.execute.CopyFileResult;
32 import org.kuali.common.util.file.DirDiff;
33 import org.kuali.common.util.file.DirRequest;
34 import org.kuali.common.util.file.MD5Result;
35 import org.slf4j.Logger;
36 import org.slf4j.LoggerFactory;
37
38 public class FileSystemUtils {
39
40 private static final Logger logger = LoggerFactory.getLogger(FileSystemUtils.class);
41
42 public static final String RECURSIVE_FILE_INCLUDE_PATTERN = "**/**";
43 public static final List<String> DEFAULT_RECURSIVE_INCLUDES = Arrays.asList(RECURSIVE_FILE_INCLUDE_PATTERN);
44
45 private static final String SVN_PATTERN = "**/.svn/**";
46 private static final String GIT_PATTERN = "**/.git/**";
47 public static final List<String> DEFAULT_SCM_IGNORE_PATTERNS = Arrays.asList(SVN_PATTERN, GIT_PATTERN);
48
49
50
51
52 public static List<File> getAllNonScmFiles(File dir) {
53 return getAllNonScmFiles(dir, DEFAULT_SCM_IGNORE_PATTERNS);
54 }
55
56
57
58
59 public static List<File> getAllNonScmFiles(File dir, List<String> scmIgnorePatterns) {
60 SimpleScanner scanner = new SimpleScanner(dir, DEFAULT_RECURSIVE_INCLUDES, scmIgnorePatterns);
61 return scanner.getFiles();
62 }
63
64
65
66
67
68
69
70
71
72
73
74
75
76 @Deprecated
77 public static DirectoryDiff prepareScmDir(PrepareScmDirRequest request) {
78 return prepareScmDir(request, null, false);
79 }
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95 @Deprecated
96 public static DirectoryDiff prepareScmDir(PrepareScmDirRequest request, File relativeDir, boolean diffOnly) {
97
98
99 Assert.notNull(request, "request is null");
100 Assert.notNull(request.getSrcDir(), "srcDir is null");
101 Assert.notNull(request.getScmDir(), "scmDir is null");
102
103
104 Assert.isExistingDir(request.getSrcDir(), "srcDir is not an existing directory");
105 Assert.isExistingDir(request.getScmDir(), "scmDir is not an existing directory");
106
107
108 DirectoryDiffRequest diffRequest = new DirectoryDiffRequest();
109 diffRequest.setDir1(request.getSrcDir());
110 diffRequest.setDir2(request.getScmDir());
111 diffRequest.setExcludes(request.getScmIgnorePatterns());
112
113
114 DirectoryDiff diff = getDiff(diffRequest);
115
116
117 if (!diffOnly) {
118 org.kuali.common.util.execute.CopyFilePatternsExecutable exec = new org.kuali.common.util.execute.CopyFilePatternsExecutable();
119 exec.setSrcDir(request.getSrcDir());
120 exec.setDstDir(request.getScmDir());
121 exec.setExcludes(request.getScmIgnorePatterns());
122 exec.setRelativeDir(relativeDir);
123 exec.execute();
124 }
125
126
127 return diff;
128 }
129
130 public static List<File> getFiles(File dir, List<String> includes, List<String> excludes) {
131 SimpleScanner scanner = new SimpleScanner(dir, includes, excludes);
132 return scanner.getFiles();
133 }
134
135 @Deprecated
136 public static DirectoryDiff getDiff(File dir1, File dir2, List<String> includes, List<String> excludes) {
137 DirectoryDiffRequest request = new DirectoryDiffRequest();
138 request.setDir1(dir1);
139 request.setDir2(dir2);
140 request.setIncludes(includes);
141 request.setExcludes(excludes);
142 return getDiff(request);
143 }
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159 public static DirDiff getMD5Diff(DirRequest request) {
160
161
162 DirDiff diff = getQuickDiff(request);
163
164
165
166 fillInMD5Results(diff);
167
168
169 return diff;
170 }
171
172 public static List<MD5Result> getMD5Results(List<File> sources, List<File> targets) {
173 Assert.isTrue(sources.size() == targets.size(), "lists are not the same size");
174 List<MD5Result> results = new ArrayList<MD5Result>();
175 for (int i = 0; i < sources.size(); i++) {
176 File source = sources.get(i);
177 File target = targets.get(i);
178 MD5Result md5Result = getMD5Result(source, target);
179 results.add(md5Result);
180 }
181 return results;
182 }
183
184 protected static void fillInMD5Results(DirDiff diff) {
185 List<File> sources = getFullPaths(diff.getSourceDir(), diff.getBoth());
186 List<File> targets = getFullPaths(diff.getTargetDir(), diff.getBoth());
187
188 List<MD5Result> results = getMD5Results(sources, targets);
189
190 List<MD5Result> different = new ArrayList<MD5Result>();
191 List<MD5Result> identical = new ArrayList<MD5Result>();
192 for (MD5Result md5Result : results) {
193 String sourceChecksum = md5Result.getSourceChecksum();
194 String targetChecksum = md5Result.getTargetChecksum();
195 Assert.notNull(sourceChecksum, "sourceChecksum is null");
196 Assert.notNull(targetChecksum, "targetChecksum is null");
197 if (StringUtils.equals(sourceChecksum, targetChecksum)) {
198 identical.add(md5Result);
199 } else {
200 different.add(md5Result);
201 }
202 }
203
204
205 diff.setDifferent(different);
206 diff.setIdentical(identical);
207 }
208
209 public static MD5Result getMD5Result(File source, File target) {
210
211 String sourceChecksum = LocationUtils.getMD5Checksum(source);
212 String targetChecksum = LocationUtils.getMD5Checksum(target);
213
214 return new MD5Result(source, sourceChecksum, target, targetChecksum);
215 }
216
217
218
219
220
221
222
223
224
225
226
227
228
229 @Deprecated
230 public static DirectoryDiff getDiff(DirectoryDiffRequest request) {
231 DirRequest newRequest = new DirRequest();
232 newRequest.setExcludes(request.getExcludes());
233 newRequest.setIncludes(request.getIncludes());
234 newRequest.setSourceDir(request.getDir1());
235 newRequest.setTargetDir(request.getDir2());
236 DirDiff diff = getQuickDiff(newRequest);
237
238 DirectoryDiff dd = new DirectoryDiff(diff.getSourceDir(), diff.getTargetDir());
239 dd.setBoth(diff.getBoth());
240 dd.setDir1Only(diff.getSourceDirOnly());
241 dd.setDir2Only(diff.getTargetDirOnly());
242 return dd;
243 }
244
245 public static DirDiff getQuickDiff(DirRequest request) {
246
247
248 List<File> sourceFiles = getFiles(request.getSourceDir(), request.getIncludes(), request.getExcludes());
249 List<File> targetFiles = getFiles(request.getTargetDir(), request.getIncludes(), request.getExcludes());
250
251
252 Set<String> sourcePaths = new HashSet<String>(getRelativePaths(request.getSourceDir(), sourceFiles));
253 Set<String> targetPaths = new HashSet<String>(getRelativePaths(request.getTargetDir(), targetFiles));
254
255
256 Set<String> both = SetUtils.intersection(sourcePaths, targetPaths);
257
258
259 Set<String> sourceOnly = SetUtils.difference(sourcePaths, targetPaths);
260
261
262 Set<String> targetOnly = SetUtils.difference(targetPaths, sourcePaths);
263
264 logger.debug("source={}, sourceOnly.size()={}", request.getSourceDir(), sourceOnly.size());
265 logger.debug("target={}, targetOnly.size()={}", request.getTargetDir(), targetOnly.size());
266
267
268 DirDiff result = new DirDiff(request.getSourceDir(), request.getTargetDir());
269
270
271 result.setBoth(new ArrayList<String>(both));
272 result.setSourceDirOnly(new ArrayList<String>(sourceOnly));
273 result.setTargetDirOnly(new ArrayList<String>(targetOnly));
274
275
276 Collections.sort(result.getBoth());
277 Collections.sort(result.getSourceDirOnly());
278 Collections.sort(result.getTargetDirOnly());
279
280
281 return result;
282 }
283
284
285
286
287 public static MonitorTextFileResult monitorTextFile(File file, String token, int intervalMillis, int timeoutMillis, String encoding) {
288
289
290 Assert.notNull(file, "file is null");
291 Assert.hasText(token, "token has no text");
292 Assert.hasText(encoding, "encoding has no text");
293 Assert.isTrue(intervalMillis > 0, "interval must be a positive integer");
294 Assert.isTrue(timeoutMillis > 0, "timeout must be a positive integer");
295
296
297 long start = System.currentTimeMillis();
298 long stop = start + timeoutMillis;
299 boolean exists = false;
300 boolean contains = false;
301 boolean timeoutExceeded = false;
302 long now = -1;
303 String content = null;
304
305
306 for (;;) {
307
308
309 if (now != -1) {
310 Threads.sleep(intervalMillis);
311 }
312
313
314 now = System.currentTimeMillis();
315 if (now > stop) {
316 timeoutExceeded = true;
317 break;
318 }
319
320
321 exists = LocationUtils.exists(file);
322 if (!exists) {
323 continue;
324 }
325
326
327 content = LocationUtils.toString(file, encoding);
328 contains = StringUtils.contains(content, token);
329 if (contains) {
330
331 break;
332 }
333 }
334
335
336 long elapsed = now - start;
337
338
339 MonitorTextFileResult mtfr = new MonitorTextFileResult(exists, contains, timeoutExceeded, elapsed);
340 mtfr.setAbsolutePath(LocationUtils.getCanonicalPath(file));
341 mtfr.setContent(content);
342 return mtfr;
343 }
344
345 public static List<SyncResult> syncFiles(List<SyncRequest> requests) throws IOException {
346 List<SyncResult> results = new ArrayList<SyncResult>();
347 for (SyncRequest request : requests) {
348 SyncResult result = syncFiles(request);
349 results.add(result);
350 }
351 return results;
352 }
353
354 public static SyncResult syncFilesQuietly(SyncRequest request) {
355 try {
356 return syncFiles(request);
357 } catch (IOException e) {
358 throw new IllegalStateException("Unexpected IO error");
359 }
360 }
361
362 public static SyncResult syncFiles(SyncRequest request) throws IOException {
363 logger.info("Sync [{}] -> [{}]", request.getSrcDir(), request.getDstDir());
364 List<File> dstFiles = getAllNonScmFiles(request.getDstDir());
365 List<File> srcFiles = request.getSrcFiles();
366
367 List<String> dstPaths = getRelativePaths(request.getDstDir(), dstFiles);
368 List<String> srcPaths = getRelativePaths(request.getSrcDir(), srcFiles);
369
370 List<String> adds = new ArrayList<String>();
371 List<String> updates = new ArrayList<String>();
372 List<String> deletes = new ArrayList<String>();
373
374 for (String srcPath : srcPaths) {
375 boolean existing = dstPaths.contains(srcPath);
376 if (existing) {
377 updates.add(srcPath);
378 } else {
379 adds.add(srcPath);
380 }
381 }
382 for (String dstPath : dstPaths) {
383 boolean extra = !srcPaths.contains(dstPath);
384 if (extra) {
385 deletes.add(dstPath);
386 }
387 }
388
389 copyFiles(request.getSrcDir(), request.getSrcFiles(), request.getDstDir());
390
391 SyncResult result = new SyncResult();
392 result.setAdds(getFullPaths(request.getDstDir(), adds));
393 result.setUpdates(getFullPaths(request.getDstDir(), updates));
394 result.setDeletes(getFullPaths(request.getDstDir(), deletes));
395 return result;
396 }
397
398 protected static void copyFiles(File srcDir, List<File> files, File dstDir) throws IOException {
399 for (File file : files) {
400 String relativePath = getRelativePath(srcDir, file);
401 File dstFile = new File(dstDir, relativePath);
402 FileUtils.copyFile(file, dstFile);
403 }
404 }
405
406 public static List<File> getFullPaths(File dir, Set<String> relativePaths) {
407 return getFullPaths(dir, new ArrayList<String>(relativePaths));
408 }
409
410 public static List<File> getSortedFullPaths(File dir, List<String> relativePaths) {
411 List<File> files = getFullPaths(dir, relativePaths);
412 Collections.sort(files);
413 return files;
414 }
415
416 public static List<File> getFullPaths(File dir, List<String> relativePaths) {
417 List<File> files = new ArrayList<File>();
418 for (String relativePath : relativePaths) {
419 File file = new File(dir, relativePath);
420 File canonical = new File(LocationUtils.getCanonicalPath(file));
421 files.add(canonical);
422 }
423 return files;
424 }
425
426 protected static List<String> getRelativePaths(File dir, List<File> files) {
427 List<String> relativePaths = new ArrayList<String>();
428 for (File file : files) {
429 String relativePath = getRelativePath(dir, file);
430 relativePaths.add(relativePath);
431 }
432 return relativePaths;
433 }
434
435
436
437
438 public static boolean isParent(File parent, File child) {
439 if (parent == null || child == null) {
440 return false;
441 }
442
443 String parentPath = LocationUtils.getCanonicalPath(parent);
444 String childPath = LocationUtils.getCanonicalPath(child);
445
446 if (StringUtils.equals(parentPath, childPath)) {
447 return false;
448 } else {
449 return StringUtils.contains(childPath, parentPath);
450 }
451 }
452
453
454
455
456
457 public static String getRelativePathQuietly(File parentDir, File file) {
458 Assert.notNull(file, "file is null");
459 if (isParent(parentDir, file)) {
460 return getRelativePath(parentDir, file);
461 } else {
462 return LocationUtils.getCanonicalPath(file);
463 }
464 }
465
466 public static String getRelativePath(File dir, File file) {
467 String dirPath = LocationUtils.getCanonicalPath(dir);
468 String filePath = LocationUtils.getCanonicalPath(file);
469 if (!StringUtils.contains(filePath, dirPath)) {
470 throw new IllegalArgumentException(file + " does not reside under " + dir);
471 }
472 return StringUtils.remove(filePath, dirPath);
473 }
474
475 public static List<CopyFileRequest> getCopyFileRequests(File srcDir, List<String> includes, List<String> excludes, File dstDir) {
476 SimpleScanner scanner = new SimpleScanner(srcDir, includes, excludes);
477 List<File> srcFiles = scanner.getFiles();
478
479 List<CopyFileRequest> requests = new ArrayList<CopyFileRequest>();
480 for (File srcFile : srcFiles) {
481 String relativePath = FileSystemUtils.getRelativePath(srcDir, srcFile);
482 File dstFile = new File(dstDir, relativePath);
483 CopyFileRequest request = new CopyFileRequest(srcFile, dstFile);
484 requests.add(request);
485 }
486 return requests;
487 }
488
489 public static CopyFileResult copyFile(File src, File dst) {
490 try {
491 long start = System.currentTimeMillis();
492 boolean overwritten = dst.exists();
493 FileUtils.copyFile(src, dst);
494 return new CopyFileResult(src, dst, overwritten, System.currentTimeMillis() - start);
495 } catch (IOException e) {
496 throw new IllegalStateException("Unexpected IO error", e);
497 }
498 }
499
500 public static List<CopyFileResult> copyFiles(List<CopyFileRequest> requests) {
501 List<CopyFileResult> results = new ArrayList<CopyFileResult>();
502 for (CopyFileRequest request : requests) {
503 CopyFileResult result = copyFile(request.getSource(), request.getDestination());
504 results.add(result);
505 }
506 return results;
507 }
508
509 }