View Javadoc
1   /**
2    * Copyright 2005-2015 The Kuali Foundation
3    *
4    * Licensed under the Educational Community License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    * http://www.opensource.org/licenses/ecl2.php
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package org.kuali.rice.testtools.selenium;
17  
18  import org.apache.commons.io.FileUtils;
19  import org.openqa.selenium.By;
20  import org.openqa.selenium.NoSuchWindowException;
21  import org.openqa.selenium.WebDriver;
22  import org.openqa.selenium.firefox.FirefoxDriver;
23  import org.openqa.selenium.firefox.FirefoxProfile;
24  import org.openqa.selenium.remote.DesiredCapabilities;
25  
26  import java.io.File;
27  import java.io.IOException;
28  import java.net.MalformedURLException;
29  import java.util.Arrays;
30  import java.util.HashMap;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.concurrent.TimeUnit;
35  
36  /**
37   * Base class for Selenium Jenkins Json saving.  Originally created to work with ci.rice.kuali.org which
38   * required login in order to see results (and I couldn't get various command line tools to work with CAS).
39   * ci.kuail.org doesn't require login, so using wget or such might be a better solution as it doesn't require a browser
40   * being opened.
41   *
42   * @author Kuali Rice Team (rice.collab@kuali.org)
43   */
44  public class JenkinsJsonJobResultsBase {
45  
46      /**
47       * -Dbrowser.download.dir= default is Downloads in user.home.
48       */
49      public static final String BROWSER_DOWNLOAD_DIR = "browser.download.dir";
50  
51      /**
52       * -Dbrowser.helperApps.neverAsk.saveToDisk= comma delimited list of MIME/Types to saveToDisk default is "application/zip".
53       */
54      public static final String BROWSER_HELPER_APPS_NEVER_ASK_SAVE_TO_DISK = "browser.helperApps.neverAsk.saveToDisk";
55  
56      /**
57       * -Dcas.username= CAS username.
58       */
59      private static final String CAS_USERNAME = "cas.username";
60  
61      /**
62       * -Dcas.password= CAS password.
63       */
64      private static final String CAS_PASSWORD = "cas.password";
65  
66      /**
67       * -Djenkins.base.url= default is http://ci.rice.kuali.org.
68       */
69      public static final String JENKINS_BASE_URL = "jenkins.base.url";
70  
71      /**
72       * REQUIRED -Djenkins.jobs= comma delimited with optional colon delimited list of jobs:jobNumbers.
73       *
74       * If no jobNumbers are included the last completed build number for the given job will be used.  If "all" is given for
75       * the jobNumbers all available job builds will be used.
76       */
77      public static final String JENKINS_JOBS = "jenkins.jobs";
78  
79      /**
80       * -Djson.output.dir= default is directory java is run from, directory must exist.
81       */
82      public static final String JSON_OUTPUT_DIR = "json.output.dir";
83  
84      WebDriver driver;
85      boolean passed = false;
86      String jenkinsBase;
87      String outputDirectory;
88      String[] jobsBuildsStrings;
89      Map<String, List<String>> jobsBuildsMap = new HashMap<String, List<String>>();
90      List<String> jobs = new LinkedList<String>();
91  
92      String downloadDir;
93  
94      public void setUp() throws MalformedURLException, InterruptedException {
95          if (System.getProperty(JENKINS_JOBS) == null) {
96              System.out.println("Don't know what jobs to retrieve.  -D" + JENKINS_JOBS + "= must be declared.");
97              System.exit(1);
98          }
99  
100         jenkinsBase = System.getProperty(JENKINS_BASE_URL, "http://ci.kuali.org");
101         outputDirectory = System.getProperty(JSON_OUTPUT_DIR);
102 
103         FirefoxProfile profile = new FirefoxProfile();
104         profile.setEnableNativeEvents(false);
105 
106         downloadDir = System.getProperty(BROWSER_DOWNLOAD_DIR, System.getProperty("user.home") + File.separator + "Downloads");
107         // download files automatically (don't prompt)
108         profile.setPreference("browser.download.folderList", 2);
109         profile.setPreference("browser.download.manager.showWhenStarting", false);
110         profile.setPreference("browser.download.dir", downloadDir);
111         profile.setPreference(BROWSER_HELPER_APPS_NEVER_ASK_SAVE_TO_DISK, System.getProperty(BROWSER_HELPER_APPS_NEVER_ASK_SAVE_TO_DISK, "application/zip"));
112 
113         DesiredCapabilities capabilities = new DesiredCapabilities();
114         capabilities.setCapability(FirefoxDriver.PROFILE, profile);
115 
116         driver = new FirefoxDriver(capabilities);
117         driver.manage().timeouts().implicitlyWait(WebDriverUtils.configuredImplicityWait(), TimeUnit.SECONDS);
118         driver.get(jenkinsBase + "/login?form");
119 
120         // CAS
121 //        WebDriverUtils.waitFor(driver, WebDriverUtils.configuredImplicityWait(), By.id("username"),
122 //                this.getClass().toString());
123 //        driver.findElement(By.id("username")).sendKeys(System.getProperty(CAS_USERNAME));
124 //        driver.findElement(By.id("password")).sendKeys(System.getProperty(CAS_PASSWORD));
125 //        driver.findElement(By.name("submit")).click();
126 //        Thread.sleep(1000);
127 //
128 //        exitOnLoginProblems();
129 
130         // Jenkins login page (don't login, we have authenticated through CAS already
131         WebDriverUtils.waitFor(driver, WebDriverUtils.configuredImplicityWait(), By.xpath("//span[contains(text(), 'Page generated')]"), this.getClass().toString());
132 
133         // setup jobs builds
134         jobsBuildsStrings = System.getProperty(JENKINS_JOBS).split("[,\\s]");
135         String job;
136         for (String jobsBuildsString : jobsBuildsStrings) {
137             if (jobsBuildsString.contains(":")) {
138                 List<String> jobBuilds = Arrays.asList(jobsBuildsString.split(":"));
139                 job = jobBuilds.get(0); // first item is the job name
140                 jobs.add(job);
141                 if (jobBuilds.size() == 2 && "all".equals(jobBuilds.get(1))) { // job:all
142                     jobsBuildsMap.put(job, fetchAllJobNumbers(job));
143                 } else { // regular usage
144                     jobsBuildsMap.put(job, jobBuilds.subList(1,jobBuilds.size())); // first item is the job name
145                 }
146             } else { // no jobNumbers specified, use last complete build number
147                 List<String> jobBuilds = new LinkedList<String>();
148                 jobBuilds.add(fetchLastCompletedBuildNumber(jobsBuildsString) + "");
149                 jobs.add(jobsBuildsString);
150                 jobsBuildsMap.put(jobsBuildsString, jobBuilds);
151             }
152         }
153 
154         passed = true;
155     }
156 
157     protected String calcOutputFile(String job, String jobNumber) {
158         String outputFile  = job + "-" + jobNumber + ".json";
159 
160         if (outputDirectory != null) {
161             outputFile = outputDirectory + File.separatorChar + outputFile;
162         }
163 
164         return outputFile;
165     }
166 
167     protected void closeAndQuitWebDriver() {
168         if (driver != null) {
169             if (WebDriverUtils.dontTearDownPropertyNotSet() && WebDriverUtils.dontTearDownOnFailure(passed)) {
170                 try {
171                     driver.close();
172                 } catch (NoSuchWindowException nswe) {
173                     System.out.println("NoSuchWindowException closing WebDriver " + nswe.getMessage());
174                 } finally {
175                     if (driver != null) {
176                         driver.quit();
177                     }
178                 }
179             }
180         } else {
181             System.out.println("WebDriver is null for " + this.getClass().toString());
182         }
183     }
184 
185     private void exitOnLoginProblems() {
186         boolean exit = false;
187         String pageSource = driver.getPageSource();
188 
189         if (pageSource.contains("Username is a required field.")) {
190             System.out.println("CAS Username is a required did you set -D" + CAS_USERNAME + "=");
191             exit = true;
192         }
193 
194         if (pageSource.contains("Password is a required field.")) {
195             System.out.println("CAS Password is a required did you set -D" + CAS_PASSWORD + "=");
196             exit = true;
197         }
198 
199         if (pageSource.contains("The credentials you provided cannot be determined to be authentic.")) {
200             System.out.println("CAS Login Error");
201             exit = true;
202         }
203 
204         if (exit) {
205             System.exit(1);
206         }
207     }
208 
209     protected List<String> fetchAllJobNumbers(String job) {
210         List<String> allJobNumbers = new LinkedList<String>();
211         String url = null;
212         String jobJson;
213 
214         url = jenkinsBase + "/job/" + job + "/api/json";
215 
216         try {
217             jobJson = retrieveJson(url);
218 
219             String jsonJobNumber;
220 
221             while (jobJson.contains(("{\"number\":"))) {
222                 jsonJobNumber = jobJson.substring(jobJson.indexOf("{\"number\":") + 10, jobJson.length());
223                 jsonJobNumber = jsonJobNumber.substring(0, jsonJobNumber.indexOf(","));
224 
225                 allJobNumbers.add(jsonJobNumber);
226 
227                 jobJson = jobJson.substring(jobJson.indexOf("{\"number\":") + 9, jobJson.length()); // strip off while condition
228             }
229         } catch (Exception e) {
230             System.err.println("Exception fetching job " + job + " with url " + url + e.getMessage());
231         }
232 
233         return allJobNumbers;
234     }
235 
236     protected void fetchArchive(String job, String jobNumber) throws InterruptedException, IOException {
237         String archiveUrl = jenkinsBase;
238 
239         // Views will need to be updated/improved to work with ci.kuali.org
240         if (job.contains("rice-2.4")) {
241             archiveUrl += "/view/rice-2.4";
242         }
243 
244         archiveUrl += "/job/" + job + "/" + jobNumber+ "/artifact/*zip*/archive.zip";
245         driver.get(archiveUrl);
246         Thread.sleep(10000); //zip needs time to download
247         FileUtils.moveFile(new File(downloadDir + File.separator + "archive.zip"), new File(downloadDir + File.separator + job + "-" + jobNumber + ".zip"));
248     }
249 
250     protected void fetchAndWriteTestReport(String job, String jobNumber) throws InterruptedException, IOException {
251         String url;
252         String json;
253         String outputFile;
254         url = jenkinsBase + "/job/" + job + "/" + jobNumber + "/testReport/api/json";
255         json = retrieveJson(url);
256 
257         outputFile = calcOutputFile(job, jobNumber);
258 
259         // Add some end of lines to avoid the entire file being written out as 1 line
260         json = json.replaceAll("}],", "}],\n\n");
261 
262         FileUtils.writeStringToFile(new File(outputFile), json);
263     }
264 
265     protected void fetchAndWriteTestReport(String job, String[] jobNumbers) throws InterruptedException, IOException {
266         for (String jobNumber : jobNumbers) {
267             fetchAndWriteTestReport(job, jobNumber);
268         }
269     }
270 
271     protected String fetchLastCompletedBuildNumber(String job) throws InterruptedException {
272         String url = jenkinsBase + "/job/" + job + "/api/json";
273         String jobNumber = null;
274         try {
275             jobNumber = retrieveJson(url);
276             jobNumber = jobNumber.substring(jobNumber.indexOf("\"lastCompletedBuild\":{\"number\":") + 31, jobNumber.length());
277             jobNumber = jobNumber.substring(0, jobNumber.indexOf(","));
278         } catch (InterruptedException e) {
279             System.err.println("Exception fetching job " + job + " with url " + url + e.getMessage());
280             throw e;
281         }
282         return jobNumber;
283     }
284 
285     protected String retrieveJson(String url) throws InterruptedException {
286         driver.get(url);
287         Thread.sleep(500);
288         int secondsWaited = 0;
289 
290         while (!driver.getPageSource().contains("</pre></body></html>") && secondsWaited++ < 2 * WebDriverUtils.configuredImplicityWait()) {
291             Thread.sleep(1000);
292         }
293 
294         String json = driver.getPageSource();
295 
296         try {
297             // index out of bounds can be cause by testReport json not existing (canceled, or a job with no tests), which results in the regular html view of the job number
298             json = json.substring(json.indexOf("<pre>") + 5, json.indexOf("</pre></body></html>"));
299         } catch (IndexOutOfBoundsException iooobe) {
300             System.out.println("No JSON results for " + url + " this can be caused by jobs with no test results, from either the test being stopped, aborted or non-test jobs.");
301         }
302 
303         return json;
304     }
305 }