package org.kuali.common.devops.jenkins.monitor;
   
   import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
   import static com.google.common.collect.Iterables.filter;
   import static com.google.common.collect.Lists.newArrayList;
   import static com.google.common.collect.Maps.newHashMap;
   import static java.lang.String.format;
   import static java.lang.System.currentTimeMillis;
   import static java.lang.System.getProperty;
  import static java.util.Arrays.asList;
  import static java.util.Collections.sort;
  import static org.apache.commons.io.FileUtils.forceDelete;
  import static org.apache.commons.io.FileUtils.touch;
  import static org.apache.commons.io.FileUtils.write;
  import static org.apache.commons.lang.StringUtils.substringBetween;
  import static org.apache.commons.lang3.StringUtils.isBlank;
  import static org.apache.commons.lang3.StringUtils.replace;
  import static org.kuali.common.core.json.jackson.JacksonContext.newDefaultObjectMapper;
  import static org.kuali.common.util.Encodings.UTF8;
  import static org.kuali.common.util.Encodings.encodeUTF8;
  import static org.kuali.common.util.FormatUtils.getTime;
  import static org.kuali.common.util.base.Exceptions.illegalState;
  import static org.kuali.common.util.base.Precondition.checkNotBlank;
  import static org.kuali.common.util.base.Precondition.checkNotNull;
  import static org.kuali.common.util.encrypt.Encryption.getDefaultEncryptor;
  import static org.kuali.common.util.file.Files.getCanonicalFile;
  import static org.kuali.common.util.log.LoggerUtils.logTable;
  import static org.kuali.common.util.log.Loggers.newLogger;
  
  import java.io.File;
  import java.io.IOException;
  import java.util.List;
  import java.util.Map;
  
  import org.apache.commons.io.FileUtils;
  import org.junit.Test;
  import org.kuali.common.aws.ec2.api.EC2Service;
  import org.kuali.common.aws.ec2.impl.DefaultEC2Service;
  import org.kuali.common.aws.model.ImmutableAWSCredentials;
  import org.kuali.common.core.cli.api.CommandLineService;
  import org.kuali.common.core.cli.plexus.PlexusCLIService;
  import org.kuali.common.core.json.api.JsonService;
  import org.kuali.common.core.json.jackson.JacksonContext;
  import org.kuali.common.core.json.jackson.JacksonJsonService;
  import org.kuali.common.devops.jenkins.monitor.model.JenkinsCredentials;
  import org.kuali.common.devops.jenkins.monitor.model.JenkinsMaster;
  import org.kuali.common.devops.jenkins.monitor.model.JenkinsNode;
  import org.kuali.common.devops.jenkins.monitor.model.JenkinsSlave;
  import org.kuali.common.devops.jenkins.monitor.model.JenkinsSlaveComparator;
  import org.kuali.common.devops.jenkins.monitor.ses.DefaultEmailService;
  import org.kuali.common.devops.jenkins.monitor.ses.Email;
  import org.kuali.common.devops.jenkins.monitor.ses.EmailService;
  import org.kuali.common.util.LocationUtils;
  import org.kuali.common.util.encrypt.Encryptor;
  import org.slf4j.Logger;
  
  import com.amazonaws.auth.AWSCredentials;
  import com.amazonaws.regions.Region;
  import com.amazonaws.regions.Regions;
  import com.amazonaws.services.ec2.model.Instance;
  import com.fasterxml.jackson.databind.ObjectMapper;
  import com.google.common.base.Predicate;
  
  public class NodeMonitorTest {
  
  	private static final Logger logger = newLogger();
  
  	private static final CommandLineService CLI = new PlexusCLIService();
  
  	@Test
  	public void test() {
  		boolean dryRun = Boolean.getBoolean("dryRun");
  		long now = currentTimeMillis();
  		String hostname = getHostname();
  		JenkinsMaster master = getJenkinsMaster(hostname);
  		List<JenkinsNode> slaves = newArrayList(filter(master.getNodes(), BuildSlavePredicate.INSTANCE));
  		logger.info(format("total nodes --> %s", master.getNodes().size()));
  		logger.info(format("total slaves -> %s", slaves.size()));
  		List<JenkinsSlave> elements = getJenkinsSlaves(slaves);
  		sort(elements, JenkinsSlaveComparator.INSTANCE);
  		List<String> columns = asList("node", "jenkins", "amazon", "age");
  		List<Object[]> rows = newArrayList();
  		for (JenkinsSlave slave : elements) {
  			String id = slave.getInstanceId();
  			String j = slave.isJenkinsOnline() ? "online" : "offline";
  			String a = slave.isAmazonOnline() ? "online" : "offline";
  			Object[] row = { id, j, a, getTime(slave.getAge()) };
  			rows.add(row);
  		}
  		logTable(columns, rows);
  		List<JenkinsSlave> badSlaves = newArrayList(filter(elements, BadSlavePredicate.INSTANCE));
  		logger.info(format("bad slaves ---> %s", badSlaves.size()));
  		if (dryRun) {
  			logger.info(format("dry run, skipping bad slave admin"));
  		} else {
  			for (JenkinsSlave badSlave : badSlaves) {
  				stopJob(hostname, badSlave);
  				connectNode(hostname, badSlave);
  			}
 			if (!badSlaves.isEmpty()) {
 				runGroovy(hostname);
 			}
 		}
 		logger.info(format("elapsed -> %s", getTime(currentTimeMillis() - now)));
 		if (!dryRun && !badSlaves.isEmpty()) {
 			EmailService service = getEmailService();
 			Email email = getEmail(badSlaves);
 			service.send(email);
 		}
 	}
 
 	private Email getEmail(List<JenkinsSlave> badSlaves) {
 		Encryptor enc = getDefaultEncryptor();
 		String from = enc.decrypt("U2FsdGVkX18ooea503uMsSTy2OMsXLCJHUp3vqu1ixxibe2yQM5AoyCYa7+KIGLH");
 		List<String> to = asList(from);
 		List<String> list = newArrayList();
 		for (JenkinsSlave badSlave : badSlaves) {
 			list.add(badSlave.getDisplayName());
 		}
 		String master = checkedProperty("JENKINS_MASTER");
 		String subject = format("[%s] -> disconnected slaves", master);
 		StringBuilder sb = new StringBuilder();
 		sb.append("<table border=1 cellpadding=0 cellspacing=0>\n");
 		String buildUrl = checkedProperty("BUILD_URL");
 		sb.append(" <tr><td style='padding:3px;' align=right>build url:</td><td style='padding:3px;'>" + buildUrl + "</td></tr>\n");
 		for (JenkinsSlave badSlave : badSlaves) {
 			sb.append(" <tr><td style='padding:3px;'>reconnected:</td><td style='padding:3px;'>" + badSlave.getDisplayName() + "</td></tr>\n");
 		}
 		sb.append("</table>\n");
 		String body = sb.toString();
 		return Email.builder().withSubject(subject).withBody(body).withFrom(from).withTo(to).withHtml(true).build();
 	}
 
 	private String checkedProperty(String key) {
 		String sys = System.getProperty(key);
 		if (!isBlank(sys)) {
 			return sys;
 		} else {
 			return checkNotBlank(System.getenv(key), key);
 		}
 	}
 
 	private static void runGroovy(String hostname) {
 		File cli = getCanonicalFile("/var/lib/tomcat7/webapps/ROOT/WEB-INF/jenkins-cli.jar");
 		String groovy = LocationUtils.toString("classpath:org/kuali/common/kuali-devops/ubuntu/12.04/jenkins/scmTriggerThreadCleaner.groovy", UTF8);
 		File groovyFile = getCanonicalFile("./target/jenkins/interrupt.groovy");
 		try {
 			write(groovyFile, groovy);
 		} catch (IOException e) {
 			throw illegalState(e);
 		}
 		String pem = "/home/tomcat7/.ssh/jcaddel.pem";
 		List<String> args = asList("-jar", cli.getPath(), "-s", "https://" + hostname + "/", "-noCertificateCheck", "-i", pem, "groovy", groovyFile.getPath());
 		logger.info(format("cleanup scm polling threads"));
 		logger.info(format("executing groovy -> %s", groovyFile));
 		CLI.execute("java", args);
 		logger.info(format("groovy executed  -> %s", groovyFile));
 	}
 
 	private static void connectNode(String hostname, JenkinsSlave slave) {
 		logger.info(format("connect node -> %s", slave.getDisplayName()));
 		List<String> args = asList("-p", "45358", format("jcaddel@%s", hostname), "connect-node", '"' + slave.getDisplayName() + '"', "-f");
 		CLI.execute("ssh", args);
 	}
 
 	private static void stopJob(String hostname, JenkinsSlave slave) {
 		JenkinsCredentials creds = getJcaddel();
 		String username = creds.getUsername();
 		String password = creds.getApiToken();
 		String url = format("https://%s:%s@%s/computer/%s/executors/0/stop", username, password, hostname, urlencode(slave.getDisplayName()));
 		logger.info(format("stop job -> %s", slave.getDisplayName()));
 		CLI.execute("curl", asList("--fail", "--silent", "--insecure", "--request", "POST", url));
 	}
 
 	private List<JenkinsSlave> getJenkinsSlaves(List<JenkinsNode> slaves) {
 		String region = getRegion();
 		AWSCredentials creds = getFoundation();
 		EC2Service ec2 = new DefaultEC2Service(creds, region);
 		List<String> instanceIds = newArrayList();
 		for (JenkinsNode slave : slaves) {
 			String instanceId = substringBetween(slave.getDisplayName(), "(", ")");
 			instanceIds.add(instanceId);
 		}
 		List<Instance> instances = ec2.getInstances(instanceIds);
 		Map<String, Instance> map = newHashMap();
 		for (Instance instance : instances) {
 			map.put(instance.getInstanceId(), instance);
 		}
 		List<JenkinsSlave> list = newArrayList();
 		long now = currentTimeMillis();
 		for (JenkinsNode slave : slaves) {
 			String instanceId = substringBetween(slave.getDisplayName(), "(", ")");
 			Instance instance = checkNotNull(map.get(instanceId), "instanceId");
 			boolean jenkinsOnline = !slave.isOffline();
 			boolean amazonOnline = ec2.isOnline(instanceId);
 			long age = now - instance.getLaunchTime().getTime();
 			String displayName = slave.getDisplayName();
 			JenkinsSlave element = JenkinsSlave.builder().withDisplayName(displayName).withAge(age).withAmazonOnline(amazonOnline).withJenkinsOnline(jenkinsOnline)
 					.withInstanceId(instanceId).build();
 			list.add(element);
 		}
 		return list;
 	}
 
 	private EmailService getEmailService() {
 		AWSCredentials creds = getRice();
 		Region region = Region.getRegion(Regions.US_EAST_1);
 		return DefaultEmailService.builder().withCredentials(creds).withRegion(region).build();
 	}
 
 	private String getRegion() {
 		String stack = getProperty("stack", "test");
 		if (stack.equals("prod")) {
 			return "us-east-1";
 		} else {
 			return "us-west-1";
 		}
 	}
 
 	private String getHostname() {
 		String stack = getProperty("stack", "test");
 		if (stack.equals("prod")) {
 			return "ci.kuali.org";
 		} else {
 			return "testci.kuali.org";
 		}
 	}
 
 	private static JenkinsMaster getJenkinsMaster(String hostname) {
 		File file = deleteCreate(getCanonicalFile("./target/jenkins/master.json"));
 		String url = "https://" + hostname + "/computer/api/json";
 		logger.info(format("query  -> %s", url));
 		logger.info(format("create -> %s", file));
 		CLI.execute("curl", asList("--silent", "--fail", "--insecure", "--output", file.getPath(), url));
 		String json = readFileToString(file);
 		ObjectMapper mapper = newDefaultObjectMapper();
 		mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
 		JacksonContext context = JacksonContext.builder().withMapper(mapper).build();
 		JsonService service = new JacksonJsonService(context);
 		return service.readString(json, JenkinsMaster.class);
 	}
 
 	private static String readFileToString(File file) {
 		try {
 			return FileUtils.readFileToString(file);
 		} catch (IOException e) {
 			throw illegalState(e);
 		}
 	}
 
 	private static File deleteCreate(File file) {
 		try {
 			if (file.exists()) {
 				forceDelete(file);
 			}
 			touch(file);
 			return file;
 		} catch (IOException e) {
 			throw illegalState(e);
 		}
 	}
 
 	private static JenkinsCredentials getJcaddel() {
 		Encryptor enc = getDefaultEncryptor();
 		String username = enc.decrypt("U2FsdGVkX182GPpXybAxPqEuhrNntHCXyb1MkleT3oU=");
 		String apiToken = enc.decrypt("U2FsdGVkX1+p+/EH9W8R19URgB25+aCzFeycX+T31gBFeRXkOvtyDAMNbiM6QC5/tCqHbegwFriKuU57pc65mA==");
 		return JenkinsCredentials.builder().withUsername(username).withApiToken(apiToken).build();
 	}
 
 	private static AWSCredentials getFoundation() {
 		Encryptor enc = getDefaultEncryptor();
 		String accessKey = enc.decrypt("U2FsdGVkX19A2e6dN/ipVfb/9n0DROCPIrLK6H8PvvPmt0h6cBqccGaJW0NSoX3S");
 		String secretKey = enc.decrypt("U2FsdGVkX19Y9SZ5GAU82/X5Z0xZdeQf7DFuVDW07R9lfyHK4VaOj5R7pviRBKmIyn7jrVT2lv8Edeu7098k1A==");
 		return new ImmutableAWSCredentials(accessKey, secretKey);
 	}
 
 	private static AWSCredentials getRice() {
 		Encryptor enc = getDefaultEncryptor();
 		String accessKey = enc.decrypt("U2FsdGVkX1+Y3FZqrwTo97GtYwtrr5F7dSJnIMiLfy6pW2AbaKduY8Qi+B7KcL6y");
 		String secretKey = enc.decrypt("U2FsdGVkX18coWWxeff1GdsD4bWbCBuWICgftVFVZbCFH3lboRLz5fWe20alukalx94q51rATj83xEeNbVlKCg==");
 		return new ImmutableAWSCredentials(accessKey, secretKey);
 	}
 
 	private enum BuildSlavePredicate implements Predicate<JenkinsNode> {
 		INSTANCE;
 
 		@Override
 		public boolean apply(JenkinsNode node) {
 			return !"master".equals(node.getDisplayName());
 		}
 
 	}
 
 	/**
 	 * Turned off the minimum age check since Jenkins always thinks the slave is online quicker than the isOnline() method from EC2Service does
 	 */
 	private enum BadSlavePredicate implements Predicate<JenkinsSlave> {
 		INSTANCE;
 
 		@Override
 		public boolean apply(JenkinsSlave slave) {
 
 			// Minimum age of 30 minutes is required before we'll act
 			// long minAge = getMillis(getProperty("ec2.minAge", "30m"));
 
 			// If Amazon thinks the node is online, but Jenkins thinks the node is offline, we have issues
 			boolean uhoh = slave.isAmazonOnline() && !slave.isJenkinsOnline();
 
 			// Make sure the slave was launched at least 30 minutes ago before we do anything to it
 			// boolean oldEnough = slave.getAge() > minAge;
 
 			// Only return true for slaves that are at least 30 minutes old, and where Amazon thinks it is online, but Jenkins thinks it's offline
 			return uhoh; // && oldEnough;
 		}
 
 	}
 
 	// curl can't handle the plus symbol, it requires %20
 	private static String urlencode(String s) {
 		return replace(encodeUTF8(s), "+", "%20");
 	}
 
 }