Source code

001/**
002 * Copyright 2005-2016 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.kuali.rice.web.health;
017
018import com.codahale.metrics.*;
019import com.codahale.metrics.health.HealthCheck;
020import com.codahale.metrics.health.HealthCheckRegistry;
021import com.codahale.metrics.jvm.*;
022import org.codehaus.jackson.map.ObjectMapper;
023import org.kuali.rice.core.api.config.property.ConfigContext;
024import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader;
025import org.kuali.rice.core.api.util.RiceConstants;
026import org.kuali.rice.core.framework.persistence.platform.DatabasePlatform;
027
028import javax.servlet.ServletException;
029import javax.servlet.http.HttpServlet;
030import javax.servlet.http.HttpServletRequest;
031import javax.servlet.http.HttpServletResponse;
032import javax.sql.DataSource;
033import java.io.IOException;
034import java.lang.management.ManagementFactory;
035import java.lang.management.RuntimeMXBean;
036import java.util.Map;
037
038/**
039 * Implements an endpoint for providing health information for a Kuali Rice server.
040 *
041 * @author Eric Westfall
042 */
043public class HealthServlet extends HttpServlet {
044    
045    private MetricRegistry metricRegistry;
046    private HealthCheckRegistry healthCheckRegistry;
047    private Config config;
048
049    @Override
050    public void init() throws ServletException {
051        this.metricRegistry = new MetricRegistry();
052        this.healthCheckRegistry = new HealthCheckRegistry();
053        this.config = new Config();
054
055        monitorMemoryUsage();
056        monitorThreads();
057        monitorGarbageCollection();
058        monitorBufferPools();
059        monitorClassLoading();
060        monitorFileDescriptors();
061        monitorRuntime();
062        monitorDataSources();
063    }
064
065    @Override
066    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
067        HealthStatus status = checkHealth();
068
069        String includeDetail = req.getParameter("detail");
070        if ("true".equals(includeDetail)) {
071            if (status.isOk()) {
072                resp.setStatus(200);
073            } else {
074                resp.setStatus(503);
075            }
076
077            ObjectMapper mapper = new ObjectMapper();
078            resp.setContentType("application/json");
079            mapper.writeValue(resp.getOutputStream(), status);
080
081        } else {
082            if (status.isOk()) {
083                resp.setStatus(204);
084            } else {
085                resp.setStatus(503);
086            }
087        }
088        resp.getOutputStream().flush();
089    }
090
091    @SuppressWarnings("unchecked")
092    private void monitorMemoryUsage() {
093        // registry memory metrics, we are going to rename this slightly using our format given the format required
094        // by the health detail specification
095        MemoryUsageGaugeSet gaugeSet = new MemoryUsageGaugeSet();
096        Map<String, Metric> metrics = gaugeSet.getMetrics();
097        for (String metricName : metrics.keySet()) {
098            this.metricRegistry.register("memory:" + metricName, metrics.get(metricName));
099        }
100
101        Gauge<Double> heapUsage = this.metricRegistry.getGauges().get("memory:heap.usage");
102        Gauge<Long> heapMaxMemory = this.metricRegistry.getGauges().get("memory:heap.max");
103        if (heapMaxMemory.getValue() != -1) {
104            this.healthCheckRegistry.register("memory:heap.usage", new MemoryUsageHealthCheck(heapUsage, config.heapMemoryUsageThreshold()));
105        }
106
107        Gauge<Double> nonHeapUsage = this.metricRegistry.getGauges().get("memory:non-heap.usage");
108        Gauge<Long> nonHeapMaxMemory = this.metricRegistry.getGauges().get("memory:non-heap.max");
109        if (nonHeapMaxMemory.getValue() != -1) {
110            this.healthCheckRegistry.register("memory:non-heap.usage", new MemoryUsageHealthCheck(nonHeapUsage, config.nonHeapMemoryUsageThreshold()));
111        }
112
113        Gauge<Long> totalUsedMemory = this.metricRegistry.getGauges().get("memory:total.used");
114        Gauge<Long> totalMaxMemory = this.metricRegistry.getGauges().get("memory:total.max");
115        if (totalMaxMemory.getValue() != -1) {
116            MemoryUsageRatio totalMemoryRatio = new MemoryUsageRatio(totalUsedMemory, totalMaxMemory);
117            this.metricRegistry.register("memory:total.usage", totalMemoryRatio);
118            this.healthCheckRegistry.register("memory:total.usage", new MemoryUsageHealthCheck(totalMemoryRatio, config.totalMemoryUsageThreshold()));
119        }
120    }
121
122    @SuppressWarnings("unchecked")
123    private void monitorThreads() {
124        ThreadStatesGaugeSet gaugeSet = new ThreadStatesGaugeSet();
125        Map<String, Metric> metrics = gaugeSet.getMetrics();
126        for (String name : metrics.keySet()) {
127            this.metricRegistry.register("thread:" + name, metrics.get(name));
128        }
129
130        // register health check for deadlock count
131        String deadlockCountName = "thread:deadlock.count";
132        final Gauge<Integer> deadlockCount = this.metricRegistry.getGauges().get(deadlockCountName);
133        this.healthCheckRegistry.register("thread:deadlock.count", new HealthCheck() {
134            @Override
135            protected Result check() throws Exception {
136                int numDeadlocks = deadlockCount.getValue();
137                if (numDeadlocks >= config.deadlockThreshold()) {
138                    return Result.unhealthy("There are " + numDeadlocks + " deadlocked threads which is greater than or equal to the threshold of " + config.deadlockThreshold());
139                }
140                return Result.healthy();
141            }
142        });
143    }
144
145    private void monitorGarbageCollection() {
146        GarbageCollectorMetricSet metricSet = new GarbageCollectorMetricSet();
147        Map<String, Metric> metrics = metricSet.getMetrics();
148        for (String name : metrics.keySet()) {
149            this.metricRegistry.register("garbage-collector:" + name, metrics.get(name));
150        }
151    }
152
153    private void monitorBufferPools() {
154        BufferPoolMetricSet metricSet = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer());
155        Map<String, Metric> metrics = metricSet.getMetrics();
156        for (String name : metrics.keySet()) {
157            this.metricRegistry.register("buffer-pool:" + name, metrics.get(name));
158        }
159
160    }
161
162    private void monitorClassLoading() {
163        ClassLoadingGaugeSet metricSet = new ClassLoadingGaugeSet();
164        Map<String, Metric> metrics = metricSet.getMetrics();
165        for (String name : metrics.keySet()) {
166            this.metricRegistry.register("classloader:" + name, metrics.get(name));
167        }
168    }
169
170    private void monitorFileDescriptors() {
171        final FileDescriptorRatioGauge gauge = new FileDescriptorRatioGauge();
172        String name = "file-descriptor:usage";
173        this.metricRegistry.register(name, gauge);
174        this.healthCheckRegistry.register(name, new HealthCheck() {
175            @Override
176            protected Result check() throws Exception {
177                double value = gauge.getValue();
178                if (value >= config.fileDescriptorUsageThreshold()) {
179                    return Result.unhealthy("File descriptor usage ratio of " + value + " was greater than or equal to threshold of " + config.fileDescriptorUsageThreshold());
180                }
181                return Result.healthy();
182            }
183        });
184    }
185
186    private void monitorRuntime() {
187        final RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
188        this.metricRegistry.register("runtime:uptime", new Gauge<Long>() {
189            @Override
190            public Long getValue() {
191                return runtime.getUptime();
192            }
193        });
194    }
195
196    private void monitorDataSources() {
197        DataSource dataSource = (DataSource) ConfigContext.getCurrentContextConfig().getObject(RiceConstants.DATASOURCE_OBJ);
198        DataSource nonTransactionalDataSource = (DataSource) ConfigContext.getCurrentContextConfig().getObject(RiceConstants.NON_TRANSACTIONAL_DATASOURCE_OBJ);
199        DataSource serverDataSource = (DataSource) ConfigContext.getCurrentContextConfig().getObject(RiceConstants.SERVER_DATASOURCE_OBJ);
200        DatabasePlatform databasePlatform = GlobalResourceLoader.getService(RiceConstants.DB_PLATFORM);
201        monitorDataSource("database.primary:", dataSource, databasePlatform, config.primaryConnectionPoolUsageThreshold());
202        monitorDataSource("database.non-transactional:", nonTransactionalDataSource, databasePlatform, config.nonTransactionalConnectionPoolUsageThreshold());
203        monitorDataSource("database.server:", serverDataSource, databasePlatform, config.serverConnectionPoolUsageThreshold());
204    }
205
206    @SuppressWarnings("unchecked")
207    private void monitorDataSource(String namePrefix, DataSource dataSource, DatabasePlatform databasePlatform, double threshold) {
208        if (databasePlatform != null && dataSource != null) {
209            // register connection metric
210            String name = namePrefix + "connected";
211            DatabaseConnectionHealthGauge healthGauge = new DatabaseConnectionHealthGauge(dataSource, databasePlatform);
212            this.metricRegistry.register(name, healthGauge);
213            this.healthCheckRegistry.register(name, healthGauge);
214
215            // register pool metrics
216            String poolUsageName = namePrefix + DatabaseConnectionPoolMetricSet.USAGE;
217            DatabaseConnectionPoolMetricSet poolMetrics = new DatabaseConnectionPoolMetricSet(namePrefix, dataSource);
218            this.metricRegistry.registerAll(poolMetrics);
219            Gauge<Double> poolUsage = this.metricRegistry.getGauges().get(poolUsageName);
220            if (poolUsage != null) {
221                this.healthCheckRegistry.register(poolUsageName, new DatabaseConnectionPoolHealthCheck(poolUsage, threshold));
222            }
223        }
224    }
225
226    private HealthStatus checkHealth() {
227        HealthStatus status = new HealthStatus();
228        runHealthChecks(status);
229        reportMetrics(status);
230        return status;
231    }
232
233    private void runHealthChecks(HealthStatus status) {
234        Map<String, HealthCheck.Result> results = this.healthCheckRegistry.runHealthChecks();
235        for (String name : results.keySet()) {
236            HealthCheck.Result result = results.get(name);
237            if (!result.isHealthy()) {
238                status.setStatusCode(HealthStatus.FAILED);
239                status.appendMessage(name, result.getMessage());
240            }
241        }
242    }
243
244    private void reportMetrics(HealthStatus status) {
245        reportGauges(this.metricRegistry.getGauges(), status);
246        reportCounters(metricRegistry.getCounters(), status);
247        reportHistograms(metricRegistry.getHistograms(), status);
248        reportMeters(metricRegistry.getMeters(), status);
249        reportTimers(metricRegistry.getTimers(), status);
250    }
251
252    private void reportGauges(Map<String, Gauge> gaugues, HealthStatus status) {
253        for (String name : gaugues.keySet()) {
254            Gauge gauge = gaugues.get(name);
255            status.getMetrics().add(new HealthMetric(name, gauge.getValue()));
256        }
257    }
258
259    private void reportCounters(Map<String, Counter> counters, HealthStatus status) {
260        for (String name : counters.keySet()) {
261            Counter counter = counters.get(name);
262            status.getMetrics().add(new HealthMetric(name, counter.getCount()));
263        }
264    }
265
266    private void reportHistograms(Map<String, Histogram> histograms, HealthStatus status) {
267        for (String name : histograms.keySet()) {
268            Histogram histogram = histograms.get(name);
269            status.getMetrics().add(new HealthMetric(name, histogram.getCount()));
270        }
271    }
272
273    private void reportMeters(Map<String, Meter> meters, HealthStatus status) {
274        for (String name : meters.keySet()) {
275            Meter meter = meters.get(name);
276            status.getMetrics().add(new HealthMetric(name, meter.getCount()));
277        }
278    }
279
280    private void reportTimers(Map<String, Timer> timers, HealthStatus status) {
281        for (String name : timers.keySet()) {
282            Timer timer = timers.get(name);
283            status.getMetrics().add(new HealthMetric(name, timer.getCount()));
284        }
285    }
286    
287    public static final class Config {
288
289        public static final String HEAP_MEMORY_THRESHOLD_PROPERTY = "rice.health.memory.heap.usageThreshold";
290        public static final String NON_HEAP_MEMORY_THRESHOLD_PROPERTY = "rice.health.memory.nonHeap.usageThreshold";
291        public static final String TOTAL_MEMORY_THRESHOLD_PROPERTY = "rice.health.memory.total.usageThreshold";
292        public static final String DEADLOCK_THRESHOLD_PROPERTY = "rice.health.thread.deadlockThreshold";
293        public static final String FILE_DESCRIPTOR_THRESHOLD_PROPERTY = "rice.health.fileDescriptor.usageThreshold";
294        public static final String PRIMARY_POOL_USAGE_THRESHOLD_PROPERTY = "rice.health.database.primary.connectionPoolUsageThreshold";
295        public static final String NON_TRANSACTIONAL_POOL_USAGE_THRESHOLD_PROPERTY = "rice.health.database.nonTransactional.connectionPoolUsageThreshold";
296        public static final String SERVER_POOL_USAGE_THRESHOLD_PROPERTY = "rice.health.database.server.connectionPoolUsageThreshold";
297
298        private static final double HEAP_MEMORY_THRESHOLD_DEFAULT = 0.95;
299        private static final double NON_HEAP_MEMORY_THRESHOLD_DEFAULT = 0.95;
300        private static final double TOTAL_MEMORY_THRESHOLD_DEFAULT = 0.95;
301        private static final int DEADLOCK_THRESHOLD_DEFAULT = 1;
302        private static final double FILE_DESCRIPTOR_THRESHOLD_DEFAULT = 0.95;
303        private static final double POOL_USAGE_THRESHOLD_DEFAULT = 1.0;
304
305
306        double heapMemoryUsageThreshold() {
307            return getDouble(HEAP_MEMORY_THRESHOLD_PROPERTY, HEAP_MEMORY_THRESHOLD_DEFAULT);
308        }
309
310        double nonHeapMemoryUsageThreshold() {
311            return getDouble(NON_HEAP_MEMORY_THRESHOLD_PROPERTY, NON_HEAP_MEMORY_THRESHOLD_DEFAULT);
312        }
313
314        double totalMemoryUsageThreshold() {
315            return getDouble(TOTAL_MEMORY_THRESHOLD_PROPERTY, TOTAL_MEMORY_THRESHOLD_DEFAULT);
316        }
317
318        int deadlockThreshold() {
319            return getInt(DEADLOCK_THRESHOLD_PROPERTY, DEADLOCK_THRESHOLD_DEFAULT);
320        }
321
322        double fileDescriptorUsageThreshold() {
323            return getDouble(FILE_DESCRIPTOR_THRESHOLD_PROPERTY, FILE_DESCRIPTOR_THRESHOLD_DEFAULT);
324        }
325
326        double primaryConnectionPoolUsageThreshold() {
327            return getDouble(PRIMARY_POOL_USAGE_THRESHOLD_PROPERTY, POOL_USAGE_THRESHOLD_DEFAULT);
328        }
329
330        double nonTransactionalConnectionPoolUsageThreshold() {
331            return getDouble(NON_TRANSACTIONAL_POOL_USAGE_THRESHOLD_PROPERTY, POOL_USAGE_THRESHOLD_DEFAULT);
332        }
333
334        double serverConnectionPoolUsageThreshold() {
335            return getDouble(SERVER_POOL_USAGE_THRESHOLD_PROPERTY, POOL_USAGE_THRESHOLD_DEFAULT);
336        }
337
338        private double getDouble(String propertyName, double defaultValue) {
339            String propertyValue = ConfigContext.getCurrentContextConfig().getProperty(propertyName);
340            if (propertyValue != null) {
341                return Double.parseDouble(propertyValue);
342            }
343            return defaultValue;
344        }
345
346        private int getInt(String propertyName, int defaultValue) {
347            String propertyValue = ConfigContext.getCurrentContextConfig().getProperty(propertyName);
348            if (propertyValue != null) {
349                return Integer.parseInt(propertyValue);
350            }
351            return defaultValue;
352        }
353        
354    }
355
356}