Expose metrics for Prometheus Monitoring Framework

.classpath

@@ -80,6 +80,12 @@
 	<classpathentry kind="lib" path="ext/pf4j-0.9.0.jar" sourcepath="ext/src/pf4j-0.9.0.jar" />
 	<classpathentry kind="lib" path="ext/tika-core-1.5.jar" sourcepath="ext/src/tika-core-1.5.jar" />
 	<classpathentry kind="lib" path="ext/jsoup-1.7.3.jar" sourcepath="ext/src/jsoup-1.7.3.jar" />
+	<classpathentry kind="lib" path="ext/simpleclient-0.0.21.jar" sourcepath="ext/src/simpleclient-0.0.21.jar" />
+	<classpathentry kind="lib" path="ext/simpleclient_hotspot-0.0.21.jar" sourcepath="ext/src/simpleclient_hotspot-0.0.21.jar" />
+	<classpathentry kind="lib" path="ext/simpleclient_log4j-0.0.21.jar" sourcepath="ext/src/simpleclient_log4j-0.0.21.jar" />
+	<classpathentry kind="lib" path="ext/simpleclient_guava-0.0.21.jar" sourcepath="ext/src/simpleclient_guava-0.0.21.jar" />
+	<classpathentry kind="lib" path="ext/simpleclient_servlet-0.0.21.jar" sourcepath="ext/src/simpleclient_servlet-0.0.21.jar" />
+	<classpathentry kind="lib" path="ext/simpleclient_common-0.0.21.jar" sourcepath="ext/src/simpleclient_common-0.0.21.jar" />
 	<classpathentry kind="lib" path="ext/junit-4.12.jar" sourcepath="ext/src/junit-4.12.jar" />
 	<classpathentry kind="lib" path="ext/hamcrest-core-1.3.jar" sourcepath="ext/src/hamcrest-core-1.3.jar" />
 	<classpathentry kind="lib" path="ext/selenium-java-2.28.0.jar" sourcepath="ext/src/selenium-java-2.28.0.jar" />

build.moxie

@@ -181,6 +181,11 @@ dependencies:
 - compile 'ro.fortsoft.pf4j:pf4j:0.9.0' :war
 - compile 'org.apache.tika:tika-core:1.5' :war
 - compile 'org.jsoup:jsoup:1.7.3' :war
+- compile 'io.prometheus:simpleclient:0.0.21' :war
+- compile 'io.prometheus:simpleclient_hotspot:0.0.21' :war
+- compile 'io.prometheus:simpleclient_log4j:0.0.21' :war
+- compile 'io.prometheus:simpleclient_guava:0.0.21' :war
+- compile 'io.prometheus:simpleclient_servlet:0.0.21' :war
 - test 'junit:junit:4.12'
 # Dependencies for Selenium web page testing
 - test 'org.seleniumhq.selenium:selenium-java:${selenium.version}' @jar

gitblit.iml

@@ -824,6 +824,72 @@
         </SOURCES>
       </library>
     </orderEntry>
+    <orderEntry type="module-library">
+      <library name="simpleclient-0.0.21.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/simpleclient-0.0.21.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/simpleclient-0.0.21.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="simpleclient_hotspot-0.0.21.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/simpleclient_hotspot-0.0.21.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/simpleclient_hotspot-0.0.21.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="simpleclient_log4j-0.0.21.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/simpleclient_log4j-0.0.21.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/simpleclient_log4j-0.0.21.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="simpleclient_guava-0.0.21.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/simpleclient_guava-0.0.21.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/simpleclient_guava-0.0.21.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="simpleclient_servlet-0.0.21.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/simpleclient_servlet-0.0.21.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/simpleclient_servlet-0.0.21.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="simpleclient_common-0.0.21.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/simpleclient_common-0.0.21.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/simpleclient_common-0.0.21.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
     <orderEntry type="module-library" scope="TEST">
       <library name="junit-4.12.jar">
         <CLASSES>

src/main/java/com/gitblit/guice/WebModule.java

@@ -30,6 +30,7 @@
 import com.gitblit.servlet.GitFilter;
 import com.gitblit.servlet.GitServlet;
 import com.gitblit.servlet.LogoServlet;
+import com.gitblit.servlet.MetricsFilter;
 import com.gitblit.servlet.PagesFilter;
 import com.gitblit.servlet.PagesServlet;
 import com.gitblit.servlet.ProxyFilter;
@@ -43,8 +44,13 @@
 import com.gitblit.servlet.SyndicationFilter;
 import com.gitblit.servlet.SyndicationServlet;
 import com.gitblit.wicket.GitblitWicketFilter;
+
 import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Scopes;
 import com.google.inject.servlet.ServletModule;
+import io.prometheus.client.exporter.MetricsServlet;
+import io.prometheus.client.hotspot.DefaultExports;
 
 /**
  * Defines all the web servlets & filters.
@@ -79,6 +85,12 @@ protected void configureServlets() {
 		serve("/robots.txt").with(RobotsTxtServlet.class);
 		serve("/logo.png").with(LogoServlet.class);
 
+		// Prometheus
+		bind(MetricsServlet.class).in(Scopes.SINGLETON);
+		bind(MetricsFilter.class).in(Scopes.SINGLETON);
+		serve("/prometheus").with(MetricsServlet.class);
+		DefaultExports.initialize();
+
 		/* Prevent accidental access to 'resources' such as GitBlit java classes
 		 *
 		 * In the GO setup the JAR containing the application and the WAR injected
@@ -91,8 +103,13 @@ protected void configureServlets() {
 		serve(fuzzy("/com/")).with(AccessDeniedServlet.class);
 
 		// global filters
-		filter(ALL).through(ProxyFilter.class);
-		filter(ALL).through(EnforceAuthenticationFilter.class);
+        filter(ALL).through(MetricsFilter.class,
+                ImmutableMap.of(
+						MetricsFilter.PARAM_DURATION_HIST_BUCKET_CONFIG, "0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10",
+                        MetricsFilter.PARAM_PATH_MAX_DEPTH, "16"
+				));
+        filter(ALL).through(ProxyFilter.class);
+        filter(ALL).through(EnforceAuthenticationFilter.class);
 
 		// security filters
 		filter(fuzzy(Constants.R_PATH), fuzzy(Constants.GIT_PATH)).through(GitFilter.class);

src/main/java/com/gitblit/service/GarbageCollectorService.java

@@ -23,6 +23,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import io.prometheus.client.Counter;
 import org.eclipse.jgit.api.GarbageCollectCommand;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.lib.Repository;
@@ -35,6 +36,8 @@
 import com.gitblit.models.RepositoryModel;
 import com.gitblit.utils.FileUtils;
 
+import static com.gitblit.service.PrometheusMetrics.GIT_GARBAGE_COLLECTS_TOTAL;
+
 /**
  * The Garbage Collector Service handles periodic garbage collection in repositories.
  *
@@ -129,6 +132,9 @@ public void close() {
 		forceClose.set(true);
 	}
 
+	private final Counter garbageCollectsTotal = Counter.build()
+			.name(GIT_GARBAGE_COLLECTS_TOTAL).help(GIT_GARBAGE_COLLECTS_TOTAL).register();
+
 	@Override
 	public void run() {
 		if (!isReady()) {
@@ -200,7 +206,7 @@ public void run() {
 
 					// do the deed
 					gc.call();
-
+					garbageCollectsTotal.inc();
 					garbageCollected = true;
 				}
 			} catch (Exception e) {

src/main/java/com/gitblit/service/LdapSyncService.java

@@ -17,20 +17,25 @@
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import io.prometheus.client.Histogram;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.gitblit.IStoredSettings;
 import com.gitblit.Keys;
 import com.gitblit.auth.LdapAuthProvider;
 
+import static com.gitblit.service.PrometheusMetrics.LDAP_SYNC_LATENCY_SECONDS;
+
 /**
  * @author Alfred Schmid
  *
  */
 public final class LdapSyncService implements Runnable {
 
 	private final Logger logger = LoggerFactory.getLogger(LdapSyncService.class);
+	private final Histogram ldapSyncLatency = Histogram.build().name(LDAP_SYNC_LATENCY_SECONDS).
+            help(LDAP_SYNC_LATENCY_SECONDS).register();
 
 	private final IStoredSettings settings;
 
@@ -51,12 +56,14 @@ public LdapSyncService(IStoredSettings settings, LdapAuthProvider ldapAuthProvid
 	public void run() {
 		logger.info("Starting user and group sync with ldap service");
 		if (!running.getAndSet(true)) {
+			Histogram.Timer requestTimer = ldapSyncLatency.startTimer();
 			try {
 				ldapAuthProvider.sync();
 			} catch (Exception e) {
 				logger.error("Failed to synchronize with ldap", e);
 			} finally {
 				running.getAndSet(false);
+				requestTimer.observeDuration();
 			}
 		}
 		logger.info("Finished user and group sync with ldap service");

src/main/java/com/gitblit/service/PrometheusMetrics.java

@@ -0,0 +1,10 @@
+package com.gitblit.service;
+
+class PrometheusMetrics {
+
+    /** Number of garbage collects  */
+    static final String GIT_GARBAGE_COLLECTS_TOTAL = "gitblit_garbage_collects_total";
+
+    /** Ldap Sync Latency in second */
+    static final String LDAP_SYNC_LATENCY_SECONDS = "gitblit_ldap_sync_latency_seconds";
+}

src/main/java/com/gitblit/servlet/MetricsFilter.java

@@ -0,0 +1,134 @@
+package com.gitblit.servlet;
+
+import io.prometheus.client.Counter;
+import io.prometheus.client.Histogram;
+
+import javax.servlet.*;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ */
+public class MetricsFilter implements Filter {
+    public static final String PARAM_PATH_MAX_DEPTH = "max-path-depth";
+    public static final String PARAM_DURATION_HIST_BUCKET_CONFIG = "request-duration-histogram-buckets";
+
+    private Histogram httpRequestDuration = null;
+    private Counter requests = null;
+
+    // Package-level for testing purposes.
+    int pathDepth = 1;
+    private double[] buckets = null;
+
+    public MetricsFilter() {
+    }
+
+    public MetricsFilter(
+            Integer maxPathDepth,
+            double[] buckets
+    ) throws ServletException {
+        this.buckets = buckets;
+        if (maxPathDepth != null) {
+            this.pathDepth = maxPathDepth;
+        }
+        this.init(null);
+    }
+
+    private boolean isEmpty(String s) {
+        return s == null || s.length() == 0;
+    }
+
+    @Override
+    public void init(FilterConfig filterConfig) throws ServletException {
+
+        Histogram.Builder httpRequestDurationBuilder = Histogram.build()
+                .name("http_request_duration_seconds")
+                .labelNames("path", "method")
+                .help("The time taken fulfilling servlet requests");
+
+        Counter.Builder requestsBuilder = Counter.build()
+                .name("http_requests_total")
+                .help("Total requests.")
+                .labelNames("path", "method", "status");
+
+        if (filterConfig == null && isEmpty("http_request_duration")) {
+            throw new ServletException("No configuration object provided, and no metricName passed via constructor");
+        }
+
+        if (filterConfig != null) {
+
+            // Allow overriding of the path "depth" to track
+            if (!isEmpty(filterConfig.getInitParameter(PARAM_PATH_MAX_DEPTH))) {
+                pathDepth = Integer.valueOf(filterConfig.getInitParameter(PARAM_PATH_MAX_DEPTH));
+            }
+
+            // Allow users to override the default bucket configuration
+            if (!isEmpty(filterConfig.getInitParameter(PARAM_DURATION_HIST_BUCKET_CONFIG))) {
+                String[] bucketParams = filterConfig.getInitParameter(PARAM_DURATION_HIST_BUCKET_CONFIG).split(",");
+                buckets = new double[bucketParams.length];
+
+                for (int i = 0; i < bucketParams.length; i++) {
+                    buckets[i] = Double.parseDouble(bucketParams[i]);
+                }
+            }
+        }
+
+        requests = requestsBuilder.register();
+
+        if (buckets != null) {
+            httpRequestDurationBuilder = httpRequestDurationBuilder.buckets(buckets);
+        }
+
+        httpRequestDuration = httpRequestDurationBuilder.register();
+    }
+
+    @Override
+    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+        if (!(servletRequest instanceof HttpServletRequest)) {
+            filterChain.doFilter(servletRequest, servletResponse);
+            return;
+        }
+
+        HttpServletRequest request = (HttpServletRequest) servletRequest;
+        HttpServletResponse response = (HttpServletResponse) servletResponse;
+
+        String path = request.getRequestURI();
+        String normalizedPath = extractPathFrom(path, pathDepth);
+
+        Histogram.Timer timer = httpRequestDuration
+                .labels(normalizedPath, request.getMethod())
+                .startTimer();
+        try {
+            filterChain.doFilter(servletRequest, servletResponse);
+            requests.labels(normalizedPath, request.getMethod().toUpperCase(), String.valueOf(response.getStatus())).inc();
+        } finally {
+            timer.observeDuration();
+        }
+    }
+
+    public String extractPathFrom(String requestUri, int maxPathDepth) {
+        if (maxPathDepth < 0 || requestUri == null) {
+            throw new IllegalArgumentException("Path depth has to >= 0");
+        }
+
+        int count = 0;
+        int pathPosition = -1;
+        do {
+            int lastPathPosition = pathPosition;
+            pathPosition = requestUri.indexOf("/", pathPosition + 1);
+            if (count > maxPathDepth || pathPosition < 0) {
+                return requestUri.substring(0, lastPathPosition + 1);
+            }
+            count++;
+        } while (count <= maxPathDepth);
+
+        return requestUri.substring(0, pathPosition + 1);
+    }
+
+    @Override
+    public void destroy() {
+    }
+
+}
+

src/main/java/log4j.properties

@@ -17,7 +17,7 @@
 #      FATAL, ERROR, WARN, INFO, DEBUG
 #
 #------------------------------------------------------------------------------
-log4j.rootCategory=INFO, S
+log4j.rootCategory=INFO, S, METRICS
 
 #log4j.rootLogger=INFO
 #log4j.logger.org=INFO
@@ -67,3 +67,6 @@ log4j.appender.H.File = logs/gitblit.html
 log4j.appender.H.MaxFileSize = 100KB
 log4j.appender.H.Append = false
 log4j.appender.H.layout = org.apache.log4j.HTMLLayout
+
+log4j.appender.METRICS = io.prometheus.client.log4j.InstrumentedAppender
+log4j.appender.METRICS.Append = false