Add additional metrics for user/client/session count (#31)

README.md

@@ -1,6 +1,6 @@
-# Keycloak Event Metrics
+# Keycloak Metrics
 
-Provides metrics for Keycloak user/admin events. Tested on Keycloak [20-21](.github/workflows/ci.yaml#L74-L77).
+Provides metrics for Keycloak user/admin events and user/client/session count. Tested on Keycloak [20-21](.github/workflows/ci.yaml#L74-L77).
 
 [![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/kokuwaio/keycloak-event-metrics.svg?label=License)](http://www.apache.org/licenses/)
 [![Maven Central](https://img.shields.io/maven-central/v/io.kokuwa.keycloak/keycloak-event-metrics.svg?label=Maven%20Central)](https://central.sonatype.com/search?namespace=io.kokuwa.keycloak&q=keycloak-event-metrics)
@@ -14,7 +14,8 @@ Provides metrics for Keycloak user/admin events. Tested on Keycloak [20-21](.git
 * no realm specific Prometheus endpoint, only `/metrics` (from Quarkus)
 * no jvm/http metrics, this is [already](https://www.keycloak.org/server/configuration-metrics#_available_metrics) included in Keycloak
 * different metric names, can relace model ids with name (see [configuration](#kc_metrics_event_replace_ids))
-* deployed to maven central and very small (10 kb vs. 229 KB [aerogear/keycloak-metrics-spi](https://github.com/aerogear/keycloak-metrics-spi))
+* deployed to maven central and very small (15 kb vs. 151 KB [aerogear/keycloak-metrics-spi](https://github.com/aerogear/keycloak-metrics-spi))
+* gauge for active/offline sessions and user/client count
 
 ## What?
 
@@ -57,7 +58,7 @@ keycloak_event_admin_total{error="",operation="CREATE",realm="9039a0b5-e8c9-437a
 
 ### `KC_METRICS_EVENT_REPLACE_IDS`
 
-Per set to `true` (the default value) than replace model ids with names:
+Set to `true` (the default value) than replace model ids from events with names:
 
 * [RealmModel#getId()](https://github.com/keycloak/keycloak/blob/main/server-spi/src/main/java/org/keycloak/models/RealmModel.java#L82) with [RealmModel#getName()](https://github.com/keycloak/keycloak/blob/main/server-spi/src/main/java/org/keycloak/models/RealmModel.java#L84)
 
@@ -69,6 +70,45 @@ keycloak_event_user_total{client="other-client",error="",realm="other-realm",typ
 keycloak_event_user_total{client="other-client",error="invalid_user_credentials",realm="other-realm",type="LOGIN_ERROR",} 1.0
 ```
 
+### `KC_METRICS_STATS_ENABLED`
+
+Set to `true` (default is `false`) to provide metrics for user/client count per realm and session count per client. Metrics:
+
+```txt
+# HELP keycloak_users  
+# TYPE keycloak_users gauge
+keycloak_users{realm="master",} 1.0
+keycloak_users{realm="my-realm",} 2.0
+keycloak_users{realm="other-realm",} 1.0# HELP keycloak_active_user_sessions
+# TYPE keycloak_active_user_sessions gauge
+keycloak_active_user_sessions{client="admin-cli",realm="userCount_1",} 0.0
+keycloak_active_user_sessions{client="admin-cli",realm="userCount_2",} 0.0
+keycloak_active_user_sessions{client="admin-cli",realm="master",} 1.0
+# TYPE keycloak_active_client_sessions gauge
+keycloak_active_client_sessions{client="admin-cli",realm="userCount_1",} 0.0
+keycloak_active_client_sessions{client="admin-cli",realm="userCount_2",} 0.0
+keycloak_active_client_sessions{client="admin-cli",realm="master",} 0.0
+# TYPE keycloak_offline_sessions gauge
+keycloak_offline_sessions{client="admin-cli",realm="userCount_1",} 0.0
+keycloak_offline_sessions{client="admin-cli",realm="userCount_2",} 0.0
+keycloak_offline_sessions{client="admin-cli",realm="master",} 0.0
+```
+
+### `KC_METRICS_STATS_INTERVAL`
+
+If `KC_METRICS_STATS_ENABLED` is `true` this will define the interval for scraping. If not configured `PT60s` will be used.
+
+### `KC_METRICS_STATS_INFO_THRESHOLD` and `KC_METRICS_STATS_WARN_THRESHOLD`
+
+If `KC_METRICS_STATS_ENABLED` is `true` this envs will define logging if scraping takes to long. Both envs are parsed as `java.lang.Duration`.
+
+Default values:
+
+* `KC_METRICS_STATS_INFO_THRESHOLD`: 50% of `KC_METRICS_STATS_INTERVAL` = 30s
+* `KC_METRICS_STATS_WARN_THRESHOLD`: 75% of `KC_METRICS_STATS_INTERVAL` = 45s
+
+If scrapping takes less than `KC_METRICS_STATS_INFO_THRESHOLD` duration will be logged on debug level.
+
 ## Installation
 
 ### Testcontainers

src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactory.java

@@ -0,0 +1,10 @@
+package io.kokuwa.keycloak.metrics.stats;
+
+import org.keycloak.provider.ProviderFactory;
+
+/**
+ * Factory for Keycloak metrics.
+ *
+ * @author Stephan Schnabel
+ */
+public interface MetricsStatsFactory extends ProviderFactory<MetricsStatsTask> {}

src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactoryImpl.java

@@ -0,0 +1,69 @@
+package io.kokuwa.keycloak.metrics.stats;
+
+import java.time.Duration;
+import java.util.Optional;
+
+import org.jboss.logging.Logger;
+import org.keycloak.Config.Scope;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.timer.TimerProvider;
+
+import io.micrometer.core.instrument.Metrics;
+
+/**
+ * Implementation of {@link MetricsStatsFactory}.
+ *
+ * @author Stephan Schnabel
+ */
+public class MetricsStatsFactoryImpl implements MetricsStatsFactory {
+
+	private static final Logger log = Logger.getLogger(MetricsStatsFactory.class);
+
+	@Override
+	public String getId() {
+		return "default";
+	}
+
+	@Override
+	public void init(Scope config) {}
+
+	@Override
+	public void postInit(KeycloakSessionFactory factory) {
+
+		if (!"true".equals(System.getenv().get("KC_METRICS_STATS_ENABLED"))) {
+			log.infov("Keycloak stats not enabled.");
+			return;
+		}
+
+		var intervalDuration = Optional
+				.ofNullable(System.getenv("KC_METRICS_STATS_INTERVAL"))
+				.map(Duration::parse)
+				.orElse(Duration.ofSeconds(60));
+		var infoThreshold = Optional
+				.ofNullable(System.getenv("KC_METRICS_STATS_INFO_THRESHOLD"))
+				.map(Duration::parse)
+				.orElse(Duration.ofMillis(Double.valueOf(intervalDuration.toMillis() * 0.5).longValue()));
+		var warnThreshold = Optional
+				.ofNullable(System.getenv("KC_METRICS_STATS_WARN_THRESHOLD"))
+				.map(Duration::parse)
+				.orElse(Duration.ofMillis(Double.valueOf(intervalDuration.toMillis() * 0.75).longValue()));
+		log.infov("Keycloak stats enabled with interval of {0} and info/warn after {1}/{2}.",
+				intervalDuration, infoThreshold, warnThreshold);
+
+		var interval = intervalDuration.toMillis();
+		var task = new MetricsStatsTask(Metrics.globalRegistry, intervalDuration, infoThreshold, warnThreshold);
+		KeycloakModelUtils.runJobInTransaction(factory, session -> session
+				.getProvider(TimerProvider.class)
+				.schedule(() -> KeycloakModelUtils.runJobInTransaction(factory, task), interval, "metrics"));
+	}
+
+	@Override
+	public MetricsStatsTask create(KeycloakSession session) {
+		return null;
+	}
+
+	@Override
+	public void close() {}
+}

src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsSpi.java

@@ -0,0 +1,34 @@
+package io.kokuwa.keycloak.metrics.stats;
+
+import org.keycloak.provider.Provider;
+import org.keycloak.provider.ProviderFactory;
+import org.keycloak.provider.Spi;
+
+/**
+ * SPI for Keycloak metrics.
+ *
+ * @author Stephan Schnabel
+ */
+public class MetricsStatsSpi implements Spi {
+
+	@Override
+	public boolean isInternal() {
+		return false;
+	}
+
+	@Override
+	public String getName() {
+		return "metrics";
+	}
+
+	@Override
+	public Class<? extends Provider> getProviderClass() {
+		return MetricsStatsTask.class;
+	}
+
+	@Override
+	public Class<? extends ProviderFactory<? extends Provider>> getProviderFactoryClass() {
+		// this must be an interface, otherwise spi will be silenty ignored
+		return MetricsStatsFactory.class;
+	}
+}

src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsTask.java

@@ -0,0 +1,89 @@
+package io.kokuwa.keycloak.metrics.stats;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.jboss.logging.Logger;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionTask;
+import org.keycloak.provider.Provider;
+
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
+
+/**
+ * Keycloak metrics.
+ *
+ * @author Stephan Schnabel
+ */
+public class MetricsStatsTask implements Provider, KeycloakSessionTask {
+
+	private static final Logger log = Logger.getLogger(MetricsStatsTask.class);
+	private final Map<String, AtomicLong> values = new HashMap<>();
+	private final MeterRegistry registry;
+	private final Duration interval;
+	private final Duration infoThreshold;
+	private final Duration warnThreshold;
+
+	MetricsStatsTask(MeterRegistry registry, Duration interval, Duration infoThreshold, Duration warnThreshold) {
+		this.registry = registry;
+		this.interval = interval;
+		this.infoThreshold = infoThreshold;
+		this.warnThreshold = warnThreshold;
+	}
+
+	@Override
+	public void run(KeycloakSession session) {
+		log.tracev("Triggered metrics stats task.");
+		var start = Instant.now();
+
+		try {
+			scrape(session);
+		} catch (Exception e) {
+			if (e instanceof org.hibernate.exception.SQLGrammarException) {
+				log.infov("Metrics status task skipped, database not ready");
+			} else {
+				log.errorv(e, "Failed to scrape stats.");
+			}
+			return;
+		}
+
+		var duration = Duration.between(start, Instant.now());
+		if (duration.compareTo(interval) > 0) {
+			log.errorv("Finished scrapping keycloak stats in {0}, consider to increase interval", duration);
+		} else if (duration.compareTo(warnThreshold) > 0) {
+			log.warnv("Finished scrapping keycloak stats in {0}, consider to increase interval", duration);
+		} else if (duration.compareTo(infoThreshold) > 0) {
+			log.infov("Finished scrapping keycloak stats in {0}", duration);
+		} else {
+			log.debugv("Finished scrapping keycloak stats in {0}", duration);
+		}
+	}
+
+	@Override
+	public void close() {}
+
+	private void scrape(KeycloakSession session) {
+		session.realms().getRealmsStream().forEach(realm -> {
+			var tagRealm = Tag.of("realm", realm.getName());
+			gauge("keycloak_users", Set.of(tagRealm), session.users().getUsersCount(realm));
+			gauge("keycloak_clients", Set.of(tagRealm), session.clients().getClientsCount(realm));
+			var sessions = session.sessions();
+			var activeSessions = sessions.getActiveClientSessionStats(realm, false);
+			realm.getClientsStream().forEach(client -> {
+				var tags = Set.of(tagRealm, Tag.of("client", client.getClientId()));
+				gauge("keycloak_offline_sessions", tags, sessions.getOfflineSessionsCount(realm, client));
+				gauge("keycloak_active_user_sessions", tags, sessions.getActiveUserSessions(realm, client));
+				gauge("keycloak_active_client_sessions", tags, activeSessions.getOrDefault(client.getId(), 0L));
+			});
+		});
+	}
+
+	private void gauge(String name, Set<Tag> tags, long value) {
+		values.computeIfAbsent(name + tags, s -> registry.gauge(name, tags, new AtomicLong())).set(value);
+	}
+}

src/main/resources/META-INF/services/io.kokuwa.keycloak.metrics.stats.MetricsStatsFactory

@@ -0,0 +1 @@
+io.kokuwa.keycloak.metrics.stats.MetricsStatsFactoryImpl

src/main/resources/META-INF/services/org.keycloak.provider.Spi

@@ -0,0 +1 @@
+io.kokuwa.keycloak.metrics.stats.MetricsStatsSpi

src/test/java/io/kokuwa/keycloak/metrics/KeycloakIT.java

@@ -1,11 +1,14 @@
 package io.kokuwa.keycloak.metrics;
 
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.time.Instant;
 import java.util.UUID;
+import java.util.function.Supplier;
 
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
@@ -73,4 +76,41 @@ public class KeycloakIT {
 				() -> assertEquals(loginErrorBefore1 + 0, loginErrorAfter1, "login failure #1"),
 				() -> assertEquals(loginErrorBefore2 + 1, loginErrorAfter2, "login failure #2"));
 	}
+
+	@DisplayName("user count")
+	@Test
+	void userCount(KeycloakClient keycloak, Prometheus prometheus) {
+
+		var realmName1 = "userCount_1";
+		var realmName2 = "userCount_2";
+		var username = UUID.randomUUID().toString();
+
+		keycloak.createRealm(realmName1);
+		keycloak.createRealm(realmName2);
+
+		await(() -> prometheus.userCount(realmName1) == 0, prometheus, "realm 1 not found");
+		await(() -> prometheus.userCount(realmName2) == 0, prometheus, "realm 2 not found");
+
+		keycloak.createUser(realmName1, username, UUID.randomUUID().toString());
+		keycloak.createUser(realmName1, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+		keycloak.createUser(realmName1, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+		keycloak.createUser(realmName2, UUID.randomUUID().toString(), UUID.randomUUID().toString());
+
+		await(() -> prometheus.userCount(realmName1) == 3, prometheus, "realm 1 shoud have 3 users");
+		await(() -> prometheus.userCount(realmName2) == 1, prometheus, "realm 2 shoud have 1 users");
+
+		keycloak.deleteUser(realmName1, username);
+
+		await(() -> prometheus.userCount(realmName1) == 2, prometheus, "realm 1 shoud have 2 users after deletion");
+		await(() -> prometheus.userCount(realmName2) == 1, prometheus, "realm 2 shoud have 1 users");
+	}
+
+	void await(Supplier<Boolean> check, Prometheus prometheus, String message) {
+		var end = Instant.now().plusSeconds(10);
+		while (Instant.now().isBefore(end) && !check.get()) {
+			assertDoesNotThrow(() -> Thread.sleep(1000));
+			prometheus.scrap();
+		}
+		assertTrue(check.get(), message);
+	}
 }

src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakClient.java

@@ -1,5 +1,7 @@
 package io.kokuwa.keycloak.metrics.junit;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -45,7 +47,8 @@ public class KeycloakClient {
 		client.setClientId(clientId);
 		client.setPublicClient(true);
 		client.setDirectAccessGrantsEnabled(true);
-		keycloak.realms().realm(realmName).clients().create(client);
+		var response = keycloak.realms().realm(realmName).clients().create(client);
+		assertEquals(201, response.getStatus());
 	}
 
 	public void createUser(String realmName, String username, String password) {
@@ -59,7 +62,15 @@ public class KeycloakClient {
 		user.setEmailVerified(true);
 		user.setUsername(username);
 		user.setCredentials(List.of(credential));
-		keycloak.realms().realm(realmName).users().create(user);
+		var response = keycloak.realms().realm(realmName).users().create(user);
+		assertEquals(201, response.getStatus());
+	}
+
+	public void deleteUser(String realmName, String username) {
+		keycloak.realms().realm(realmName).users()
+				.searchByUsername(username, true).stream()
+				.map(UserRepresentation::getId)
+				.forEach(keycloak.realms().realm(realmName).users()::delete);
 	}
 
 	public boolean login(String clientId, String realmName, String username, String password) {

src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakExtension.java

@@ -55,8 +55,11 @@ public class KeycloakExtension implements BeforeAllCallback, ParameterResolver {
 				.withEnv("KEYCLOAK_ADMIN", "admin")
 				.withEnv("KEYCLOAK_ADMIN_PASSWORD", "password")
 				.withEnv("KC_LOG_CONSOLE_COLOR", "true")
+				.withEnv("KC_LOG_LEVEL", "io.kokuwa:trace")
 				.withEnv("KC_HEALTH_ENABLED", "true")
 				.withEnv("KC_METRICS_ENABLED", "true")
+				.withEnv("KC_METRICS_STATS_ENABLED", "true")
+				.withEnv("KC_METRICS_STATS_INTERVAL", "PT1s")
 				.withCopyFileToContainer(MountableFile.forHostPath(jar), "/opt/keycloak/providers/metrics.jar")
 				.withLogConsumer(out -> System.out.print(out.getUtf8String()))
 				.withExposedPorts(8080)

src/test/java/io/kokuwa/keycloak/metrics/junit/Prometheus.java

@@ -41,6 +41,14 @@ public class Prometheus {
 				.sum();
 	}
 
+	public int userCount(String realm) {
+		return state.stream()
+				.filter(metric -> Objects.equals(metric.name(), "keycloak_users"))
+				.filter(metric -> Objects.equals(metric.tags().get("realm"), realm))
+				.mapToInt(metric -> metric.value().intValue())
+				.sum();
+	}
+
 	public void scrap() {
 		state.clear();
 		Stream.of(client.scrap().split("[\\r\\n]+"))