From 37dcc073095dad277d2f50d9d1a1e28374a67de1 Mon Sep 17 00:00:00 2001 From: Stephan Schnabel Date: Tue, 25 Apr 2023 10:28:30 +0200 Subject: [PATCH] Add additional metrics for user/client/session count (#31) --- README.md | 48 +++++++++- .../metrics/stats/MetricsStatsFactory.java | 10 +++ .../stats/MetricsStatsFactoryImpl.java | 69 ++++++++++++++ .../metrics/stats/MetricsStatsSpi.java | 34 +++++++ .../metrics/stats/MetricsStatsTask.java | 89 +++++++++++++++++++ ...keycloak.metrics.stats.MetricsStatsFactory | 1 + .../services/org.keycloak.provider.Spi | 1 + .../kokuwa/keycloak/metrics/KeycloakIT.java | 40 +++++++++ .../metrics/junit/KeycloakClient.java | 15 +++- .../metrics/junit/KeycloakExtension.java | 3 + .../keycloak/metrics/junit/Prometheus.java | 8 ++ 11 files changed, 312 insertions(+), 6 deletions(-) create mode 100644 src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactory.java create mode 100644 src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactoryImpl.java create mode 100644 src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsSpi.java create mode 100644 src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsTask.java create mode 100644 src/main/resources/META-INF/services/io.kokuwa.keycloak.metrics.stats.MetricsStatsFactory create mode 100644 src/main/resources/META-INF/services/org.keycloak.provider.Spi diff --git a/README.md b/README.md index 9b1f341..f9f4236 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Keycloak Event Metrics +# Keycloak Metrics -Provides metrics for Keycloak user/admin events. Tested on Keycloak [20-21](.github/workflows/ci.yaml#L74-L77). +Provides metrics for Keycloak user/admin events and user/client/session count. Tested on Keycloak [20-21](.github/workflows/ci.yaml#L74-L77). [![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/kokuwaio/keycloak-event-metrics.svg?label=License)](http://www.apache.org/licenses/) [![Maven Central](https://img.shields.io/maven-central/v/io.kokuwa.keycloak/keycloak-event-metrics.svg?label=Maven%20Central)](https://central.sonatype.com/search?namespace=io.kokuwa.keycloak&q=keycloak-event-metrics) @@ -14,7 +14,8 @@ Provides metrics for Keycloak user/admin events. Tested on Keycloak [20-21](.git * no realm specific Prometheus endpoint, only `/metrics` (from Quarkus) * no jvm/http metrics, this is [already](https://www.keycloak.org/server/configuration-metrics#_available_metrics) included in Keycloak * different metric names, can relace model ids with name (see [configuration](#kc_metrics_event_replace_ids)) -* deployed to maven central and very small (10 kb vs. 229 KB [aerogear/keycloak-metrics-spi](https://github.com/aerogear/keycloak-metrics-spi)) +* deployed to maven central and very small (15 kb vs. 151 KB [aerogear/keycloak-metrics-spi](https://github.com/aerogear/keycloak-metrics-spi)) +* gauge for active/offline sessions and user/client count ## What? @@ -57,7 +58,7 @@ keycloak_event_admin_total{error="",operation="CREATE",realm="9039a0b5-e8c9-437a ### `KC_METRICS_EVENT_REPLACE_IDS` -Per set to `true` (the default value) than replace model ids with names: +Set to `true` (the default value) than replace model ids from events with names: * [RealmModel#getId()](https://github.com/keycloak/keycloak/blob/main/server-spi/src/main/java/org/keycloak/models/RealmModel.java#L82) with [RealmModel#getName()](https://github.com/keycloak/keycloak/blob/main/server-spi/src/main/java/org/keycloak/models/RealmModel.java#L84) @@ -69,6 +70,45 @@ keycloak_event_user_total{client="other-client",error="",realm="other-realm",typ keycloak_event_user_total{client="other-client",error="invalid_user_credentials",realm="other-realm",type="LOGIN_ERROR",} 1.0 ``` +### `KC_METRICS_STATS_ENABLED` + +Set to `true` (default is `false`) to provide metrics for user/client count per realm and session count per client. Metrics: + +```txt +# HELP keycloak_users +# TYPE keycloak_users gauge +keycloak_users{realm="master",} 1.0 +keycloak_users{realm="my-realm",} 2.0 +keycloak_users{realm="other-realm",} 1.0# HELP keycloak_active_user_sessions +# TYPE keycloak_active_user_sessions gauge +keycloak_active_user_sessions{client="admin-cli",realm="userCount_1",} 0.0 +keycloak_active_user_sessions{client="admin-cli",realm="userCount_2",} 0.0 +keycloak_active_user_sessions{client="admin-cli",realm="master",} 1.0 +# TYPE keycloak_active_client_sessions gauge +keycloak_active_client_sessions{client="admin-cli",realm="userCount_1",} 0.0 +keycloak_active_client_sessions{client="admin-cli",realm="userCount_2",} 0.0 +keycloak_active_client_sessions{client="admin-cli",realm="master",} 0.0 +# TYPE keycloak_offline_sessions gauge +keycloak_offline_sessions{client="admin-cli",realm="userCount_1",} 0.0 +keycloak_offline_sessions{client="admin-cli",realm="userCount_2",} 0.0 +keycloak_offline_sessions{client="admin-cli",realm="master",} 0.0 +``` + +### `KC_METRICS_STATS_INTERVAL` + +If `KC_METRICS_STATS_ENABLED` is `true` this will define the interval for scraping. If not configured `PT60s` will be used. + +### `KC_METRICS_STATS_INFO_THRESHOLD` and `KC_METRICS_STATS_WARN_THRESHOLD` + +If `KC_METRICS_STATS_ENABLED` is `true` this envs will define logging if scraping takes to long. Both envs are parsed as `java.lang.Duration`. + +Default values: + +* `KC_METRICS_STATS_INFO_THRESHOLD`: 50% of `KC_METRICS_STATS_INTERVAL` = 30s +* `KC_METRICS_STATS_WARN_THRESHOLD`: 75% of `KC_METRICS_STATS_INTERVAL` = 45s + +If scrapping takes less than `KC_METRICS_STATS_INFO_THRESHOLD` duration will be logged on debug level. + ## Installation ### Testcontainers diff --git a/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactory.java b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactory.java new file mode 100644 index 0000000..13b626e --- /dev/null +++ b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactory.java @@ -0,0 +1,10 @@ +package io.kokuwa.keycloak.metrics.stats; + +import org.keycloak.provider.ProviderFactory; + +/** + * Factory for Keycloak metrics. + * + * @author Stephan Schnabel + */ +public interface MetricsStatsFactory extends ProviderFactory {} diff --git a/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactoryImpl.java b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactoryImpl.java new file mode 100644 index 0000000..46cb9f5 --- /dev/null +++ b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsFactoryImpl.java @@ -0,0 +1,69 @@ +package io.kokuwa.keycloak.metrics.stats; + +import java.time.Duration; +import java.util.Optional; + +import org.jboss.logging.Logger; +import org.keycloak.Config.Scope; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.KeycloakSessionFactory; +import org.keycloak.models.utils.KeycloakModelUtils; +import org.keycloak.timer.TimerProvider; + +import io.micrometer.core.instrument.Metrics; + +/** + * Implementation of {@link MetricsStatsFactory}. + * + * @author Stephan Schnabel + */ +public class MetricsStatsFactoryImpl implements MetricsStatsFactory { + + private static final Logger log = Logger.getLogger(MetricsStatsFactory.class); + + @Override + public String getId() { + return "default"; + } + + @Override + public void init(Scope config) {} + + @Override + public void postInit(KeycloakSessionFactory factory) { + + if (!"true".equals(System.getenv().get("KC_METRICS_STATS_ENABLED"))) { + log.infov("Keycloak stats not enabled."); + return; + } + + var intervalDuration = Optional + .ofNullable(System.getenv("KC_METRICS_STATS_INTERVAL")) + .map(Duration::parse) + .orElse(Duration.ofSeconds(60)); + var infoThreshold = Optional + .ofNullable(System.getenv("KC_METRICS_STATS_INFO_THRESHOLD")) + .map(Duration::parse) + .orElse(Duration.ofMillis(Double.valueOf(intervalDuration.toMillis() * 0.5).longValue())); + var warnThreshold = Optional + .ofNullable(System.getenv("KC_METRICS_STATS_WARN_THRESHOLD")) + .map(Duration::parse) + .orElse(Duration.ofMillis(Double.valueOf(intervalDuration.toMillis() * 0.75).longValue())); + log.infov("Keycloak stats enabled with interval of {0} and info/warn after {1}/{2}.", + intervalDuration, infoThreshold, warnThreshold); + + var interval = intervalDuration.toMillis(); + var task = new MetricsStatsTask(Metrics.globalRegistry, intervalDuration, infoThreshold, warnThreshold); + KeycloakModelUtils.runJobInTransaction(factory, session -> session + .getProvider(TimerProvider.class) + .schedule(() -> KeycloakModelUtils.runJobInTransaction(factory, task), interval, "metrics")); + } + + @Override + public MetricsStatsTask create(KeycloakSession session) { + return null; + } + + @Override + public void close() {} +} diff --git a/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsSpi.java b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsSpi.java new file mode 100644 index 0000000..35db096 --- /dev/null +++ b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsSpi.java @@ -0,0 +1,34 @@ +package io.kokuwa.keycloak.metrics.stats; + +import org.keycloak.provider.Provider; +import org.keycloak.provider.ProviderFactory; +import org.keycloak.provider.Spi; + +/** + * SPI for Keycloak metrics. + * + * @author Stephan Schnabel + */ +public class MetricsStatsSpi implements Spi { + + @Override + public boolean isInternal() { + return false; + } + + @Override + public String getName() { + return "metrics"; + } + + @Override + public Class getProviderClass() { + return MetricsStatsTask.class; + } + + @Override + public Class> getProviderFactoryClass() { + // this must be an interface, otherwise spi will be silenty ignored + return MetricsStatsFactory.class; + } +} diff --git a/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsTask.java b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsTask.java new file mode 100644 index 0000000..a0767a8 --- /dev/null +++ b/src/main/java/io/kokuwa/keycloak/metrics/stats/MetricsStatsTask.java @@ -0,0 +1,89 @@ +package io.kokuwa.keycloak.metrics.stats; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import org.jboss.logging.Logger; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.KeycloakSessionTask; +import org.keycloak.provider.Provider; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; + +/** + * Keycloak metrics. + * + * @author Stephan Schnabel + */ +public class MetricsStatsTask implements Provider, KeycloakSessionTask { + + private static final Logger log = Logger.getLogger(MetricsStatsTask.class); + private final Map values = new HashMap<>(); + private final MeterRegistry registry; + private final Duration interval; + private final Duration infoThreshold; + private final Duration warnThreshold; + + MetricsStatsTask(MeterRegistry registry, Duration interval, Duration infoThreshold, Duration warnThreshold) { + this.registry = registry; + this.interval = interval; + this.infoThreshold = infoThreshold; + this.warnThreshold = warnThreshold; + } + + @Override + public void run(KeycloakSession session) { + log.tracev("Triggered metrics stats task."); + var start = Instant.now(); + + try { + scrape(session); + } catch (Exception e) { + if (e instanceof org.hibernate.exception.SQLGrammarException) { + log.infov("Metrics status task skipped, database not ready"); + } else { + log.errorv(e, "Failed to scrape stats."); + } + return; + } + + var duration = Duration.between(start, Instant.now()); + if (duration.compareTo(interval) > 0) { + log.errorv("Finished scrapping keycloak stats in {0}, consider to increase interval", duration); + } else if (duration.compareTo(warnThreshold) > 0) { + log.warnv("Finished scrapping keycloak stats in {0}, consider to increase interval", duration); + } else if (duration.compareTo(infoThreshold) > 0) { + log.infov("Finished scrapping keycloak stats in {0}", duration); + } else { + log.debugv("Finished scrapping keycloak stats in {0}", duration); + } + } + + @Override + public void close() {} + + private void scrape(KeycloakSession session) { + session.realms().getRealmsStream().forEach(realm -> { + var tagRealm = Tag.of("realm", realm.getName()); + gauge("keycloak_users", Set.of(tagRealm), session.users().getUsersCount(realm)); + gauge("keycloak_clients", Set.of(tagRealm), session.clients().getClientsCount(realm)); + var sessions = session.sessions(); + var activeSessions = sessions.getActiveClientSessionStats(realm, false); + realm.getClientsStream().forEach(client -> { + var tags = Set.of(tagRealm, Tag.of("client", client.getClientId())); + gauge("keycloak_offline_sessions", tags, sessions.getOfflineSessionsCount(realm, client)); + gauge("keycloak_active_user_sessions", tags, sessions.getActiveUserSessions(realm, client)); + gauge("keycloak_active_client_sessions", tags, activeSessions.getOrDefault(client.getId(), 0L)); + }); + }); + } + + private void gauge(String name, Set tags, long value) { + values.computeIfAbsent(name + tags, s -> registry.gauge(name, tags, new AtomicLong())).set(value); + } +} diff --git a/src/main/resources/META-INF/services/io.kokuwa.keycloak.metrics.stats.MetricsStatsFactory b/src/main/resources/META-INF/services/io.kokuwa.keycloak.metrics.stats.MetricsStatsFactory new file mode 100644 index 0000000..45c8b40 --- /dev/null +++ b/src/main/resources/META-INF/services/io.kokuwa.keycloak.metrics.stats.MetricsStatsFactory @@ -0,0 +1 @@ +io.kokuwa.keycloak.metrics.stats.MetricsStatsFactoryImpl diff --git a/src/main/resources/META-INF/services/org.keycloak.provider.Spi b/src/main/resources/META-INF/services/org.keycloak.provider.Spi new file mode 100644 index 0000000..f80dc90 --- /dev/null +++ b/src/main/resources/META-INF/services/org.keycloak.provider.Spi @@ -0,0 +1 @@ +io.kokuwa.keycloak.metrics.stats.MetricsStatsSpi \ No newline at end of file diff --git a/src/test/java/io/kokuwa/keycloak/metrics/KeycloakIT.java b/src/test/java/io/kokuwa/keycloak/metrics/KeycloakIT.java index 2423820..4475113 100644 --- a/src/test/java/io/kokuwa/keycloak/metrics/KeycloakIT.java +++ b/src/test/java/io/kokuwa/keycloak/metrics/KeycloakIT.java @@ -1,11 +1,14 @@ package io.kokuwa.keycloak.metrics; import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.time.Instant; import java.util.UUID; +import java.util.function.Supplier; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -73,4 +76,41 @@ public class KeycloakIT { () -> assertEquals(loginErrorBefore1 + 0, loginErrorAfter1, "login failure #1"), () -> assertEquals(loginErrorBefore2 + 1, loginErrorAfter2, "login failure #2")); } + + @DisplayName("user count") + @Test + void userCount(KeycloakClient keycloak, Prometheus prometheus) { + + var realmName1 = "userCount_1"; + var realmName2 = "userCount_2"; + var username = UUID.randomUUID().toString(); + + keycloak.createRealm(realmName1); + keycloak.createRealm(realmName2); + + await(() -> prometheus.userCount(realmName1) == 0, prometheus, "realm 1 not found"); + await(() -> prometheus.userCount(realmName2) == 0, prometheus, "realm 2 not found"); + + keycloak.createUser(realmName1, username, UUID.randomUUID().toString()); + keycloak.createUser(realmName1, UUID.randomUUID().toString(), UUID.randomUUID().toString()); + keycloak.createUser(realmName1, UUID.randomUUID().toString(), UUID.randomUUID().toString()); + keycloak.createUser(realmName2, UUID.randomUUID().toString(), UUID.randomUUID().toString()); + + await(() -> prometheus.userCount(realmName1) == 3, prometheus, "realm 1 shoud have 3 users"); + await(() -> prometheus.userCount(realmName2) == 1, prometheus, "realm 2 shoud have 1 users"); + + keycloak.deleteUser(realmName1, username); + + await(() -> prometheus.userCount(realmName1) == 2, prometheus, "realm 1 shoud have 2 users after deletion"); + await(() -> prometheus.userCount(realmName2) == 1, prometheus, "realm 2 shoud have 1 users"); + } + + void await(Supplier check, Prometheus prometheus, String message) { + var end = Instant.now().plusSeconds(10); + while (Instant.now().isBefore(end) && !check.get()) { + assertDoesNotThrow(() -> Thread.sleep(1000)); + prometheus.scrap(); + } + assertTrue(check.get(), message); + } } diff --git a/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakClient.java b/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakClient.java index 3d8f902..6d48781 100644 --- a/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakClient.java +++ b/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakClient.java @@ -1,5 +1,7 @@ package io.kokuwa.keycloak.metrics.junit; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.List; import java.util.Map; import java.util.UUID; @@ -45,7 +47,8 @@ public class KeycloakClient { client.setClientId(clientId); client.setPublicClient(true); client.setDirectAccessGrantsEnabled(true); - keycloak.realms().realm(realmName).clients().create(client); + var response = keycloak.realms().realm(realmName).clients().create(client); + assertEquals(201, response.getStatus()); } public void createUser(String realmName, String username, String password) { @@ -59,7 +62,15 @@ public class KeycloakClient { user.setEmailVerified(true); user.setUsername(username); user.setCredentials(List.of(credential)); - keycloak.realms().realm(realmName).users().create(user); + var response = keycloak.realms().realm(realmName).users().create(user); + assertEquals(201, response.getStatus()); + } + + public void deleteUser(String realmName, String username) { + keycloak.realms().realm(realmName).users() + .searchByUsername(username, true).stream() + .map(UserRepresentation::getId) + .forEach(keycloak.realms().realm(realmName).users()::delete); } public boolean login(String clientId, String realmName, String username, String password) { diff --git a/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakExtension.java b/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakExtension.java index 8238320..b5d7775 100644 --- a/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakExtension.java +++ b/src/test/java/io/kokuwa/keycloak/metrics/junit/KeycloakExtension.java @@ -55,8 +55,11 @@ public class KeycloakExtension implements BeforeAllCallback, ParameterResolver { .withEnv("KEYCLOAK_ADMIN", "admin") .withEnv("KEYCLOAK_ADMIN_PASSWORD", "password") .withEnv("KC_LOG_CONSOLE_COLOR", "true") + .withEnv("KC_LOG_LEVEL", "io.kokuwa:trace") .withEnv("KC_HEALTH_ENABLED", "true") .withEnv("KC_METRICS_ENABLED", "true") + .withEnv("KC_METRICS_STATS_ENABLED", "true") + .withEnv("KC_METRICS_STATS_INTERVAL", "PT1s") .withCopyFileToContainer(MountableFile.forHostPath(jar), "/opt/keycloak/providers/metrics.jar") .withLogConsumer(out -> System.out.print(out.getUtf8String())) .withExposedPorts(8080) diff --git a/src/test/java/io/kokuwa/keycloak/metrics/junit/Prometheus.java b/src/test/java/io/kokuwa/keycloak/metrics/junit/Prometheus.java index 7bcc4ff..74a0cb9 100644 --- a/src/test/java/io/kokuwa/keycloak/metrics/junit/Prometheus.java +++ b/src/test/java/io/kokuwa/keycloak/metrics/junit/Prometheus.java @@ -41,6 +41,14 @@ public class Prometheus { .sum(); } + public int userCount(String realm) { + return state.stream() + .filter(metric -> Objects.equals(metric.name(), "keycloak_users")) + .filter(metric -> Objects.equals(metric.tags().get("realm"), realm)) + .mapToInt(metric -> metric.value().intValue()) + .sum(); + } + public void scrap() { state.clear(); Stream.of(client.scrap().split("[\\r\\n]+"))