From d95167350da10be9b48fb64bced27efd1bb5d9f6 Mon Sep 17 00:00:00 2001
From: Gwendolyn <me@gwendolyn.dev>
Date: Fri, 27 Dec 2024 01:44:20 +0100
Subject: [PATCH] use redis MGET for metrics instead of django's cache.get
 should improve performance

---
 src/c3nav/mapdata/metrics.py | 62 ++++++++++++++++++++++--------------
 src/c3nav/settings.py        |  1 +
 2 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/src/c3nav/mapdata/metrics.py b/src/c3nav/mapdata/metrics.py
index bf7a0d49..9df06058 100644
--- a/src/c3nav/mapdata/metrics.py
+++ b/src/c3nav/mapdata/metrics.py
@@ -1,3 +1,4 @@
+import itertools
 import re
 from typing import Optional, Sequence
 
@@ -7,6 +8,17 @@ from django.core.cache import cache
 
 from c3nav.mapdata.models.report import Report
 
+
+def chunked_iterable(iterable, size):
+    it = iter(iterable)
+    while True:
+        chunk = tuple(itertools.islice(it, size))
+        if not chunk:
+            break
+        yield chunk
+
+
+
 if settings.METRICS:
     from prometheus_client import Gauge
     from prometheus_client.core import CounterMetricFamily
@@ -28,35 +40,37 @@ if settings.METRICS:
             metrics: dict[str, CounterMetricFamily] = dict()
             if settings.CACHES['default']['BACKEND'] == 'django.core.cache.backends.redis.RedisCache':
                 client = cache._cache.get_client()
-                for key in client.keys(f"*{settings.CACHES['default'].get('KEY_PREFIX', '')}apistats__*"):
-                    key: str = key.decode('utf-8').split(':', 2)[2]
-                    value = cache.get(key)
-                    key = key[10:]  # trim apistats__ from the beginning
+                keys = client.keys(f"*{settings.CACHES['default'].get('KEY_PREFIX', '')}apistats__*")
 
-                    # some routing stats don't use double underscores to separate fields, workaround for now
-                    if key.startswith('route_tuple_'):
-                        key = re.sub(r'^route_tuple_(.*)_(.*)$', r'route_tuple__\1__\2', key)
-                    if key.startswith('route_origin_') or key.startswith('route_destination_'):
-                        key = re.sub(r'^route_(origin|destination)_(.*)$', r'route_\1__\2', key)
+                for group in chunked_iterable(keys, settings.METRICS_REDIS_CHUNK_SIZE):
+                    values = client.mget(group)
+                    for key, value in zip(group, values):
+                        key: str = key.decode('utf-8').split(':', 2)[2]
+                        key = key[10:]  # trim apistats__ from the beginning
+                        # some routing stats don't use double underscores to separate fields, workaround for now
+                        if key.startswith('route_tuple_'):
+                            key = re.sub(r'^route_tuple_(.*)_(.*)$', r'route_tuple__\1__\2', key)
+                        if key.startswith('route_origin_') or key.startswith('route_destination_'):
+                            key = re.sub(r'^route_(origin|destination)_(.*)$', r'route_\1__\2', key)
 
-                    name, *labels = key.split('__')
-                    try:
-                        label_names = self.name_registry[name]
-                    except KeyError:
-                        continue
+                        name, *labels = key.split('__')
+                        try:
+                            label_names = self.name_registry[name]
+                        except KeyError:
+                            continue
 
-                    if label_names is None:
-                        label_names = list()
+                        if label_names is None:
+                            label_names = list()
 
-                    if len(label_names) != len(labels):
-                        raise ValueError('configured labels and number of extracted labels doesn\'t match.')
+                        if len(label_names) != len(labels):
+                            raise ValueError('configured labels and number of extracted labels doesn\'t match.')
 
-                    try:
-                        counter = metrics[name]
-                    except KeyError:
-                        counter = metrics[name] = CounterMetricFamily(f'c3nav_{name}', f'c3nav_{name}',
-                                                                      labels=label_names)
-                    counter.add_metric(labels, value)
+                        try:
+                            counter = metrics[name]
+                        except KeyError:
+                            counter = metrics[name] = CounterMetricFamily(f'c3nav_{name}', f'c3nav_{name}',
+                                                                          labels=label_names)
+                        counter.add_metric(labels, value)
             return metrics.values()
 
         def describe(self):
diff --git a/src/c3nav/settings.py b/src/c3nav/settings.py
index 2a6280f6..c458ba39 100644
--- a/src/c3nav/settings.py
+++ b/src/c3nav/settings.py
@@ -413,6 +413,7 @@ with suppress(ImportError):
     INSTALLED_APPS.append('django_extensions')
 
 METRICS = config.getboolean('c3nav', 'metrics', fallback=False)
+METRICS_REDIS_CHUNK_SIZE = config.getint('c3nav', 'metrics_redis_chunk_size', fallback=10)
 if METRICS:
     try:
         import django_prometheus  # noqa