use redis MGET for metrics instead of django's cache.get
should improve performance
This commit is contained in:
parent
0a10de795d
commit
d95167350d
2 changed files with 39 additions and 24 deletions
|
@ -1,3 +1,4 @@
|
|||
import itertools
|
||||
import re
|
||||
from typing import Optional, Sequence
|
||||
|
||||
|
@ -7,6 +8,17 @@ from django.core.cache import cache
|
|||
|
||||
from c3nav.mapdata.models.report import Report
|
||||
|
||||
|
||||
def chunked_iterable(iterable, size):
|
||||
it = iter(iterable)
|
||||
while True:
|
||||
chunk = tuple(itertools.islice(it, size))
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
|
||||
|
||||
if settings.METRICS:
|
||||
from prometheus_client import Gauge
|
||||
from prometheus_client.core import CounterMetricFamily
|
||||
|
@ -28,11 +40,13 @@ if settings.METRICS:
|
|||
metrics: dict[str, CounterMetricFamily] = dict()
|
||||
if settings.CACHES['default']['BACKEND'] == 'django.core.cache.backends.redis.RedisCache':
|
||||
client = cache._cache.get_client()
|
||||
for key in client.keys(f"*{settings.CACHES['default'].get('KEY_PREFIX', '')}apistats__*"):
|
||||
key: str = key.decode('utf-8').split(':', 2)[2]
|
||||
value = cache.get(key)
|
||||
key = key[10:] # trim apistats__ from the beginning
|
||||
keys = client.keys(f"*{settings.CACHES['default'].get('KEY_PREFIX', '')}apistats__*")
|
||||
|
||||
for group in chunked_iterable(keys, settings.METRICS_REDIS_CHUNK_SIZE):
|
||||
values = client.mget(group)
|
||||
for key, value in zip(group, values):
|
||||
key: str = key.decode('utf-8').split(':', 2)[2]
|
||||
key = key[10:] # trim apistats__ from the beginning
|
||||
# some routing stats don't use double underscores to separate fields, workaround for now
|
||||
if key.startswith('route_tuple_'):
|
||||
key = re.sub(r'^route_tuple_(.*)_(.*)$', r'route_tuple__\1__\2', key)
|
||||
|
|
|
@ -413,6 +413,7 @@ with suppress(ImportError):
|
|||
INSTALLED_APPS.append('django_extensions')
|
||||
|
||||
METRICS = config.getboolean('c3nav', 'metrics', fallback=False)
|
||||
METRICS_REDIS_CHUNK_SIZE = config.getint('c3nav', 'metrics_redis_chunk_size', fallback=10)
|
||||
if METRICS:
|
||||
try:
|
||||
import django_prometheus # noqa
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue