diff --git a/local_run/SENSOR_README.md b/local_run/SENSOR_README.md new file mode 100644 index 00000000..4b5db906 --- /dev/null +++ b/local_run/SENSOR_README.md @@ -0,0 +1,152 @@ +# NOI Sensor Management System + +This system allows you to manage environmental sensors from the NOI Open Data Hub in c3nav, displaying them as overlay features on different levels/floors. + +## Overview + +The system supports: +- Multiple sensors on the same overlay but on different levels +- Dynamic addition of new sensors through Django management commands +- Automatic data scraping from NOI Open Data Hub APIs +- Real-time display of CO2, temperature, humidity and other environmental data + +## Architecture + +- **Single Overlay**: All NOI environmental sensors are managed under one `DataOverlay` +- **Multiple Levels**: Sensors can be placed on different floors (floor0, floor1, etc.) +- **Flexible Configuration**: Sensor locations and properties are configurable via the overlay's `sensor_config` field +- **Dynamic Discovery**: The system can automatically discover and display any sensor data from the NOI API + +## Setup + +The main setup is handled by the `up.sh` script, which: + +1. Creates a single "NOI Environmental Sensors" overlay +2. Configures initial sensors with their coordinates and levels +3. Scrapes initial data from the NOI Open Data Hub +4. Applies necessary database migrations + +## Managing Sensors + +### 1. List All Sensors +```bash +# Using the helper script +./manage_noi_sensors.sh list + +# Or directly +docker compose exec -T c3nav-core python manage.py list_sensors --overlay-id 1 +``` + +### 2. Add a New Sensor +```bash +# Using the helper script +./manage_noi_sensors.sh add 'NOI:YourSensorID' 'Sensor Display Name' 300.0 250.0 floor1 + +# Or directly +docker compose exec -T c3nav-core python manage.py add_sensor \ + --overlay-id 1 \ + --sensor-id 'NOI:YourSensorID' \ + --name 'Sensor Display Name' \ + --x 300.0 \ + --y 250.0 \ + --level floor1 +``` + +### 3. Scrape Data for All Sensors +```bash +# Using the helper script +./manage_noi_sensors.sh scrape + +# Or directly +docker compose exec -T c3nav-core python manage.py manage_sensors --scrape-data --overlay-id 1 +``` + +## Configuration Structure + +The overlay's `sensor_config` field contains: + +```json +{ + "data_path": "data", + "mappings": { + "id_field": "scode", + "name_field": "sname", + "x_field": "scoordinate.x", + "y_field": "scoordinate.y" + }, + "sensors": [ + { + "id": "NOI:FreeSoftwareLab-Temperature", + "coordinates": {"x": 291.0, "y": 241.0}, + "level": "floor1" + }, + { + "id": "NOI:NOI-A1-Floor1-CO2", + "coordinates": {"x": 270.0, "y": 241.0}, + "level": "floor1" + } + ] +} +``` + +## Database Schema + +### DataOverlay fields: +- `data_source_url`: URL to scrape sensor data from +- `sensor_config`: JSON configuration for sensor mapping and processing + +### DataOverlayFeature fields: +- `sensor_id`: Unique identifier for the sensor +- `sensor_type`: Type of sensor (e.g., 'environmental') +- `sensor_value`: Single sensor value (nullable for multi-measurement sensors) +- `sensor_unit`: Unit of measurement (nullable for multi-measurement sensors) +- `coordinates_x`, `coordinates_y`: Position in c3nav coordinate system +- `last_updated`: Timestamp of last data update +- `sensor_data`: Raw sensor data for debugging +- `extra_data`: Processed sensor readings for display + +## Data Flow + +1. **Configuration**: Sensors are configured in the overlay's `sensor_config` +2. **Scraping**: The `manage_sensors` command fetches data from NOI Open Data Hub +3. **Processing**: Data is processed according to sensor configuration +4. **Storage**: Sensor features are created/updated in the database +5. **Display**: Sensors appear as interactive points on the map + +## Adding New Sensor Types + +To add a new sensor from the NOI Open Data Hub: + +1. Find the sensor ID in the NOI API (usually starts with "NOI:") +2. Determine the coordinates where it should appear on the map +3. Choose the appropriate level/floor +4. Add it using the `add_sensor` command +5. Run the scrape command to fetch initial data + +## Troubleshooting + +### Sensor not appearing on map +- Check if the level exists: `docker compose exec -T c3nav-core python manage.py shell -c "from c3nav.mapdata.models import Level; print([l.short_label for l in Level.objects.all()])"` +- Verify coordinates are within the map bounds +- Check if the overlay is enabled and visible + +### No data being scraped +- Verify the sensor ID exists in the NOI Open Data Hub API +- Check the API URL is accessible: https://mobility.api.opendatahub.com/v2/flat/IndoorStation/*/latest +- Review logs during scraping for errors + +### Data not updating +- Check the `last_updated` field in the sensor feature +- Verify the scraping command completed successfully +- Consider running the scrape command more frequently + +## Files + +- `up.sh`: Main setup script +- `manage_noi_sensors.sh`: Helper script for sensor management +- `src/c3nav/mapdata/management/commands/manage_sensors.py`: Core sensor management command +- `src/c3nav/mapdata/management/commands/add_sensor.py`: Command to add new sensors +- `src/c3nav/mapdata/management/commands/list_sensors.py`: Command to list sensors +- `src/c3nav/mapdata/models/overlay.py`: Database models +- `src/c3nav/mapdata/migrations/0140_add_temperature_fields.py`: Migration for sensor fields +- `src/c3nav/mapdata/migrations/0141_add_sensor_data_field.py`: Migration for sensor_data field diff --git a/local_run/manage_noi_sensors.sh b/local_run/manage_noi_sensors.sh new file mode 100755 index 00000000..d6c23b0a --- /dev/null +++ b/local_run/manage_noi_sensors.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Helper script to manage NOI sensors +# Usage: ./manage_noi_sensors.sh [add|list|scrape] [args...] + +COMPOSE_EXEC="docker compose exec -T c3nav-core python manage.py" + +case "$1" in + "add") + if [ $# -lt 6 ]; then + echo "Usage: $0 add " + echo "Example: $0 add 'NOI:MyNewSensor' 'My New Sensor' 300.0 250.0 floor1" + exit 1 + fi + SENSOR_ID="$2" + NAME="$3" + X="$4" + Y="$5" + LEVEL="$6" + echo "Adding sensor: $NAME ($SENSOR_ID) at ($X, $Y) on $LEVEL" + $COMPOSE_EXEC add_sensor --overlay-id 1 --sensor-id "$SENSOR_ID" --name "$NAME" --x "$X" --y "$Y" --level "$LEVEL" + ;; + "list") + echo "Listing all sensors in overlay 1:" + $COMPOSE_EXEC list_sensors --overlay-id 1 + ;; + "scrape") + echo "Scraping data for all sensors in overlay 1:" + $COMPOSE_EXEC manage_sensors --scrape-data --overlay-id 1 + ;; + *) + echo "NOI Sensor Management Helper" + echo "Usage: $0 [add|list|scrape] [args...]" + echo "" + echo "Commands:" + echo " add - Add a new sensor" + echo " list - List all sensors" + echo " scrape - Scrape data for all sensors" + echo "" + echo "Examples:" + echo " $0 add 'NOI:NewSensor' 'My Sensor' 300.0 250.0 floor1" + echo " $0 list" + echo " $0 scrape" + exit 1 + ;; +esac diff --git a/local_run/up.sh b/local_run/up.sh index 131a9a4c..8caabb74 100755 --- a/local_run/up.sh +++ b/local_run/up.sh @@ -61,61 +61,57 @@ from c3nav.mapdata.models import DataOverlay, DataOverlayFeature DataOverlay.objects.filter(titles__en__icontains='Environmental').delete() DataOverlay.objects.filter(titles__en__icontains='Temperature').delete() -# Create NOI environmental sensor overlay with real data configuration +# Create single NOI environmental sensor overlay with multiple sensors configuration overlay = DataOverlay.objects.create( titles={'en': 'NOI Environmental Sensors'}, description='Real-time CO2 and temperature sensors from NOI Open Data Hub - displays current readings with values and units', default_geomtype='point', - data_source_url='https://mobility.api.opendatahub.com/v2/flat/IndoorStation/*/latest?where=and(scode.eq.%22NOI:FreeSoftwareLab-Temperature%22)', + data_source_url='https://mobility.api.opendatahub.com/v2/flat/IndoorStation/*/latest', sensor_config={ 'data_path': 'data', - 'level': 'floor1', # Specify which floor/level to place sensors on 'mappings': { 'id_field': 'scode', - 'name_field': 'sname', - 'fixed_coordinates': { - 'x': 291.0, - 'y': 241.0 + 'name_field': 'sname', + 'x_field': 'scoordinate.x', + 'y_field': 'scoordinate.y' + }, + 'sensors': [ + { + 'id': 'NOI:FreeSoftwareLab-Temperature', + 'coordinates': {'x': 291.0, 'y': 241.0}, + 'level': 'floor1' + }, + { + 'id': 'NOI:NOI-A1-Floor1-CO2', + 'coordinates': {'x': 270.0, 'y': 241.0}, + 'level': 'floor1' } - } - }, - update_interval=120 -) -overlay2 = DataOverlay.objects.create( - titles={'en': 'NOI Environmental Sensors 2'}, - description='Real-time CO2 and temperature sensors from NOI Open Data Hub - displays current readings with values and units', - default_geomtype='point', - data_source_url='https://mobility.api.opendatahub.com/v2/flat/IndoorStation/*/latest?where=and(scode.eq.%22NOI:NOI-A1-Floor1-CO2%22)', - sensor_config={ - 'data_path': 'data', - 'level': 'floor1', # Specify which floor/level to place sensors on - 'mappings': { - 'id_field': 'scode', - 'name_field': 'sname', - 'fixed_coordinates': { - 'x': 270.0, - 'y': 241.0 - } - } + ] }, update_interval=120 ) print(f"NOI sensor overlay created with ID {overlay.id}") -print(f"NOI sensor overlay 2 created with ID {overlay2.id}") EOF -# Scrape real NOI sensor data for both overlays +# Scrape real NOI sensor data echo "Scraping NOI sensor data..." # Give the database a moment to settle after overlay creation sleep 2 -# Scrape the overlays directly using their expected IDs (1 and 2) -echo "Scraping first overlay (ID: 1)..." +# Scrape the overlay data (should automatically discover all configured sensors) +echo "Scraping overlay data (ID: 1)..." docker compose exec -T c3nav-core python manage.py manage_sensors --scrape-data --overlay-id 1 -echo "Scraping second overlay (ID: 2)..." -docker compose exec -T c3nav-core python manage.py manage_sensors --scrape-data --overlay-id 2 +# List all sensors to verify setup +echo "Listing all sensors in the overlay..." +docker compose exec -T c3nav-core python manage.py list_sensors --overlay-id 1 echo "Sensor setup completed!" +echo "" +echo "To add a new sensor to the overlay, use:" +echo "docker compose exec -T c3nav-core python manage.py add_sensor --overlay-id 1 --sensor-id 'NOI:YourSensorID' --name 'Your Sensor Name' --x 300.0 --y 250.0 --level floor1" +echo "" +echo "To scrape data for all sensors:" +echo "docker compose exec -T c3nav-core python manage.py manage_sensors --scrape-data --overlay-id 1" diff --git a/src/c3nav/mapdata/management/commands/add_sensor.py b/src/c3nav/mapdata/management/commands/add_sensor.py new file mode 100644 index 00000000..829f8d68 --- /dev/null +++ b/src/c3nav/mapdata/management/commands/add_sensor.py @@ -0,0 +1,141 @@ +import json +from django.core.management.base import BaseCommand +from django.utils import timezone +from shapely.geometry import Point + +from c3nav.mapdata.models import DataOverlay, DataOverlayFeature, Level + + +class Command(BaseCommand): + help = 'Add a new sensor to an existing overlay' + + def add_arguments(self, parser): + parser.add_argument( + '--overlay-id', + type=int, + required=True, + help='ID of the overlay to add the sensor to', + ) + parser.add_argument( + '--sensor-id', + type=str, + required=True, + help='Unique ID for the sensor (e.g., NOI:Sensor-ID)', + ) + parser.add_argument( + '--name', + type=str, + required=True, + help='Display name for the sensor', + ) + parser.add_argument( + '--x', + type=float, + required=True, + help='X coordinate in c3nav coordinate system', + ) + parser.add_argument( + '--y', + type=float, + required=True, + help='Y coordinate in c3nav coordinate system', + ) + parser.add_argument( + '--level', + type=str, + default='floor0', + help='Level/floor where the sensor is located (default: floor0)', + ) + parser.add_argument( + '--sensor-type', + type=str, + default='environmental', + help='Type of sensor (default: environmental)', + ) + + def handle(self, *args, **options): + try: + overlay = DataOverlay.objects.get(id=options['overlay_id']) + except DataOverlay.DoesNotExist: + self.stderr.write(f'Overlay with ID {options["overlay_id"]} not found') + return + + try: + level = Level.objects.get(short_label=options['level']) + except Level.DoesNotExist: + self.stderr.write(f'Level "{options["level"]}" not found') + return + + # Update overlay configuration to include the new sensor + sensor_config = overlay.sensor_config or {} + if 'sensors' not in sensor_config: + sensor_config['sensors'] = [] + + # Check if sensor already exists in config + existing_sensor = None + for i, sensor in enumerate(sensor_config['sensors']): + if sensor['id'] == options['sensor_id']: + existing_sensor = i + break + + new_sensor_config = { + 'id': options['sensor_id'], + 'coordinates': {'x': options['x'], 'y': options['y']}, + 'level': options['level'] + } + + if existing_sensor is not None: + sensor_config['sensors'][existing_sensor] = new_sensor_config + self.stdout.write(f'Updated sensor configuration for {options["sensor_id"]}') + else: + sensor_config['sensors'].append(new_sensor_config) + self.stdout.write(f'Added sensor configuration for {options["sensor_id"]}') + + overlay.sensor_config = sensor_config + overlay.save() + + # Create the sensor feature (or update if it exists) + point = Point(options['x'], options['y']) + + feature, created = DataOverlayFeature.objects.update_or_create( + overlay=overlay, + sensor_id=options['sensor_id'], + defaults={ + 'titles': {'en': options['name']}, + 'geometry': point, + 'level': level, + 'sensor_type': options['sensor_type'], + 'coordinates_x': options['x'], + 'coordinates_y': options['y'], + 'fill_color': '#95A5A6', # Default gray + 'stroke_color': '#95A5A6', + 'stroke_width': 2, + 'fill_opacity': 0.8, + 'show_label': True, + 'show_geometry': True, + 'interactive': True, + 'point_icon': 'sensors', + 'last_updated': timezone.now(), + 'extra_data': { + 'Status': 'No data yet', + 'Last Updated': timezone.now().strftime('%Y-%m-%d %H:%M:%S'), + 'Data Source': 'Manual configuration', + 'Station ID': options['sensor_id'] + } + } + ) + + action = 'Created' if created else 'Updated' + self.stdout.write( + self.style.SUCCESS( + f'{action} sensor "{options["name"]}" (ID: {options["sensor_id"]}) ' + f'at coordinates ({options["x"]}, {options["y"]}) on level {options["level"]}' + ) + ) + + self.stdout.write( + 'You can now run the scrape command to fetch data for this sensor:' + ) + self.stdout.write( + f'python manage.py manage_sensors --scrape-data --overlay-id {options["overlay_id"]}' + ) diff --git a/src/c3nav/mapdata/management/commands/list_sensors.py b/src/c3nav/mapdata/management/commands/list_sensors.py new file mode 100644 index 00000000..75d1e6b8 --- /dev/null +++ b/src/c3nav/mapdata/management/commands/list_sensors.py @@ -0,0 +1,60 @@ +from django.core.management.base import BaseCommand +from c3nav.mapdata.models import DataOverlay, DataOverlayFeature + + +class Command(BaseCommand): + help = 'List all sensors in overlays' + + def add_arguments(self, parser): + parser.add_argument( + '--overlay-id', + type=int, + help='ID of a specific overlay to list sensors for', + ) + + def handle(self, *args, **options): + if options['overlay_id']: + try: + overlay = DataOverlay.objects.get(id=options['overlay_id']) + overlays = [overlay] + except DataOverlay.DoesNotExist: + self.stderr.write(f'Overlay with ID {options["overlay_id"]} not found') + return + else: + overlays = DataOverlay.objects.all() + + for overlay in overlays: + self.stdout.write(f'\n=== Overlay {overlay.id}: {overlay.titles.get("en", "Unknown")} ===') + + # Show overlay configuration + sensor_config = overlay.sensor_config or {} + configured_sensors = sensor_config.get('sensors', []) + if configured_sensors: + self.stdout.write('Configured sensors:') + for sensor in configured_sensors: + self.stdout.write(f' - {sensor["id"]} at ({sensor["coordinates"]["x"]}, {sensor["coordinates"]["y"]}) on level {sensor.get("level", "default")}') + + # Show actual sensor features in database + features = DataOverlayFeature.objects.filter(overlay=overlay) + if features: + self.stdout.write(f'\nSensor features in database ({features.count()}):') + for feature in features: + title = feature.titles.get('en', 'Unknown') if feature.titles else 'Unknown' + level_name = feature.level.short_label if feature.level else 'No level' + coords = f'({feature.coordinates_x}, {feature.coordinates_y})' if feature.coordinates_x is not None else 'No coords' + last_updated = feature.last_updated.strftime('%Y-%m-%d %H:%M:%S') if feature.last_updated else 'Never' + + self.stdout.write(f' - {feature.sensor_id}: {title}') + self.stdout.write(f' Level: {level_name}, Coords: {coords}') + self.stdout.write(f' Type: {feature.sensor_type or "Unknown"}, Last updated: {last_updated}') + + if feature.extra_data: + readings = [f'{k}: {v}' for k, v in feature.extra_data.items() + if k not in ['Last Updated', 'Data Source', 'Station ID']] + if readings: + self.stdout.write(f' Readings: {", ".join(readings)}') + else: + self.stdout.write('No sensor features found in database') + + if not overlays: + self.stdout.write('No overlays found') diff --git a/src/c3nav/mapdata/management/commands/manage_sensors.py b/src/c3nav/mapdata/management/commands/manage_sensors.py index 2fc29c13..8996d06c 100644 --- a/src/c3nav/mapdata/management/commands/manage_sensors.py +++ b/src/c3nav/mapdata/management/commands/manage_sensors.py @@ -214,9 +214,9 @@ class Command(BaseCommand): "id_field": "scode", "name_field": "sname", "x_field": "scoordinate.x", - "y_field": "scoordinate.y", - "fixed_coordinates": {"x": 0.0, "y": 0.0} - } + "y_field": "scoordinate.y" + }, + "sensors": [] # List of specific sensors to process } config = {**default_config, **sensor_config} @@ -227,20 +227,6 @@ class Command(BaseCommand): for path_part in config["data_path"].split("."): api_data = api_data.get(path_part, []) - # Get level for sensors - use configured level or default to ground floor - level_name = config.get('level', 'floor0') - try: - level = Level.objects.get(short_label=level_name) - except Level.DoesNotExist: - self.stderr.write(f'Level "{level_name}" not found, using ground floor') - try: - level = Level.objects.get(short_label='floor0') - except Level.DoesNotExist: - level = Level.objects.first() # Final fallback - if not level: - self.stderr.write('No levels found in database') - return - updated_count = 0 created_count = 0 @@ -262,14 +248,26 @@ class Command(BaseCommand): stations[station_id]['measurements'][measurement_type] = item + # If specific sensors are configured, only process those + configured_sensors = config.get('sensors', []) + sensor_configs = {s['id']: s for s in configured_sensors} if configured_sensors else {} + # Process each station and its measurements for station_id, station_data in stations.items(): - # Get coordinates - use fixed coordinates if specified - if config["mappings"].get("fixed_coordinates"): - x_coord = config["mappings"]["fixed_coordinates"]["x"] - y_coord = config["mappings"]["fixed_coordinates"]["y"] + # Skip if we have specific sensors configured and this isn't one of them + if sensor_configs and station_id not in sensor_configs: + continue + + # Get sensor-specific configuration + sensor_specific_config = sensor_configs.get(station_id, {}) + + # Determine coordinates and level for this sensor + if sensor_specific_config.get('coordinates'): + # Use sensor-specific coordinates + x_coord = sensor_specific_config['coordinates']['x'] + y_coord = sensor_specific_config['coordinates']['y'] else: - # Get coordinates from any measurement (they should be the same for all measurements from same station) + # Get coordinates from measurement data first_measurement = next(iter(station_data['measurements'].values())) x_coord = self.get_nested_field(first_measurement, config["mappings"]["x_field"]) y_coord = self.get_nested_field(first_measurement, config["mappings"]["y_field"]) @@ -277,11 +275,24 @@ class Command(BaseCommand): if x_coord is None or y_coord is None: continue - # Convert coordinates if needed (this is simplified) - # For NOI data, coordinates might already be in the right format + # Convert coordinates if needed x_coord = float(x_coord) y_coord = float(y_coord) + # Determine level for this sensor + level_name = sensor_specific_config.get('level', config.get('level', 'floor0')) + try: + level = Level.objects.get(short_label=level_name) + except Level.DoesNotExist: + self.stderr.write(f'Level "{level_name}" not found for sensor {station_id}, using ground floor') + try: + level = Level.objects.get(short_label='floor0') + except Level.DoesNotExist: + level = Level.objects.first() # Final fallback + if not level: + self.stderr.write(f'No levels found in database for sensor {station_id}') + continue + # Collect all sensor data for this station in one feature sensor_readings = {} raw_measurements = {} @@ -381,11 +392,11 @@ class Command(BaseCommand): if created: created_count += 1 readings_str = ', '.join([f"{k}: {v}" for k, v in sensor_readings.items()]) - self.stdout.write(f'Created sensor {sensor_id}: {readings_str}') + self.stdout.write(f'Created sensor {sensor_id} on level {level.short_label}: {readings_str}') else: updated_count += 1 readings_str = ', '.join([f"{k}: {v}" for k, v in sensor_readings.items()]) - self.stdout.write(f'Updated sensor {sensor_id}: {readings_str}') + self.stdout.write(f'Updated sensor {sensor_id} on level {level.short_label}: {readings_str}') self.stdout.write( f'Processed overlay {overlay.id}: {created_count} created, {updated_count} updated' diff --git a/src/c3nav/mapdata/migrations/0141_add_sensor_data_field.py b/src/c3nav/mapdata/migrations/0141_add_sensor_data_field.py new file mode 100644 index 00000000..392b5921 --- /dev/null +++ b/src/c3nav/mapdata/migrations/0141_add_sensor_data_field.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.5 on 2025-08-02 12:00 + +from django.db import migrations +import django_pydantic_field + + +class Migration(migrations.Migration): + + dependencies = [ + ('mapdata', '0140_add_temperature_fields'), + ] + + operations = [ + migrations.AddField( + model_name='dataoverlayfeature', + name='sensor_data', + field=django_pydantic_field.SchemaField( + schema=dict, blank=True, null=True, + verbose_name='Raw Sensor Data', + help_text='Raw data from sensor for debugging and additional info' + ), + ), + ]