From 863e0161b0e15b02d39b2ce6596a599c3eefd239 Mon Sep 17 00:00:00 2001 From: rafaeldpsilva Date: Wed, 8 Oct 2025 14:03:57 +0100 Subject: [PATCH] Implement dynamic energy data collections per building - Store energy data in separate MongoDB collections for each SLGs/Community/Building directory - Update FTP monitor and database manager to track directory paths and select appropriate collections - Add collection stats to database statistics API - Update sensor and token services for improved API consistency - Add 'rb' (rebuild and restart) option to deploy.sh script --- bootstrap_sensors.py | 559 +++++++++++++++++++++++++++++++++++++ data_simulator_enhanced.py | 354 +++++++++++++++++++++++ microservices/tasks | 47 ++++ 3 files changed, 960 insertions(+) create mode 100644 bootstrap_sensors.py create mode 100644 data_simulator_enhanced.py create mode 100644 microservices/tasks diff --git a/bootstrap_sensors.py b/bootstrap_sensors.py new file mode 100644 index 0000000..9af3da5 --- /dev/null +++ b/bootstrap_sensors.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python3 +""" +Bootstrap script to populate the database with sensors and room configurations. +This script creates a realistic smart building sensor setup for testing and development. +""" + +import asyncio +import json +import sys +import logging +from datetime import datetime +from typing import List, Dict, Any +import aiohttp + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Base URLs +API_BASE_URL = "http://localhost:8000" +SENSOR_SERVICE_URL = "http://localhost:8007" + +s = { + 'id': 'sensor_1', + 'name': 'Energy Monitor 1', + 'type': 'energy', + 'room': 'Conference Room A', + 'status': 'online', + 'lastSeen': Date.now() / 1000, + 'capabilities': { + 'monitoring': ['energy'], + 'actions': [], + }, + 'metadata': { + 'location': 'Wall mounted', + 'model': 'EM-100', + 'firmware': '2.1.0', + }, + },{ + 'id': 'sensor_2', + 'name': 'HVAC Controller 1', + 'type': 'hvac', + 'room': 'Conference Room A', + 'status': 'online', + 'lastSeen': Date.now() / 1000, + 'capabilities': { + 'monitoring': ['temperature', 'co2'], + 'actions': [ + { + 'id': 'temp_adjust', + 'name': 'Adjust Temperature', + 'type': 'adjust', + 'icon': '🌡️', + 'parameters': { min: 18, max: 28, step: 0.5 }, + }, + { + 'id': 'fan_speed', + 'name': 'Fan Speed', + 'type': 'adjust', + 'icon': '💨', + 'parameters': { min: 0, max: 5, step: 1 }, + }, + { 'id': 'power_toggle', 'name': 'Power', 'type': 'toggle', 'icon': '⚡' }, + ], + }, + metadata: { + location: 'Ceiling mounted', + model: 'HVAC-200', + firmware: '3.2.1', + }, + }, + { + id: 'sensor_3', + name: 'Smart Light Controller', + type: 'lighting', + room: 'Office Floor 1', + status: 'online', + lastSeen: Date.now() / 1000, + capabilities: { + monitoring: ['energy'], + actions: [ + { + id: 'brightness', + name: 'Brightness', + type: 'adjust', + icon: '💡', + parameters: { min: 0, max: 100, step: 5 }, + }, + { id: 'power_toggle', name: 'Power', type: 'toggle', icon: '⚡' }, + { + id: 'scene', + name: 'Scene', + type: 'adjust', + icon: '🎨', + parameters: { options: ['Work', 'Meeting', 'Presentation', 'Relax'] }, + }, + ], + }, + metadata: { + location: 'Ceiling grid', + model: 'SL-300', + firmware: '1.5.2', + }, + }, + { + id: 'sensor_4', + name: 'CO2 Sensor', + type: 'co2', + room: 'Meeting Room 1', + status: 'online', + lastSeen: Date.now() / 1000, + capabilities: { + monitoring: ['co2', 'temperature', 'humidity'], + actions: [{ id: 'calibrate', name: 'Calibrate', type: 'trigger', icon: '⚙️' }], + }, + metadata: { + location: 'Wall mounted', + model: 'CO2-150', + firmware: '2.0.3', + battery: 85, + }, + }, + { + id: 'sensor_5', + name: 'Security Camera', + type: 'security', + room: 'Lobby', + status: 'online', + lastSeen: Date.now() / 1000, + capabilities: { + monitoring: ['motion'], + actions: [ + { id: 'record_toggle', name: 'Recording', type: 'toggle', icon: '📹' }, + { id: 'ptz_control', name: 'Pan/Tilt/Zoom', type: 'trigger', icon: '🎥' }, + { id: 'night_mode', name: 'Night Mode', type: 'toggle', icon: '🌙' }, + ], + }, + metadata: { + location: 'Corner ceiling', + model: 'SEC-400', + firmware: '4.1.0', + }, + }, +# Bootstrap data configuration +BOOTSTRAP_SENSORS = [ + # Living Room Sensors + { + "sensor_id": "lr_energy_001", + "name": "Living Room Main Energy Monitor", + "sensor_type": "energy", + "room": "living_room", + "location": "Main electrical panel - Living Room circuit", + "floor": "1", + "manufacturer": "SmartMeter Co", + "model": "SM-E300", + "status": "online" + }, + { + "sensor_id": "lr_co2_001", + "name": "Living Room Air Quality Monitor", + "sensor_type": "co2", + "room": "living_room", + "location": "Wall mounted near seating area", + "floor": "1", + "manufacturer": "AirSense", + "model": "AS-CO2-Pro", + "status": "online" + }, + { + "sensor_id": "lr_temp_001", + "name": "Living Room Temperature Sensor", + "sensor_type": "temperature", + "room": "living_room", + "location": "Central wall position", + "floor": "1", + "manufacturer": "TempTech", + "model": "TT-T200", + "status": "online" + }, + + # Kitchen Sensors + { + "sensor_id": "kt_energy_001", + "name": "Kitchen Appliances Energy Monitor", + "sensor_type": "energy", + "room": "kitchen", + "location": "Kitchen appliance circuit", + "floor": "1", + "manufacturer": "SmartMeter Co", + "model": "SM-E300", + "status": "online" + }, + { + "sensor_id": "kt_humidity_001", + "name": "Kitchen Humidity Sensor", + "sensor_type": "humidity", + "room": "kitchen", + "location": "Above sink area", + "floor": "1", + "manufacturer": "HumidSense", + "model": "HS-H150", + "status": "online" + }, + { + "sensor_id": "kt_temp_001", + "name": "Kitchen Temperature Monitor", + "sensor_type": "temperature", + "room": "kitchen", + "location": "Central kitchen position", + "floor": "1", + "manufacturer": "TempTech", + "model": "TT-T200", + "status": "online" + }, + + # Bedroom Sensors + { + "sensor_id": "br_energy_001", + "name": "Bedroom Energy Monitor", + "sensor_type": "energy", + "room": "bedroom", + "location": "Bedroom electrical circuit", + "floor": "1", + "manufacturer": "SmartMeter Co", + "model": "SM-E200", + "status": "online" + }, + { + "sensor_id": "br_co2_001", + "name": "Bedroom Air Quality Monitor", + "sensor_type": "co2", + "room": "bedroom", + "location": "Bedside wall mount", + "floor": "1", + "manufacturer": "AirSense", + "model": "AS-CO2-Basic", + "status": "online" + }, + { + "sensor_id": "br_temp_001", + "name": "Bedroom Temperature Sensor", + "sensor_type": "temperature", + "room": "bedroom", + "location": "Opposite wall from bed", + "floor": "1", + "manufacturer": "TempTech", + "model": "TT-T100", + "status": "online" + }, + + # Office Sensors + { + "sensor_id": "of_energy_001", + "name": "Office Equipment Energy Monitor", + "sensor_type": "energy", + "room": "office", + "location": "Office equipment circuit", + "floor": "1", + "manufacturer": "SmartMeter Co", + "model": "SM-E300", + "status": "online" + }, + { + "sensor_id": "of_co2_001", + "name": "Office Air Quality Monitor", + "sensor_type": "co2", + "room": "office", + "location": "Desk area wall mount", + "floor": "1", + "manufacturer": "AirSense", + "model": "AS-CO2-Pro", + "status": "online" + }, + { + "sensor_id": "of_motion_001", + "name": "Office Motion Detector", + "sensor_type": "motion", + "room": "office", + "location": "Ceiling mounted - center", + "floor": "1", + "manufacturer": "MotionTech", + "model": "MT-M100", + "status": "online" + }, + + # Bathroom Sensors + { + "sensor_id": "bt_humidity_001", + "name": "Bathroom Humidity Monitor", + "sensor_type": "humidity", + "room": "bathroom", + "location": "Ceiling mounted", + "floor": "1", + "manufacturer": "HumidSense", + "model": "HS-H200", + "status": "online" + }, + { + "sensor_id": "bt_temp_001", + "name": "Bathroom Temperature Sensor", + "sensor_type": "temperature", + "room": "bathroom", + "location": "Wall mounted near mirror", + "floor": "1", + "manufacturer": "TempTech", + "model": "TT-T150", + "status": "online" + }, + + # Garage Sensors + { + "sensor_id": "gr_energy_001", + "name": "Garage Energy Monitor", + "sensor_type": "energy", + "room": "garage", + "location": "Garage main circuit", + "floor": "0", + "manufacturer": "SmartMeter Co", + "model": "SM-E100", + "status": "online" + }, + { + "sensor_id": "gr_motion_001", + "name": "Garage Motion Detector", + "sensor_type": "motion", + "room": "garage", + "location": "Ceiling mounted - entrance", + "floor": "0", + "manufacturer": "MotionTech", + "model": "MT-M200", + "status": "online" + } +] + +BOOTSTRAP_ROOMS = [ + { + "name": "living_room", + "display_name": "Living Room", + "description": "Main living area with entertainment center", + "floor": "1", + "area_sqm": 35.5, + "room_type": "living" + }, + { + "name": "kitchen", + "display_name": "Kitchen", + "description": "Main kitchen with appliances", + "floor": "1", + "area_sqm": 15.2, + "room_type": "kitchen" + }, + { + "name": "bedroom", + "display_name": "Master Bedroom", + "description": "Primary bedroom", + "floor": "1", + "area_sqm": 20.1, + "room_type": "bedroom" + }, + { + "name": "office", + "display_name": "Home Office", + "description": "Work from home office space", + "floor": "1", + "area_sqm": 12.8, + "room_type": "office" + }, + { + "name": "bathroom", + "display_name": "Main Bathroom", + "description": "Primary bathroom", + "floor": "1", + "area_sqm": 8.5, + "room_type": "bathroom" + }, + { + "name": "garage", + "display_name": "Garage", + "description": "Two-car garage with workshop area", + "floor": "0", + "area_sqm": 42.0, + "room_type": "garage" + } +] + +async def generate_auth_token() -> str: + """Generate authentication token for API calls""" + token_payload = { + "name": "bootstrap_user", + "list_of_resources": ["sensors", "rooms", "analytics", "health", "data", "export", "events"], + "data_aggregation": True, + "time_aggregation": True, + "embargo": 0, + "exp_hours": 24 + } + + async with aiohttp.ClientSession() as session: + async with session.post( + f"{API_BASE_URL}/api/v1/tokens/generate", + json=token_payload, + headers={"Content-Type": "application/json"} + ) as response: + if response.status == 200: + data = await response.json() + logger.info("Successfully generated authentication token") + return data["token"] + else: + error_text = await response.text() + raise Exception(f"Failed to generate token: {response.status} - {error_text}") + +async def create_rooms(auth_token: str) -> bool: + """Create rooms in the database""" + logger.info("Creating bootstrap rooms...") + + headers = { + "Authorization": f"Bearer {auth_token}", + "Content-Type": "application/json" + } + + success_count = 0 + async with aiohttp.ClientSession() as session: + for room in BOOTSTRAP_ROOMS: + try: + async with session.post( + f"{SENSOR_SERVICE_URL}/rooms", + json=room, + headers=headers + ) as response: + if response.status in [200, 201]: + success_count += 1 + logger.info(f"✓ Created room: {room['display_name']}") + elif response.status == 400: + # Room might already exist + error_data = await response.json() + if "already exists" in error_data.get("detail", ""): + logger.info(f"○ Room already exists: {room['display_name']}") + success_count += 1 + else: + logger.error(f"✗ Failed to create room {room['name']}: {error_data}") + else: + error_text = await response.text() + logger.error(f"✗ Failed to create room {room['name']}: {response.status} - {error_text}") + except Exception as e: + logger.error(f"✗ Exception creating room {room['name']}: {e}") + + logger.info(f"Rooms created: {success_count}/{len(BOOTSTRAP_ROOMS)}") + return success_count > 0 + +async def create_sensors(auth_token: str) -> bool: + """Create sensors in the database""" + logger.info("Creating bootstrap sensors...") + + headers = { + "Authorization": f"Bearer {auth_token}", + "Content-Type": "application/json" + } + + success_count = 0 + async with aiohttp.ClientSession() as session: + for sensor in BOOTSTRAP_SENSORS: + try: + async with session.post( + f"{SENSOR_SERVICE_URL}/sensors", + json=sensor, + headers=headers + ) as response: + if response.status in [200, 201]: + success_count += 1 + logger.info(f"✓ Created sensor: {sensor['name']} ({sensor['sensor_id']})") + elif response.status == 400: + # Sensor might already exist + error_data = await response.json() + if "already exists" in error_data.get("detail", ""): + logger.info(f"○ Sensor already exists: {sensor['sensor_id']}") + success_count += 1 + else: + logger.error(f"✗ Failed to create sensor {sensor['sensor_id']}: {error_data}") + else: + error_text = await response.text() + logger.error(f"✗ Failed to create sensor {sensor['sensor_id']}: {response.status} - {error_text}") + except Exception as e: + logger.error(f"✗ Exception creating sensor {sensor['sensor_id']}: {e}") + + logger.info(f"Sensors created: {success_count}/{len(BOOTSTRAP_SENSORS)}") + return success_count > 0 + +async def verify_bootstrap() -> None: + """Verify that sensors were created successfully""" + logger.info("Verifying bootstrap results...") + + try: + # Check sensors directly on sensor service (no auth required for health checks) + async with aiohttp.ClientSession() as session: + async with session.get(f"{SENSOR_SERVICE_URL}/sensors/get") as response: + if response.status == 200: + data = await response.json() + logger.info(f"✓ Total sensors in database: {data['count']}") + + # Group by room + rooms = {} + for sensor in data.get('sensors', []): + room = sensor.get('room', 'unknown') + if room not in rooms: + rooms[room] = [] + rooms[room].append(sensor['sensor_id']) + + for room, sensors in rooms.items(): + logger.info(f" - {room}: {len(sensors)} sensors") + else: + logger.error(f"Failed to verify sensors: {response.status}") + + async with session.get(f"{SENSOR_SERVICE_URL}/rooms") as response: + if response.status == 200: + data = await response.json() + logger.info(f"✓ Total rooms in database: {data.get('count', 0)}") + else: + logger.error(f"Failed to verify rooms: {response.status}") + + except Exception as e: + logger.error(f"✗ Exception during verification: {e}") + +async def main(): + """Main bootstrap function""" + logger.info("=== Starting Sensor Bootstrap Process ===") + + try: + # Step 1: Generate authentication token + logger.info("Step 1: Generating authentication token...") + auth_token = await generate_auth_token() + + # Step 2: Create rooms + logger.info("Step 2: Creating rooms...") + rooms_success = await create_rooms(auth_token) + + # Step 3: Create sensors + logger.info("Step 3: Creating sensors...") + sensors_success = await create_sensors(auth_token) + + # Step 4: Verify results + logger.info("Step 4: Verifying bootstrap...") + await verify_bootstrap() + + if rooms_success and sensors_success: + logger.info("=== Bootstrap Complete! ===") + logger.info("You can now run the data simulator to generate real-time sensor data.") + logger.info("Run: python data_simulator_enhanced.py") + return True + else: + logger.error("=== Bootstrap Failed ===") + return False + + except Exception as e: + logger.error(f"Bootstrap failed with error: {e}") + return False + +if __name__ == "__main__": + # Run the bootstrap + success = asyncio.run(main()) + sys.exit(0 if success else 1) diff --git a/data_simulator_enhanced.py b/data_simulator_enhanced.py new file mode 100644 index 0000000..e88b1a3 --- /dev/null +++ b/data_simulator_enhanced.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +""" +Enhanced Data Simulator for Bootstrap Sensors +Generates realistic real-time sensor data for the bootstrap sensors created by bootstrap_sensors.py +""" + +import redis +import time +import random +import json +import logging +from datetime import datetime, timedelta +from typing import Dict, List, Any +import math + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Redis configuration +REDIS_HOST = 'localhost' +REDIS_PORT = 6379 +REDIS_CHANNEL = "energy_data" + +# Bootstrap sensor IDs (must match bootstrap_sensors.py) +BOOTSTRAP_SENSORS = { + # Living Room Sensors + "lr_energy_001": {"type": "energy", "room": "living_room", "base_value": 2.5, "variance": 1.2}, + "lr_co2_001": {"type": "co2", "room": "living_room", "base_value": 420, "variance": 80}, + "lr_temp_001": {"type": "temperature", "room": "living_room", "base_value": 22.0, "variance": 2.0}, + + # Kitchen Sensors + "kt_energy_001": {"type": "energy", "room": "kitchen", "base_value": 3.8, "variance": 2.1}, + "kt_humidity_001": {"type": "humidity", "room": "kitchen", "base_value": 45.0, "variance": 15.0}, + "kt_temp_001": {"type": "temperature", "room": "kitchen", "base_value": 24.0, "variance": 3.0}, + + # Bedroom Sensors + "br_energy_001": {"type": "energy", "room": "bedroom", "base_value": 1.2, "variance": 0.8}, + "br_co2_001": {"type": "co2", "room": "bedroom", "base_value": 480, "variance": 120}, + "br_temp_001": {"type": "temperature", "room": "bedroom", "base_value": 20.5, "variance": 1.5}, + + # Office Sensors + "of_energy_001": {"type": "energy", "room": "office", "base_value": 2.1, "variance": 1.5}, + "of_co2_001": {"type": "co2", "room": "office", "base_value": 450, "variance": 100}, + "of_motion_001": {"type": "motion", "room": "office", "base_value": 0, "variance": 1}, + + # Bathroom Sensors + "bt_humidity_001": {"type": "humidity", "room": "bathroom", "base_value": 65.0, "variance": 20.0}, + "bt_temp_001": {"type": "temperature", "room": "bathroom", "base_value": 23.0, "variance": 2.5}, + + # Garage Sensors + "gr_energy_001": {"type": "energy", "room": "garage", "base_value": 0.8, "variance": 0.5}, + "gr_motion_001": {"type": "motion", "room": "garage", "base_value": 0, "variance": 1} +} + +class SensorDataGenerator: + """Generates realistic sensor data with time-based patterns""" + + def __init__(self): + self.start_time = time.time() + self.motion_states = {} # Track motion sensor states + + # Initialize motion states + for sensor_id, config in BOOTSTRAP_SENSORS.items(): + if config["type"] == "motion": + self.motion_states[sensor_id] = {"active": False, "last_change": time.time()} + + def get_time_factor(self) -> float: + """Get time-based multiplier for realistic daily patterns""" + current_hour = datetime.now().hour + + # Energy usage patterns (higher during day, lower at night) + if 6 <= current_hour <= 22: # Daytime + return 1.0 + 0.3 * math.sin((current_hour - 6) * math.pi / 16) + else: # Nighttime + return 0.3 + 0.2 * random.random() + + def get_occupancy_factor(self, room: str) -> float: + """Get occupancy-based multiplier for different rooms""" + current_hour = datetime.now().hour + + occupancy_patterns = { + "living_room": 1.2 if 18 <= current_hour <= 23 else 0.8, + "kitchen": 1.5 if 7 <= current_hour <= 9 or 17 <= current_hour <= 20 else 0.6, + "bedroom": 1.3 if 22 <= current_hour or current_hour <= 7 else 0.4, + "office": 1.4 if 9 <= current_hour <= 17 else 0.3, + "bathroom": 1.0, # Consistent usage + "garage": 0.8 if 7 <= current_hour <= 9 or 17 <= current_hour <= 19 else 0.2 + } + + return occupancy_patterns.get(room, 1.0) + + def generate_energy_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]: + """Generate realistic energy consumption reading""" + base_value = config["base_value"] + variance = config["variance"] + room = config["room"] + + # Apply time and occupancy factors + time_factor = self.get_time_factor() + occupancy_factor = self.get_occupancy_factor(room) + + # Add some randomness + random_factor = 1.0 + (random.random() - 0.5) * 0.4 + + # Calculate final value + value = base_value * time_factor * occupancy_factor * random_factor + value = max(0.1, value) # Ensure minimum consumption + + return { + "sensor_id": sensor_id, + "room": room, + "sensor_type": "energy", + "timestamp": int(time.time()), + "energy": { + "value": round(value, 3), + "unit": "kWh" + }, + "metadata": { + "time_factor": round(time_factor, 2), + "occupancy_factor": round(occupancy_factor, 2) + } + } + + def generate_co2_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]: + """Generate realistic CO2 level reading""" + base_value = config["base_value"] + variance = config["variance"] + room = config["room"] + + # CO2 increases with occupancy + occupancy_factor = self.get_occupancy_factor(room) + co2_increase = (occupancy_factor - 0.5) * 150 + + # Add random fluctuation + random_variation = (random.random() - 0.5) * variance + + value = base_value + co2_increase + random_variation + value = max(350, min(2000, value)) # Realistic CO2 range + + return { + "sensor_id": sensor_id, + "room": room, + "sensor_type": "co2", + "timestamp": int(time.time()), + "co2": { + "value": round(value, 1), + "unit": "ppm" + }, + "metadata": { + "quality_level": "good" if value < 600 else "moderate" if value < 1000 else "poor" + } + } + + def generate_temperature_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]: + """Generate realistic temperature reading""" + base_value = config["base_value"] + variance = config["variance"] + room = config["room"] + + # Temperature varies with time of day and occupancy + current_hour = datetime.now().hour + daily_variation = 2 * math.sin((current_hour - 6) * math.pi / 12) + + occupancy_factor = self.get_occupancy_factor(room) + occupancy_heat = (occupancy_factor - 0.5) * 1.5 + + random_variation = (random.random() - 0.5) * variance + + value = base_value + daily_variation + occupancy_heat + random_variation + + return { + "sensor_id": sensor_id, + "room": room, + "sensor_type": "temperature", + "timestamp": int(time.time()), + "temperature": { + "value": round(value, 1), + "unit": "°C" + } + } + + def generate_humidity_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]: + """Generate realistic humidity reading""" + base_value = config["base_value"] + variance = config["variance"] + room = config["room"] + + # Humidity patterns based on room usage + if room == "bathroom": + # Higher spikes during usage times + current_hour = datetime.now().hour + if 7 <= current_hour <= 9 or 19 <= current_hour <= 22: + usage_spike = random.uniform(10, 25) + else: + usage_spike = 0 + elif room == "kitchen": + # Cooking increases humidity + current_hour = datetime.now().hour + if 17 <= current_hour <= 20: + usage_spike = random.uniform(5, 15) + else: + usage_spike = 0 + else: + usage_spike = 0 + + random_variation = (random.random() - 0.5) * variance + value = base_value + usage_spike + random_variation + value = max(20, min(95, value)) # Realistic humidity range + + return { + "sensor_id": sensor_id, + "room": room, + "sensor_type": "humidity", + "timestamp": int(time.time()), + "humidity": { + "value": round(value, 1), + "unit": "%" + } + } + + def generate_motion_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]: + """Generate realistic motion detection reading""" + room = config["room"] + current_time = time.time() + + # Get current state + if sensor_id not in self.motion_states: + self.motion_states[sensor_id] = {"active": False, "last_change": current_time} + + state = self.motion_states[sensor_id] + + # Determine if motion should be detected based on occupancy patterns + occupancy_factor = self.get_occupancy_factor(room) + motion_probability = occupancy_factor * 0.3 # 30% chance when occupied + + # Change state based on probability and time since last change + time_since_change = current_time - state["last_change"] + + if state["active"]: + # If motion is active, chance to stop after some time + if time_since_change > 30: # At least 30 seconds of motion + if random.random() < 0.4: # 40% chance to stop + state["active"] = False + state["last_change"] = current_time + else: + # If no motion, chance to start based on occupancy + if time_since_change > 10: # At least 10 seconds of no motion + if random.random() < motion_probability: + state["active"] = True + state["last_change"] = current_time + + return { + "sensor_id": sensor_id, + "room": room, + "sensor_type": "motion", + "timestamp": int(time.time()), + "motion": { + "value": 1 if state["active"] else 0, + "unit": "detected" + }, + "metadata": { + "duration_seconds": int(time_since_change) if state["active"] else 0 + } + } + + def generate_sensor_reading(self, sensor_id: str) -> Dict[str, Any]: + """Generate appropriate reading based on sensor type""" + if sensor_id not in BOOTSTRAP_SENSORS: + logger.warning(f"Unknown sensor ID: {sensor_id}") + return None + + config = BOOTSTRAP_SENSORS[sensor_id] + sensor_type = config["type"] + + if sensor_type == "energy": + return self.generate_energy_reading(sensor_id, config) + elif sensor_type == "co2": + return self.generate_co2_reading(sensor_id, config) + elif sensor_type == "temperature": + return self.generate_temperature_reading(sensor_id, config) + elif sensor_type == "humidity": + return self.generate_humidity_reading(sensor_id, config) + elif sensor_type == "motion": + return self.generate_motion_reading(sensor_id, config) + else: + logger.warning(f"Unknown sensor type: {sensor_type}") + return None + +def main(): + """Main simulation loop""" + logger.info("=== Starting Enhanced Data Simulator ===") + + # Connect to Redis + try: + redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=0, decode_responses=True) + redis_client.ping() + logger.info(f"Successfully connected to Redis at {REDIS_HOST}:{REDIS_PORT}") + except redis.exceptions.ConnectionError as e: + logger.error(f"Could not connect to Redis: {e}") + return + + # Initialize data generator + generator = SensorDataGenerator() + + logger.info(f"Loaded {len(BOOTSTRAP_SENSORS)} bootstrap sensors") + logger.info(f"Publishing to Redis channel: '{REDIS_CHANNEL}'") + logger.info("Press Ctrl+C to stop simulation") + + sensor_ids = list(BOOTSTRAP_SENSORS.keys()) + + try: + while True: + sensors_produced = [] + for a in range(5): + + # Generate data for a random sensor + sensor_id = random.choice(sensor_ids) + sensors_produced.append(sensor_id) + reading = generator.generate_sensor_reading(sensor_id) + + if reading: + # Publish to Redis + payload = json.dumps(reading) + redis_client.publish(REDIS_CHANNEL, payload) + + # Log the reading + sensor_type = reading["sensor_type"] + room = reading["room"] + value_info = "" + + if "energy" in reading: + value_info = f"{reading['energy']['value']} {reading['energy']['unit']}" + elif "co2" in reading: + value_info = f"{reading['co2']['value']} {reading['co2']['unit']}" + elif "temperature" in reading: + value_info = f"{reading['temperature']['value']} {reading['temperature']['unit']}" + elif "humidity" in reading: + value_info = f"{reading['humidity']['value']} {reading['humidity']['unit']}" + elif "motion" in reading: + value_info = f"{'DETECTED' if reading['motion']['value'] else 'CLEAR'}" + + logger.info(f"📊 {sensor_id} ({room}/{sensor_type}): {value_info}") + + # Random interval between readings (1-5 seconds) + time.sleep(random.uniform(1, 5)) + + except KeyboardInterrupt: + logger.info("Stopping data simulation...") + except Exception as e: + logger.error(f"Simulation error: {e}") + +if __name__ == "__main__": + main() diff --git a/microservices/tasks b/microservices/tasks new file mode 100644 index 0000000..aa90dc6 --- /dev/null +++ b/microservices/tasks @@ -0,0 +1,47 @@ +api-gateway +- Critical: Extend `SERVICES` and `service_requests` to include every routed microservice so proxy endpoints stop returning 404/KeyError for battery/demand-response/p2p/forecasting/iot routes (api-gateway/main.py:70-169). +- High: Guard `request_stats` updates against unknown services or look up configs dynamically before incrementing counters to avoid crashes (api-gateway/main.py:88-169). +- Possible Feature: Add per-service rate limiting and fallback routing with circuit breakers to keep the gateway responsive during downstream outages. +- Data to Store: Persist rolling latency/throughput metrics per backend plus authentication decision logs for audit and tuning. + +battery-service +- High: Handle zero or missing max charge/discharge power before dividing when optimising SOC to prevent ZeroDivisionError (battery-service/battery_service.py:205-213). +- Medium: Use the stored `capacity_kwh`/`stored_energy_kwh` fields when computing fleet totals so analytics reflect real values (battery-service/main.py:95-96). +- Possible Feature: Expose predictive maintenance recommendations based on usage profiles and integrate battery grouping/aggregation endpoints. +- Data to Store: Track per-cycle metadata (depth of discharge, temperatures) and maintenance events to support lifecycle analytics. + +data-ingestion-service +- High: Wrap the initial `wait_for(check_for_new_files)` call in error handling so startup connection/timeout issues don't kill the monitoring task (data-ingestion-service/src/ftp_monitor.py:62-100). +- Possible Feature: Provide a dashboard/REST endpoint for real-time ingestion status with manual retry controls and support for additional protocols (SFTP/HTTPS). +- Data to Store: Record per-file ingestion outcomes, error traces, and FTP scan history to analyse gaps and retry logic effectiveness. + +demand-response-service +- Critical: Restore the missing `models`, `database`, and `demand_response_service` modules referenced during import so the app can boot (demand-response-service/main.py:15-20). +- Possible Feature: Implement participant opt-in/opt-out scheduling, incentive tracking, and automated curtailment verification reports. +- Data to Store: Persist device participation history, response accuracy, and incentive payouts to evaluate program efficiency. + +forecasting-service +- Critical: Recreate the forecasting microservice implementation; the directory is empty so nothing can start (forecasting-service/). +- Possible Feature: Offer multiple forecast horizons with confidence intervals and expose model version management APIs. +- Data to Store: Keep training dataset metadata, forecast error metrics, and model artefacts to support retraining and auditing. + +iot-control-service +- Critical: Recreate the IoT control microservice implementation; the directory is empty so nothing can start (iot-control-service/). +- Possible Feature: Add device scheduling/policy engines with rules-based automation and rollback support for failed commands. +- Data to Store: Log all device capabilities, issued commands, acknowledgements, and firmware status to manage the fleet safely. + +p2p-trading-service +- Critical: Recreate the P2P trading microservice implementation; the directory is empty so nothing can start (p2p-trading-service/). +- Possible Feature: Build order-book style trading with price discovery, bidding windows, and settlement workflows. +- Data to Store: Capture trade offers, matched transactions, settlement receipts, and participant credit balances for compliance. + +sensor-service +- High: Fix the aggregation pipeline so sensor types are grouped correctly and room metrics use real readings instead of the constant "energy" fallback (sensor-service/room_service.py:408-420). +- Medium: Filter system events using comparable types (e.g., `created_at` or int timestamps) so queries return results (sensor-service/sensor_service.py:218-227). +- Possible Feature: Add anomaly detection on sensor streams and configurable alerting thresholds surfaced via dashboards/WebSockets. +- Data to Store: Maintain sensor calibration history, room occupancy patterns, and WebSocket subscription metrics for optimisation. + +token-service +- Medium: Stop forcibly overriding client-provided expiry/embargo values so custom lifetimes survive token generation (token-service/main.py:99-108). +- Possible Feature: Support role/permission templates with bulk token provisioning and self-service revocation flows. +- Data to Store: Persist token usage logs (IP, endpoint, timestamp) and refresh token metadata to improve security monitoring.