Implement dynamic energy data collections per building
- Store energy data in separate MongoDB collections for each SLGs/Community/Building directory - Update FTP monitor and database manager to track directory paths and select appropriate collections - Add collection stats to database statistics API - Update sensor and token services for improved API consistency - Add 'rb' (rebuild and restart) option to deploy.sh script
This commit is contained in:
rafaeldpsilva
559
bootstrap_sensors.py
Normal file
559
bootstrap_sensors.py
Normal file
@@ -0,0 +1,559 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Bootstrap script to populate the database with sensors and room configurations.
|
||||
This script creates a realistic smart building sensor setup for testing and development.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any
|
||||
import aiohttp
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Base URLs
|
||||
API_BASE_URL = "http://localhost:8000"
|
||||
SENSOR_SERVICE_URL = "http://localhost:8007"
|
||||
|
||||
s = {
|
||||
'id': 'sensor_1',
|
||||
'name': 'Energy Monitor 1',
|
||||
'type': 'energy',
|
||||
'room': 'Conference Room A',
|
||||
'status': 'online',
|
||||
'lastSeen': Date.now() / 1000,
|
||||
'capabilities': {
|
||||
'monitoring': ['energy'],
|
||||
'actions': [],
|
||||
},
|
||||
'metadata': {
|
||||
'location': 'Wall mounted',
|
||||
'model': 'EM-100',
|
||||
'firmware': '2.1.0',
|
||||
},
|
||||
},{
|
||||
'id': 'sensor_2',
|
||||
'name': 'HVAC Controller 1',
|
||||
'type': 'hvac',
|
||||
'room': 'Conference Room A',
|
||||
'status': 'online',
|
||||
'lastSeen': Date.now() / 1000,
|
||||
'capabilities': {
|
||||
'monitoring': ['temperature', 'co2'],
|
||||
'actions': [
|
||||
{
|
||||
'id': 'temp_adjust',
|
||||
'name': 'Adjust Temperature',
|
||||
'type': 'adjust',
|
||||
'icon': '🌡️',
|
||||
'parameters': { min: 18, max: 28, step: 0.5 },
|
||||
},
|
||||
{
|
||||
'id': 'fan_speed',
|
||||
'name': 'Fan Speed',
|
||||
'type': 'adjust',
|
||||
'icon': '💨',
|
||||
'parameters': { min: 0, max: 5, step: 1 },
|
||||
},
|
||||
{ 'id': 'power_toggle', 'name': 'Power', 'type': 'toggle', 'icon': '⚡' },
|
||||
],
|
||||
},
|
||||
metadata: {
|
||||
location: 'Ceiling mounted',
|
||||
model: 'HVAC-200',
|
||||
firmware: '3.2.1',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'sensor_3',
|
||||
name: 'Smart Light Controller',
|
||||
type: 'lighting',
|
||||
room: 'Office Floor 1',
|
||||
status: 'online',
|
||||
lastSeen: Date.now() / 1000,
|
||||
capabilities: {
|
||||
monitoring: ['energy'],
|
||||
actions: [
|
||||
{
|
||||
id: 'brightness',
|
||||
name: 'Brightness',
|
||||
type: 'adjust',
|
||||
icon: '💡',
|
||||
parameters: { min: 0, max: 100, step: 5 },
|
||||
},
|
||||
{ id: 'power_toggle', name: 'Power', type: 'toggle', icon: '⚡' },
|
||||
{
|
||||
id: 'scene',
|
||||
name: 'Scene',
|
||||
type: 'adjust',
|
||||
icon: '🎨',
|
||||
parameters: { options: ['Work', 'Meeting', 'Presentation', 'Relax'] },
|
||||
},
|
||||
],
|
||||
},
|
||||
metadata: {
|
||||
location: 'Ceiling grid',
|
||||
model: 'SL-300',
|
||||
firmware: '1.5.2',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'sensor_4',
|
||||
name: 'CO2 Sensor',
|
||||
type: 'co2',
|
||||
room: 'Meeting Room 1',
|
||||
status: 'online',
|
||||
lastSeen: Date.now() / 1000,
|
||||
capabilities: {
|
||||
monitoring: ['co2', 'temperature', 'humidity'],
|
||||
actions: [{ id: 'calibrate', name: 'Calibrate', type: 'trigger', icon: '⚙️' }],
|
||||
},
|
||||
metadata: {
|
||||
location: 'Wall mounted',
|
||||
model: 'CO2-150',
|
||||
firmware: '2.0.3',
|
||||
battery: 85,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'sensor_5',
|
||||
name: 'Security Camera',
|
||||
type: 'security',
|
||||
room: 'Lobby',
|
||||
status: 'online',
|
||||
lastSeen: Date.now() / 1000,
|
||||
capabilities: {
|
||||
monitoring: ['motion'],
|
||||
actions: [
|
||||
{ id: 'record_toggle', name: 'Recording', type: 'toggle', icon: '📹' },
|
||||
{ id: 'ptz_control', name: 'Pan/Tilt/Zoom', type: 'trigger', icon: '🎥' },
|
||||
{ id: 'night_mode', name: 'Night Mode', type: 'toggle', icon: '🌙' },
|
||||
],
|
||||
},
|
||||
metadata: {
|
||||
location: 'Corner ceiling',
|
||||
model: 'SEC-400',
|
||||
firmware: '4.1.0',
|
||||
},
|
||||
},
|
||||
# Bootstrap data configuration
|
||||
BOOTSTRAP_SENSORS = [
|
||||
# Living Room Sensors
|
||||
{
|
||||
"sensor_id": "lr_energy_001",
|
||||
"name": "Living Room Main Energy Monitor",
|
||||
"sensor_type": "energy",
|
||||
"room": "living_room",
|
||||
"location": "Main electrical panel - Living Room circuit",
|
||||
"floor": "1",
|
||||
"manufacturer": "SmartMeter Co",
|
||||
"model": "SM-E300",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "lr_co2_001",
|
||||
"name": "Living Room Air Quality Monitor",
|
||||
"sensor_type": "co2",
|
||||
"room": "living_room",
|
||||
"location": "Wall mounted near seating area",
|
||||
"floor": "1",
|
||||
"manufacturer": "AirSense",
|
||||
"model": "AS-CO2-Pro",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "lr_temp_001",
|
||||
"name": "Living Room Temperature Sensor",
|
||||
"sensor_type": "temperature",
|
||||
"room": "living_room",
|
||||
"location": "Central wall position",
|
||||
"floor": "1",
|
||||
"manufacturer": "TempTech",
|
||||
"model": "TT-T200",
|
||||
"status": "online"
|
||||
},
|
||||
|
||||
# Kitchen Sensors
|
||||
{
|
||||
"sensor_id": "kt_energy_001",
|
||||
"name": "Kitchen Appliances Energy Monitor",
|
||||
"sensor_type": "energy",
|
||||
"room": "kitchen",
|
||||
"location": "Kitchen appliance circuit",
|
||||
"floor": "1",
|
||||
"manufacturer": "SmartMeter Co",
|
||||
"model": "SM-E300",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "kt_humidity_001",
|
||||
"name": "Kitchen Humidity Sensor",
|
||||
"sensor_type": "humidity",
|
||||
"room": "kitchen",
|
||||
"location": "Above sink area",
|
||||
"floor": "1",
|
||||
"manufacturer": "HumidSense",
|
||||
"model": "HS-H150",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "kt_temp_001",
|
||||
"name": "Kitchen Temperature Monitor",
|
||||
"sensor_type": "temperature",
|
||||
"room": "kitchen",
|
||||
"location": "Central kitchen position",
|
||||
"floor": "1",
|
||||
"manufacturer": "TempTech",
|
||||
"model": "TT-T200",
|
||||
"status": "online"
|
||||
},
|
||||
|
||||
# Bedroom Sensors
|
||||
{
|
||||
"sensor_id": "br_energy_001",
|
||||
"name": "Bedroom Energy Monitor",
|
||||
"sensor_type": "energy",
|
||||
"room": "bedroom",
|
||||
"location": "Bedroom electrical circuit",
|
||||
"floor": "1",
|
||||
"manufacturer": "SmartMeter Co",
|
||||
"model": "SM-E200",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "br_co2_001",
|
||||
"name": "Bedroom Air Quality Monitor",
|
||||
"sensor_type": "co2",
|
||||
"room": "bedroom",
|
||||
"location": "Bedside wall mount",
|
||||
"floor": "1",
|
||||
"manufacturer": "AirSense",
|
||||
"model": "AS-CO2-Basic",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "br_temp_001",
|
||||
"name": "Bedroom Temperature Sensor",
|
||||
"sensor_type": "temperature",
|
||||
"room": "bedroom",
|
||||
"location": "Opposite wall from bed",
|
||||
"floor": "1",
|
||||
"manufacturer": "TempTech",
|
||||
"model": "TT-T100",
|
||||
"status": "online"
|
||||
},
|
||||
|
||||
# Office Sensors
|
||||
{
|
||||
"sensor_id": "of_energy_001",
|
||||
"name": "Office Equipment Energy Monitor",
|
||||
"sensor_type": "energy",
|
||||
"room": "office",
|
||||
"location": "Office equipment circuit",
|
||||
"floor": "1",
|
||||
"manufacturer": "SmartMeter Co",
|
||||
"model": "SM-E300",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "of_co2_001",
|
||||
"name": "Office Air Quality Monitor",
|
||||
"sensor_type": "co2",
|
||||
"room": "office",
|
||||
"location": "Desk area wall mount",
|
||||
"floor": "1",
|
||||
"manufacturer": "AirSense",
|
||||
"model": "AS-CO2-Pro",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "of_motion_001",
|
||||
"name": "Office Motion Detector",
|
||||
"sensor_type": "motion",
|
||||
"room": "office",
|
||||
"location": "Ceiling mounted - center",
|
||||
"floor": "1",
|
||||
"manufacturer": "MotionTech",
|
||||
"model": "MT-M100",
|
||||
"status": "online"
|
||||
},
|
||||
|
||||
# Bathroom Sensors
|
||||
{
|
||||
"sensor_id": "bt_humidity_001",
|
||||
"name": "Bathroom Humidity Monitor",
|
||||
"sensor_type": "humidity",
|
||||
"room": "bathroom",
|
||||
"location": "Ceiling mounted",
|
||||
"floor": "1",
|
||||
"manufacturer": "HumidSense",
|
||||
"model": "HS-H200",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "bt_temp_001",
|
||||
"name": "Bathroom Temperature Sensor",
|
||||
"sensor_type": "temperature",
|
||||
"room": "bathroom",
|
||||
"location": "Wall mounted near mirror",
|
||||
"floor": "1",
|
||||
"manufacturer": "TempTech",
|
||||
"model": "TT-T150",
|
||||
"status": "online"
|
||||
},
|
||||
|
||||
# Garage Sensors
|
||||
{
|
||||
"sensor_id": "gr_energy_001",
|
||||
"name": "Garage Energy Monitor",
|
||||
"sensor_type": "energy",
|
||||
"room": "garage",
|
||||
"location": "Garage main circuit",
|
||||
"floor": "0",
|
||||
"manufacturer": "SmartMeter Co",
|
||||
"model": "SM-E100",
|
||||
"status": "online"
|
||||
},
|
||||
{
|
||||
"sensor_id": "gr_motion_001",
|
||||
"name": "Garage Motion Detector",
|
||||
"sensor_type": "motion",
|
||||
"room": "garage",
|
||||
"location": "Ceiling mounted - entrance",
|
||||
"floor": "0",
|
||||
"manufacturer": "MotionTech",
|
||||
"model": "MT-M200",
|
||||
"status": "online"
|
||||
}
|
||||
]
|
||||
|
||||
BOOTSTRAP_ROOMS = [
|
||||
{
|
||||
"name": "living_room",
|
||||
"display_name": "Living Room",
|
||||
"description": "Main living area with entertainment center",
|
||||
"floor": "1",
|
||||
"area_sqm": 35.5,
|
||||
"room_type": "living"
|
||||
},
|
||||
{
|
||||
"name": "kitchen",
|
||||
"display_name": "Kitchen",
|
||||
"description": "Main kitchen with appliances",
|
||||
"floor": "1",
|
||||
"area_sqm": 15.2,
|
||||
"room_type": "kitchen"
|
||||
},
|
||||
{
|
||||
"name": "bedroom",
|
||||
"display_name": "Master Bedroom",
|
||||
"description": "Primary bedroom",
|
||||
"floor": "1",
|
||||
"area_sqm": 20.1,
|
||||
"room_type": "bedroom"
|
||||
},
|
||||
{
|
||||
"name": "office",
|
||||
"display_name": "Home Office",
|
||||
"description": "Work from home office space",
|
||||
"floor": "1",
|
||||
"area_sqm": 12.8,
|
||||
"room_type": "office"
|
||||
},
|
||||
{
|
||||
"name": "bathroom",
|
||||
"display_name": "Main Bathroom",
|
||||
"description": "Primary bathroom",
|
||||
"floor": "1",
|
||||
"area_sqm": 8.5,
|
||||
"room_type": "bathroom"
|
||||
},
|
||||
{
|
||||
"name": "garage",
|
||||
"display_name": "Garage",
|
||||
"description": "Two-car garage with workshop area",
|
||||
"floor": "0",
|
||||
"area_sqm": 42.0,
|
||||
"room_type": "garage"
|
||||
}
|
||||
]
|
||||
|
||||
async def generate_auth_token() -> str:
|
||||
"""Generate authentication token for API calls"""
|
||||
token_payload = {
|
||||
"name": "bootstrap_user",
|
||||
"list_of_resources": ["sensors", "rooms", "analytics", "health", "data", "export", "events"],
|
||||
"data_aggregation": True,
|
||||
"time_aggregation": True,
|
||||
"embargo": 0,
|
||||
"exp_hours": 24
|
||||
}
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
f"{API_BASE_URL}/api/v1/tokens/generate",
|
||||
json=token_payload,
|
||||
headers={"Content-Type": "application/json"}
|
||||
) as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
logger.info("Successfully generated authentication token")
|
||||
return data["token"]
|
||||
else:
|
||||
error_text = await response.text()
|
||||
raise Exception(f"Failed to generate token: {response.status} - {error_text}")
|
||||
|
||||
async def create_rooms(auth_token: str) -> bool:
|
||||
"""Create rooms in the database"""
|
||||
logger.info("Creating bootstrap rooms...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
success_count = 0
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for room in BOOTSTRAP_ROOMS:
|
||||
try:
|
||||
async with session.post(
|
||||
f"{SENSOR_SERVICE_URL}/rooms",
|
||||
json=room,
|
||||
headers=headers
|
||||
) as response:
|
||||
if response.status in [200, 201]:
|
||||
success_count += 1
|
||||
logger.info(f"✓ Created room: {room['display_name']}")
|
||||
elif response.status == 400:
|
||||
# Room might already exist
|
||||
error_data = await response.json()
|
||||
if "already exists" in error_data.get("detail", ""):
|
||||
logger.info(f"○ Room already exists: {room['display_name']}")
|
||||
success_count += 1
|
||||
else:
|
||||
logger.error(f"✗ Failed to create room {room['name']}: {error_data}")
|
||||
else:
|
||||
error_text = await response.text()
|
||||
logger.error(f"✗ Failed to create room {room['name']}: {response.status} - {error_text}")
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Exception creating room {room['name']}: {e}")
|
||||
|
||||
logger.info(f"Rooms created: {success_count}/{len(BOOTSTRAP_ROOMS)}")
|
||||
return success_count > 0
|
||||
|
||||
async def create_sensors(auth_token: str) -> bool:
|
||||
"""Create sensors in the database"""
|
||||
logger.info("Creating bootstrap sensors...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
success_count = 0
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for sensor in BOOTSTRAP_SENSORS:
|
||||
try:
|
||||
async with session.post(
|
||||
f"{SENSOR_SERVICE_URL}/sensors",
|
||||
json=sensor,
|
||||
headers=headers
|
||||
) as response:
|
||||
if response.status in [200, 201]:
|
||||
success_count += 1
|
||||
logger.info(f"✓ Created sensor: {sensor['name']} ({sensor['sensor_id']})")
|
||||
elif response.status == 400:
|
||||
# Sensor might already exist
|
||||
error_data = await response.json()
|
||||
if "already exists" in error_data.get("detail", ""):
|
||||
logger.info(f"○ Sensor already exists: {sensor['sensor_id']}")
|
||||
success_count += 1
|
||||
else:
|
||||
logger.error(f"✗ Failed to create sensor {sensor['sensor_id']}: {error_data}")
|
||||
else:
|
||||
error_text = await response.text()
|
||||
logger.error(f"✗ Failed to create sensor {sensor['sensor_id']}: {response.status} - {error_text}")
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Exception creating sensor {sensor['sensor_id']}: {e}")
|
||||
|
||||
logger.info(f"Sensors created: {success_count}/{len(BOOTSTRAP_SENSORS)}")
|
||||
return success_count > 0
|
||||
|
||||
async def verify_bootstrap() -> None:
|
||||
"""Verify that sensors were created successfully"""
|
||||
logger.info("Verifying bootstrap results...")
|
||||
|
||||
try:
|
||||
# Check sensors directly on sensor service (no auth required for health checks)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f"{SENSOR_SERVICE_URL}/sensors/get") as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
logger.info(f"✓ Total sensors in database: {data['count']}")
|
||||
|
||||
# Group by room
|
||||
rooms = {}
|
||||
for sensor in data.get('sensors', []):
|
||||
room = sensor.get('room', 'unknown')
|
||||
if room not in rooms:
|
||||
rooms[room] = []
|
||||
rooms[room].append(sensor['sensor_id'])
|
||||
|
||||
for room, sensors in rooms.items():
|
||||
logger.info(f" - {room}: {len(sensors)} sensors")
|
||||
else:
|
||||
logger.error(f"Failed to verify sensors: {response.status}")
|
||||
|
||||
async with session.get(f"{SENSOR_SERVICE_URL}/rooms") as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
logger.info(f"✓ Total rooms in database: {data.get('count', 0)}")
|
||||
else:
|
||||
logger.error(f"Failed to verify rooms: {response.status}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"✗ Exception during verification: {e}")
|
||||
|
||||
async def main():
|
||||
"""Main bootstrap function"""
|
||||
logger.info("=== Starting Sensor Bootstrap Process ===")
|
||||
|
||||
try:
|
||||
# Step 1: Generate authentication token
|
||||
logger.info("Step 1: Generating authentication token...")
|
||||
auth_token = await generate_auth_token()
|
||||
|
||||
# Step 2: Create rooms
|
||||
logger.info("Step 2: Creating rooms...")
|
||||
rooms_success = await create_rooms(auth_token)
|
||||
|
||||
# Step 3: Create sensors
|
||||
logger.info("Step 3: Creating sensors...")
|
||||
sensors_success = await create_sensors(auth_token)
|
||||
|
||||
# Step 4: Verify results
|
||||
logger.info("Step 4: Verifying bootstrap...")
|
||||
await verify_bootstrap()
|
||||
|
||||
if rooms_success and sensors_success:
|
||||
logger.info("=== Bootstrap Complete! ===")
|
||||
logger.info("You can now run the data simulator to generate real-time sensor data.")
|
||||
logger.info("Run: python data_simulator_enhanced.py")
|
||||
return True
|
||||
else:
|
||||
logger.error("=== Bootstrap Failed ===")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Bootstrap failed with error: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run the bootstrap
|
||||
success = asyncio.run(main())
|
||||
sys.exit(0 if success else 1)
|
||||
354
data_simulator_enhanced.py
Normal file
354
data_simulator_enhanced.py
Normal file
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enhanced Data Simulator for Bootstrap Sensors
|
||||
Generates realistic real-time sensor data for the bootstrap sensors created by bootstrap_sensors.py
|
||||
"""
|
||||
|
||||
import redis
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Any
|
||||
import math
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis configuration
|
||||
REDIS_HOST = 'localhost'
|
||||
REDIS_PORT = 6379
|
||||
REDIS_CHANNEL = "energy_data"
|
||||
|
||||
# Bootstrap sensor IDs (must match bootstrap_sensors.py)
|
||||
BOOTSTRAP_SENSORS = {
|
||||
# Living Room Sensors
|
||||
"lr_energy_001": {"type": "energy", "room": "living_room", "base_value": 2.5, "variance": 1.2},
|
||||
"lr_co2_001": {"type": "co2", "room": "living_room", "base_value": 420, "variance": 80},
|
||||
"lr_temp_001": {"type": "temperature", "room": "living_room", "base_value": 22.0, "variance": 2.0},
|
||||
|
||||
# Kitchen Sensors
|
||||
"kt_energy_001": {"type": "energy", "room": "kitchen", "base_value": 3.8, "variance": 2.1},
|
||||
"kt_humidity_001": {"type": "humidity", "room": "kitchen", "base_value": 45.0, "variance": 15.0},
|
||||
"kt_temp_001": {"type": "temperature", "room": "kitchen", "base_value": 24.0, "variance": 3.0},
|
||||
|
||||
# Bedroom Sensors
|
||||
"br_energy_001": {"type": "energy", "room": "bedroom", "base_value": 1.2, "variance": 0.8},
|
||||
"br_co2_001": {"type": "co2", "room": "bedroom", "base_value": 480, "variance": 120},
|
||||
"br_temp_001": {"type": "temperature", "room": "bedroom", "base_value": 20.5, "variance": 1.5},
|
||||
|
||||
# Office Sensors
|
||||
"of_energy_001": {"type": "energy", "room": "office", "base_value": 2.1, "variance": 1.5},
|
||||
"of_co2_001": {"type": "co2", "room": "office", "base_value": 450, "variance": 100},
|
||||
"of_motion_001": {"type": "motion", "room": "office", "base_value": 0, "variance": 1},
|
||||
|
||||
# Bathroom Sensors
|
||||
"bt_humidity_001": {"type": "humidity", "room": "bathroom", "base_value": 65.0, "variance": 20.0},
|
||||
"bt_temp_001": {"type": "temperature", "room": "bathroom", "base_value": 23.0, "variance": 2.5},
|
||||
|
||||
# Garage Sensors
|
||||
"gr_energy_001": {"type": "energy", "room": "garage", "base_value": 0.8, "variance": 0.5},
|
||||
"gr_motion_001": {"type": "motion", "room": "garage", "base_value": 0, "variance": 1}
|
||||
}
|
||||
|
||||
class SensorDataGenerator:
|
||||
"""Generates realistic sensor data with time-based patterns"""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = time.time()
|
||||
self.motion_states = {} # Track motion sensor states
|
||||
|
||||
# Initialize motion states
|
||||
for sensor_id, config in BOOTSTRAP_SENSORS.items():
|
||||
if config["type"] == "motion":
|
||||
self.motion_states[sensor_id] = {"active": False, "last_change": time.time()}
|
||||
|
||||
def get_time_factor(self) -> float:
|
||||
"""Get time-based multiplier for realistic daily patterns"""
|
||||
current_hour = datetime.now().hour
|
||||
|
||||
# Energy usage patterns (higher during day, lower at night)
|
||||
if 6 <= current_hour <= 22: # Daytime
|
||||
return 1.0 + 0.3 * math.sin((current_hour - 6) * math.pi / 16)
|
||||
else: # Nighttime
|
||||
return 0.3 + 0.2 * random.random()
|
||||
|
||||
def get_occupancy_factor(self, room: str) -> float:
|
||||
"""Get occupancy-based multiplier for different rooms"""
|
||||
current_hour = datetime.now().hour
|
||||
|
||||
occupancy_patterns = {
|
||||
"living_room": 1.2 if 18 <= current_hour <= 23 else 0.8,
|
||||
"kitchen": 1.5 if 7 <= current_hour <= 9 or 17 <= current_hour <= 20 else 0.6,
|
||||
"bedroom": 1.3 if 22 <= current_hour or current_hour <= 7 else 0.4,
|
||||
"office": 1.4 if 9 <= current_hour <= 17 else 0.3,
|
||||
"bathroom": 1.0, # Consistent usage
|
||||
"garage": 0.8 if 7 <= current_hour <= 9 or 17 <= current_hour <= 19 else 0.2
|
||||
}
|
||||
|
||||
return occupancy_patterns.get(room, 1.0)
|
||||
|
||||
def generate_energy_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]:
|
||||
"""Generate realistic energy consumption reading"""
|
||||
base_value = config["base_value"]
|
||||
variance = config["variance"]
|
||||
room = config["room"]
|
||||
|
||||
# Apply time and occupancy factors
|
||||
time_factor = self.get_time_factor()
|
||||
occupancy_factor = self.get_occupancy_factor(room)
|
||||
|
||||
# Add some randomness
|
||||
random_factor = 1.0 + (random.random() - 0.5) * 0.4
|
||||
|
||||
# Calculate final value
|
||||
value = base_value * time_factor * occupancy_factor * random_factor
|
||||
value = max(0.1, value) # Ensure minimum consumption
|
||||
|
||||
return {
|
||||
"sensor_id": sensor_id,
|
||||
"room": room,
|
||||
"sensor_type": "energy",
|
||||
"timestamp": int(time.time()),
|
||||
"energy": {
|
||||
"value": round(value, 3),
|
||||
"unit": "kWh"
|
||||
},
|
||||
"metadata": {
|
||||
"time_factor": round(time_factor, 2),
|
||||
"occupancy_factor": round(occupancy_factor, 2)
|
||||
}
|
||||
}
|
||||
|
||||
def generate_co2_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]:
|
||||
"""Generate realistic CO2 level reading"""
|
||||
base_value = config["base_value"]
|
||||
variance = config["variance"]
|
||||
room = config["room"]
|
||||
|
||||
# CO2 increases with occupancy
|
||||
occupancy_factor = self.get_occupancy_factor(room)
|
||||
co2_increase = (occupancy_factor - 0.5) * 150
|
||||
|
||||
# Add random fluctuation
|
||||
random_variation = (random.random() - 0.5) * variance
|
||||
|
||||
value = base_value + co2_increase + random_variation
|
||||
value = max(350, min(2000, value)) # Realistic CO2 range
|
||||
|
||||
return {
|
||||
"sensor_id": sensor_id,
|
||||
"room": room,
|
||||
"sensor_type": "co2",
|
||||
"timestamp": int(time.time()),
|
||||
"co2": {
|
||||
"value": round(value, 1),
|
||||
"unit": "ppm"
|
||||
},
|
||||
"metadata": {
|
||||
"quality_level": "good" if value < 600 else "moderate" if value < 1000 else "poor"
|
||||
}
|
||||
}
|
||||
|
||||
def generate_temperature_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]:
|
||||
"""Generate realistic temperature reading"""
|
||||
base_value = config["base_value"]
|
||||
variance = config["variance"]
|
||||
room = config["room"]
|
||||
|
||||
# Temperature varies with time of day and occupancy
|
||||
current_hour = datetime.now().hour
|
||||
daily_variation = 2 * math.sin((current_hour - 6) * math.pi / 12)
|
||||
|
||||
occupancy_factor = self.get_occupancy_factor(room)
|
||||
occupancy_heat = (occupancy_factor - 0.5) * 1.5
|
||||
|
||||
random_variation = (random.random() - 0.5) * variance
|
||||
|
||||
value = base_value + daily_variation + occupancy_heat + random_variation
|
||||
|
||||
return {
|
||||
"sensor_id": sensor_id,
|
||||
"room": room,
|
||||
"sensor_type": "temperature",
|
||||
"timestamp": int(time.time()),
|
||||
"temperature": {
|
||||
"value": round(value, 1),
|
||||
"unit": "°C"
|
||||
}
|
||||
}
|
||||
|
||||
def generate_humidity_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]:
|
||||
"""Generate realistic humidity reading"""
|
||||
base_value = config["base_value"]
|
||||
variance = config["variance"]
|
||||
room = config["room"]
|
||||
|
||||
# Humidity patterns based on room usage
|
||||
if room == "bathroom":
|
||||
# Higher spikes during usage times
|
||||
current_hour = datetime.now().hour
|
||||
if 7 <= current_hour <= 9 or 19 <= current_hour <= 22:
|
||||
usage_spike = random.uniform(10, 25)
|
||||
else:
|
||||
usage_spike = 0
|
||||
elif room == "kitchen":
|
||||
# Cooking increases humidity
|
||||
current_hour = datetime.now().hour
|
||||
if 17 <= current_hour <= 20:
|
||||
usage_spike = random.uniform(5, 15)
|
||||
else:
|
||||
usage_spike = 0
|
||||
else:
|
||||
usage_spike = 0
|
||||
|
||||
random_variation = (random.random() - 0.5) * variance
|
||||
value = base_value + usage_spike + random_variation
|
||||
value = max(20, min(95, value)) # Realistic humidity range
|
||||
|
||||
return {
|
||||
"sensor_id": sensor_id,
|
||||
"room": room,
|
||||
"sensor_type": "humidity",
|
||||
"timestamp": int(time.time()),
|
||||
"humidity": {
|
||||
"value": round(value, 1),
|
||||
"unit": "%"
|
||||
}
|
||||
}
|
||||
|
||||
def generate_motion_reading(self, sensor_id: str, config: Dict) -> Dict[str, Any]:
|
||||
"""Generate realistic motion detection reading"""
|
||||
room = config["room"]
|
||||
current_time = time.time()
|
||||
|
||||
# Get current state
|
||||
if sensor_id not in self.motion_states:
|
||||
self.motion_states[sensor_id] = {"active": False, "last_change": current_time}
|
||||
|
||||
state = self.motion_states[sensor_id]
|
||||
|
||||
# Determine if motion should be detected based on occupancy patterns
|
||||
occupancy_factor = self.get_occupancy_factor(room)
|
||||
motion_probability = occupancy_factor * 0.3 # 30% chance when occupied
|
||||
|
||||
# Change state based on probability and time since last change
|
||||
time_since_change = current_time - state["last_change"]
|
||||
|
||||
if state["active"]:
|
||||
# If motion is active, chance to stop after some time
|
||||
if time_since_change > 30: # At least 30 seconds of motion
|
||||
if random.random() < 0.4: # 40% chance to stop
|
||||
state["active"] = False
|
||||
state["last_change"] = current_time
|
||||
else:
|
||||
# If no motion, chance to start based on occupancy
|
||||
if time_since_change > 10: # At least 10 seconds of no motion
|
||||
if random.random() < motion_probability:
|
||||
state["active"] = True
|
||||
state["last_change"] = current_time
|
||||
|
||||
return {
|
||||
"sensor_id": sensor_id,
|
||||
"room": room,
|
||||
"sensor_type": "motion",
|
||||
"timestamp": int(time.time()),
|
||||
"motion": {
|
||||
"value": 1 if state["active"] else 0,
|
||||
"unit": "detected"
|
||||
},
|
||||
"metadata": {
|
||||
"duration_seconds": int(time_since_change) if state["active"] else 0
|
||||
}
|
||||
}
|
||||
|
||||
def generate_sensor_reading(self, sensor_id: str) -> Dict[str, Any]:
|
||||
"""Generate appropriate reading based on sensor type"""
|
||||
if sensor_id not in BOOTSTRAP_SENSORS:
|
||||
logger.warning(f"Unknown sensor ID: {sensor_id}")
|
||||
return None
|
||||
|
||||
config = BOOTSTRAP_SENSORS[sensor_id]
|
||||
sensor_type = config["type"]
|
||||
|
||||
if sensor_type == "energy":
|
||||
return self.generate_energy_reading(sensor_id, config)
|
||||
elif sensor_type == "co2":
|
||||
return self.generate_co2_reading(sensor_id, config)
|
||||
elif sensor_type == "temperature":
|
||||
return self.generate_temperature_reading(sensor_id, config)
|
||||
elif sensor_type == "humidity":
|
||||
return self.generate_humidity_reading(sensor_id, config)
|
||||
elif sensor_type == "motion":
|
||||
return self.generate_motion_reading(sensor_id, config)
|
||||
else:
|
||||
logger.warning(f"Unknown sensor type: {sensor_type}")
|
||||
return None
|
||||
|
||||
def main():
|
||||
"""Main simulation loop"""
|
||||
logger.info("=== Starting Enhanced Data Simulator ===")
|
||||
|
||||
# Connect to Redis
|
||||
try:
|
||||
redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=0, decode_responses=True)
|
||||
redis_client.ping()
|
||||
logger.info(f"Successfully connected to Redis at {REDIS_HOST}:{REDIS_PORT}")
|
||||
except redis.exceptions.ConnectionError as e:
|
||||
logger.error(f"Could not connect to Redis: {e}")
|
||||
return
|
||||
|
||||
# Initialize data generator
|
||||
generator = SensorDataGenerator()
|
||||
|
||||
logger.info(f"Loaded {len(BOOTSTRAP_SENSORS)} bootstrap sensors")
|
||||
logger.info(f"Publishing to Redis channel: '{REDIS_CHANNEL}'")
|
||||
logger.info("Press Ctrl+C to stop simulation")
|
||||
|
||||
sensor_ids = list(BOOTSTRAP_SENSORS.keys())
|
||||
|
||||
try:
|
||||
while True:
|
||||
sensors_produced = []
|
||||
for a in range(5):
|
||||
|
||||
# Generate data for a random sensor
|
||||
sensor_id = random.choice(sensor_ids)
|
||||
sensors_produced.append(sensor_id)
|
||||
reading = generator.generate_sensor_reading(sensor_id)
|
||||
|
||||
if reading:
|
||||
# Publish to Redis
|
||||
payload = json.dumps(reading)
|
||||
redis_client.publish(REDIS_CHANNEL, payload)
|
||||
|
||||
# Log the reading
|
||||
sensor_type = reading["sensor_type"]
|
||||
room = reading["room"]
|
||||
value_info = ""
|
||||
|
||||
if "energy" in reading:
|
||||
value_info = f"{reading['energy']['value']} {reading['energy']['unit']}"
|
||||
elif "co2" in reading:
|
||||
value_info = f"{reading['co2']['value']} {reading['co2']['unit']}"
|
||||
elif "temperature" in reading:
|
||||
value_info = f"{reading['temperature']['value']} {reading['temperature']['unit']}"
|
||||
elif "humidity" in reading:
|
||||
value_info = f"{reading['humidity']['value']} {reading['humidity']['unit']}"
|
||||
elif "motion" in reading:
|
||||
value_info = f"{'DETECTED' if reading['motion']['value'] else 'CLEAR'}"
|
||||
|
||||
logger.info(f"📊 {sensor_id} ({room}/{sensor_type}): {value_info}")
|
||||
|
||||
# Random interval between readings (1-5 seconds)
|
||||
time.sleep(random.uniform(1, 5))
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Stopping data simulation...")
|
||||
except Exception as e:
|
||||
logger.error(f"Simulation error: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
47
microservices/tasks
Normal file
47
microservices/tasks
Normal file
@@ -0,0 +1,47 @@
|
||||
api-gateway
|
||||
- Critical: Extend `SERVICES` and `service_requests` to include every routed microservice so proxy endpoints stop returning 404/KeyError for battery/demand-response/p2p/forecasting/iot routes (api-gateway/main.py:70-169).
|
||||
- High: Guard `request_stats` updates against unknown services or look up configs dynamically before incrementing counters to avoid crashes (api-gateway/main.py:88-169).
|
||||
- Possible Feature: Add per-service rate limiting and fallback routing with circuit breakers to keep the gateway responsive during downstream outages.
|
||||
- Data to Store: Persist rolling latency/throughput metrics per backend plus authentication decision logs for audit and tuning.
|
||||
|
||||
battery-service
|
||||
- High: Handle zero or missing max charge/discharge power before dividing when optimising SOC to prevent ZeroDivisionError (battery-service/battery_service.py:205-213).
|
||||
- Medium: Use the stored `capacity_kwh`/`stored_energy_kwh` fields when computing fleet totals so analytics reflect real values (battery-service/main.py:95-96).
|
||||
- Possible Feature: Expose predictive maintenance recommendations based on usage profiles and integrate battery grouping/aggregation endpoints.
|
||||
- Data to Store: Track per-cycle metadata (depth of discharge, temperatures) and maintenance events to support lifecycle analytics.
|
||||
|
||||
data-ingestion-service
|
||||
- High: Wrap the initial `wait_for(check_for_new_files)` call in error handling so startup connection/timeout issues don't kill the monitoring task (data-ingestion-service/src/ftp_monitor.py:62-100).
|
||||
- Possible Feature: Provide a dashboard/REST endpoint for real-time ingestion status with manual retry controls and support for additional protocols (SFTP/HTTPS).
|
||||
- Data to Store: Record per-file ingestion outcomes, error traces, and FTP scan history to analyse gaps and retry logic effectiveness.
|
||||
|
||||
demand-response-service
|
||||
- Critical: Restore the missing `models`, `database`, and `demand_response_service` modules referenced during import so the app can boot (demand-response-service/main.py:15-20).
|
||||
- Possible Feature: Implement participant opt-in/opt-out scheduling, incentive tracking, and automated curtailment verification reports.
|
||||
- Data to Store: Persist device participation history, response accuracy, and incentive payouts to evaluate program efficiency.
|
||||
|
||||
forecasting-service
|
||||
- Critical: Recreate the forecasting microservice implementation; the directory is empty so nothing can start (forecasting-service/).
|
||||
- Possible Feature: Offer multiple forecast horizons with confidence intervals and expose model version management APIs.
|
||||
- Data to Store: Keep training dataset metadata, forecast error metrics, and model artefacts to support retraining and auditing.
|
||||
|
||||
iot-control-service
|
||||
- Critical: Recreate the IoT control microservice implementation; the directory is empty so nothing can start (iot-control-service/).
|
||||
- Possible Feature: Add device scheduling/policy engines with rules-based automation and rollback support for failed commands.
|
||||
- Data to Store: Log all device capabilities, issued commands, acknowledgements, and firmware status to manage the fleet safely.
|
||||
|
||||
p2p-trading-service
|
||||
- Critical: Recreate the P2P trading microservice implementation; the directory is empty so nothing can start (p2p-trading-service/).
|
||||
- Possible Feature: Build order-book style trading with price discovery, bidding windows, and settlement workflows.
|
||||
- Data to Store: Capture trade offers, matched transactions, settlement receipts, and participant credit balances for compliance.
|
||||
|
||||
sensor-service
|
||||
- High: Fix the aggregation pipeline so sensor types are grouped correctly and room metrics use real readings instead of the constant "energy" fallback (sensor-service/room_service.py:408-420).
|
||||
- Medium: Filter system events using comparable types (e.g., `created_at` or int timestamps) so queries return results (sensor-service/sensor_service.py:218-227).
|
||||
- Possible Feature: Add anomaly detection on sensor streams and configurable alerting thresholds surfaced via dashboards/WebSockets.
|
||||
- Data to Store: Maintain sensor calibration history, room occupancy patterns, and WebSocket subscription metrics for optimisation.
|
||||
|
||||
token-service
|
||||
- Medium: Stop forcibly overriding client-provided expiry/embargo values so custom lifetimes survive token generation (token-service/main.py:99-108).
|
||||
- Possible Feature: Support role/permission templates with bulk token provisioning and self-service revocation flows.
|
||||
- Data to Store: Persist token usage logs (IP, endpoint, timestamp) and refresh token metadata to improve security monitoring.
|
||||
Reference in New Issue
Block a user