Add data-ingestion-service for SA4CPS FTP integration

- Implement FTP monitoring and ingestion for SA4CPS .slg_v2 files - Add robust data processor with multi-format and unit inference support - Publish parsed data to Redis topics for real-time dashboard simulation - Include validation, monitoring, and auto-configuration scripts - Provide documentation and test scripts for SA4CPS integration
2025-09-10 14:43:30 +01:00
parent d4f280de93
commit 5fdce00e5d
16 changed files with 6353 additions and 0 deletions
--- a/microservices/data-ingestion-service/startup_sa4cps.py
+++ b/microservices/data-ingestion-service/startup_sa4cps.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Startup script to automatically configure SA4CPS data source
+Run this after the data-ingestion-service starts
+"""
+
+import asyncio
+import logging
+import sys
+import os
+from sa4cps_config import SA4CPSConfigurator
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+async def setup_sa4cps():
+    """Setup SA4CPS data source with environment variables"""
+    logger.info("Starting SA4CPS configuration setup...")
+    
+    configurator = SA4CPSConfigurator()
+    
+    # Get configuration from environment
+    ftp_host = os.getenv('FTP_SA4CPS_HOST', 'ftp.sa4cps.pt')
+    ftp_username = os.getenv('FTP_SA4CPS_USERNAME', 'anonymous')
+    ftp_password = os.getenv('FTP_SA4CPS_PASSWORD', '')
+    ftp_remote_path = os.getenv('FTP_SA4CPS_REMOTE_PATH', '/')
+    ftp_use_ssl = os.getenv('FTP_SA4CPS_USE_SSL', 'false').lower() == 'true'
+    
+    logger.info(f"Configuring SA4CPS FTP: {ftp_host} (user: {ftp_username})")
+    
+    # Create SA4CPS data source
+    result = await configurator.create_sa4cps_data_source(
+        username=ftp_username,
+        password=ftp_password,
+        remote_path=ftp_remote_path,
+        use_ssl=ftp_use_ssl
+    )
+    
+    if result['success']:
+        logger.info(f"✅ SA4CPS data source configured successfully: {result['source_id']}")
+        
+        # Test the connection
+        logger.info("Testing FTP connection...")
+        test_result = await configurator.test_sa4cps_connection()
+        
+        if test_result['success']:
+            logger.info(f"✅ FTP connection test successful - Found {test_result.get('files_found', 0)} files")
+            if test_result.get('file_list'):
+                logger.info(f"Sample files: {', '.join(test_result['file_list'][:3])}")
+        else:
+            logger.warning(f"⚠️  FTP connection test failed: {test_result['message']}")
+        
+        # Show status
+        status = await configurator.get_sa4cps_status()
+        logger.info(f"SA4CPS Status: {status.get('status', 'unknown')}")
+        logger.info(f"Topics: {', '.join(status.get('topics', []))}")
+        
+    else:
+        logger.error(f"❌ Failed to configure SA4CPS data source: {result['message']}")
+        return False
+    
+    return True
+
+async def main():
+    """Main function"""
+    try:
+        success = await setup_sa4cps()
+        if success:
+            logger.info("🎉 SA4CPS configuration completed successfully!")
+            sys.exit(0)
+        else:
+            logger.error("💥 SA4CPS configuration failed!")
+            sys.exit(1)
+    except Exception as e:
+        logger.error(f"💥 Error during SA4CPS setup: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    asyncio.run(main())