aa07347604
- Add iterative directory scanning to prevent infinite recursion - Cache processed files in memory to avoid redundant database lookups - Skip already processed files using cache and database fallback - Add tests for skip logic and iterative scan behavior - Change logging for MongoDB connection and file storage to debug level - Clean up FastAPI app and remove redundant docstrings
123 lines
3.1 KiB
Python
123 lines
3.1 KiB
Python
from fastapi import FastAPI, HTTPException
|
|
from contextlib import asynccontextmanager
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
from ftp_monitor import FTPMonitor
|
|
from database import DatabaseManager
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ftp_monitor = None
|
|
db_manager = None
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
global ftp_monitor, db_manager
|
|
|
|
logger.info("Starting SA4CPS Data Ingestion Service...")
|
|
|
|
db_manager = DatabaseManager()
|
|
await db_manager.connect()
|
|
|
|
ftp_monitor = FTPMonitor(db_manager)
|
|
|
|
monitoring_task = asyncio.create_task(ftp_monitor.start_monitoring())
|
|
|
|
logger.info("Service started successfully")
|
|
|
|
yield
|
|
|
|
logger.info("Shutting down service...")
|
|
monitoring_task.cancel()
|
|
await db_manager.close()
|
|
logger.info("Service shutdown complete")
|
|
|
|
|
|
app = FastAPI(
|
|
title="SA4CPS Data Ingestion Service",
|
|
description="Monitors FTP server for .sgl_v2 files and stores data in MongoDB",
|
|
version="1.0.0",
|
|
lifespan=lifespan
|
|
)
|
|
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
return {
|
|
"service": "SA4CPS Data Ingestion Service",
|
|
"status": "running",
|
|
"timestamp": datetime.now().isoformat()
|
|
}
|
|
|
|
|
|
@app.get("/health")
|
|
async def health_check():
|
|
global ftp_monitor, db_manager
|
|
|
|
health_status = {
|
|
"service": "healthy",
|
|
"timestamp": datetime.now().isoformat(),
|
|
"database": "unknown",
|
|
"ftp_monitor": "unknown"
|
|
}
|
|
|
|
# Check database connection
|
|
if db_manager:
|
|
try:
|
|
await db_manager.ping()
|
|
health_status["database"] = "connected"
|
|
except Exception:
|
|
health_status["database"] = "disconnected"
|
|
health_status["service"] = "degraded"
|
|
|
|
# Check FTP monitor status
|
|
if ftp_monitor:
|
|
health_status["ftp_monitor"] = ftp_monitor.get_status()
|
|
health_status["last_check"] = ftp_monitor.get_last_check_time()
|
|
health_status["files_processed"] = ftp_monitor.get_processed_count()
|
|
|
|
return health_status
|
|
|
|
|
|
@app.get("/status")
|
|
async def get_status():
|
|
global ftp_monitor, db_manager
|
|
|
|
if not ftp_monitor:
|
|
raise HTTPException(status_code=503, detail="FTP monitor not initialized")
|
|
|
|
return {
|
|
"ftp_monitor": ftp_monitor.get_detailed_status(),
|
|
"database": await db_manager.get_stats() if db_manager else None,
|
|
"timestamp": datetime.now().isoformat()
|
|
}
|
|
|
|
|
|
@app.post("/trigger-check")
|
|
async def trigger_manual_check():
|
|
global ftp_monitor
|
|
|
|
if not ftp_monitor:
|
|
raise HTTPException(status_code=503, detail="FTP monitor not initialized")
|
|
|
|
try:
|
|
result = await ftp_monitor.check_for_new_files()
|
|
return {
|
|
"message": "Manual check completed",
|
|
"result": result,
|
|
"timestamp": datetime.now().isoformat()
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Manual check failed: {e}")
|
|
raise HTTPException(status_code=500, detail=f"Check failed: {str(e)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run("main:app", host="0.0.0.0", port=8008, reload=True)
|