Implement iterative FTP scan and skip logic with processed file cache

- Add iterative directory scanning to prevent infinite recursion - Cache
processed files in memory to avoid redundant database lookups - Skip
already processed files using cache and database fallback - Add tests
for skip logic and iterative scan behavior - Change logging for MongoDB
connection and file storage to debug level - Clean up FastAPI app and
remove redundant docstrings

microservices/data-ingestion-service/src/database.py

@@ -58,7 +58,7 @@ class DatabaseManager:
         """Close MongoDB connection"""
         if self.client:
             self.client.close()
-            logger.info("MongoDB connection closed")
+            logger.debug("MongoDB connection closed")
 
     async def ping(self):
         """Test database connection"""
@@ -68,7 +68,7 @@ class DatabaseManager:
         try:
             # The ping command is cheap and does not require auth.
             self.client.admin.command('ping')
-            logger.info("MongoDB ping successful")
+            logger.debug("MongoDB ping successful")
         except ConnectionFailure as e:
             logger.error(f"MongoDB ping failed - Server not available: {e}")
             raise
@@ -121,7 +121,7 @@ class DatabaseManager:
             if records:
                 result = self.collections['energy_data'].insert_many(records)
                 inserted_count = len(result.inserted_ids)
-                logger.info(f"Stored {inserted_count} records from {filename}")
+                logger.debug(f"Stored {inserted_count} records from {filename}")
                 return True
 
             return False
@@ -163,6 +163,10 @@ class DatabaseManager:
             logger.error(f"Error getting processed files: {e}")
             return []
 
+    async def is_file_processed(self, filename: str) -> bool:
+        """Mock check if file is processed"""
+        return filename in await self.get_processed_files()
+
     async def get_file_info(self, filename: str) -> Optional[Dict[str, Any]]:
         """Get information about a specific file"""
         try: