validate_modules.py

"""Validation Script - Tests all O2C AI modules end-to-end"""
import sys
import os
from pathlib import Path

# Configure UTF-8 encoding for Windows consoles
if sys.platform == "win32":
    try:
        sys.stdout.reconfigure(encoding="utf-8")
        sys.stderr.reconfigure(encoding="utf-8")
    except AttributeError:
        pass

# Databricks Workspace Path Support
DATABRICKS_PATH = Path("/Workspace/Users/ayyash.a@tcs.com/O2C_AI")
if DATABRICKS_PATH.exists() or "DATABRICKS_RUNTIME_VERSION" in os.environ:
    project_root = DATABRICKS_PATH
else:
    try:
        project_root = Path(__file__).resolve().parent
    except NameError:
        project_root = DATABRICKS_PATH

if str(project_root) not in sys.path:
    sys.path.insert(0, str(project_root))

print("=" * 80)
print("🔍 O2C AI MODULES VALIDATION SUITE")
print("=" * 80)

# Test 1: Import all modules
print("\n[1/7] Testing module imports...")
try:
    from modules import (
        DatabaseManager,
        WeatherService,
        NewsService,
        RAGEngine,
        DocumentLoader,
        TextChunker,
        VectorStore,
        RAGQueryEngine,
        WeatherPolicyGenerator,
        StrikeIntelligenceGenerator,
        MLDatabaseExtension,
        PredictiveEngine,
    )
    print("   ✅ All 12 core modules imported successfully")
except Exception as e:
    print(f"   ❌ Import failed: {e}")
    sys.exit(1)

# Test 2: Check configuration
print("\n[2/7] Testing configuration...")
try:
    from modules.config import (
        DB_PATH, DOCS_DIR, VECTOR_DIR, CHUNKS_DIR,
        BASE_DIR, RAG_DIR, EMBEDDING_MODEL, INDIA_CITIES
    )
    print(f"   ✅ Config loaded successfully")
    print(f"      - BASE_DIR: {BASE_DIR}")
    print(f"      - DB_PATH: {DB_PATH}")
    print(f"      - DOCS_DIR: {DOCS_DIR}")
    print(f"      - Embedding Model: {EMBEDDING_MODEL}")
    print(f"      - Cities Configured: {len(INDIA_CITIES)}")
except Exception as e:
    print(f"   ❌ Config failed: {e}")
    sys.exit(1)

# Test 3: Check database
print("\n[3/7] Testing database connection...")
try:
    db_manager = DatabaseManager()
    stats = db_manager.get_stats()
    print("   ✅ DatabaseManager initialized")
    print(f"      - Weather records: {stats.get('weather_records', 0)}")
    print(f"      - Strike articles: {stats.get('strike_articles', 0)}")
    print(f"      - RAG analyses: {stats.get('rag_analyses', 0)}")
except Exception as e:
    print(f"   ❌ DatabaseManager failed: {e}")

# Test 4: Check RAG document corpus structure
print("\n[4/7] Testing RAG document structure (78 docs target)...")
try:
    if DOCS_DIR.exists():
        categories = [d for d in DOCS_DIR.iterdir() if d.is_dir()]
        print(f"   ✅ DOCS_DIR exists: {DOCS_DIR}")
        print(f"      Found {len(categories)} document categories:")
        total_docs = 0
        for cat in sorted(categories, key=lambda x: x.name):
            doc_files = list(cat.glob("*.docx")) + list(cat.glob("*.pdf")) + list(cat.glob("*.txt"))
            doc_count = len(doc_files)
            total_docs += doc_count
            print(f"      - {cat.name:<32}: {doc_count:2d} documents")
        print(f"      TOTAL DOCUMENTS IN CORPUS: {total_docs}")
    else:
        print(f"   ⚠️  DOCS_DIR not found: {DOCS_DIR}")
except Exception as e:
    print(f"   ❌ Document structure check failed: {e}")

# Test 5: Check policy generators
print("\n[5/7] Testing policy & intelligence generators...")
try:
    weather_gen = WeatherPolicyGenerator()
    print(f"   ✅ WeatherPolicyGenerator initialized → {weather_gen.output_dir.name}")
    
    strike_gen = StrikeIntelligenceGenerator()
    print(f"   ✅ StrikeIntelligenceGenerator initialized → {strike_gen.output_dir.name}")
except Exception as e:
    print(f"   ❌ Generator initialization failed: {e}")

# Test 6: Check SAP Input Files
print("\n[6/7] Testing SAP data files in 'Input Files'...")
try:
    sap_dir = project_root / "Input Files"
    if sap_dir.exists():
        sap_files = list(sap_dir.glob("*.csv"))
        print(f"   ✅ SAP data directory exists: {sap_dir}")
        print(f"      Found {len(sap_files)} CSV tables:")
        for f in sorted(sap_files, key=lambda x: x.name):
            size_kb = f.stat().st_size / 1024
            print(f"      - {f.name:<12} ({size_kb:7.1f} KB)")
    else:
        print(f"   ⚠️  SAP data directory not found: {sap_dir}")
except Exception as e:
    print(f"   ❌ SAP data check failed: {e}")

# Test 7: Validate ML Engine A Integration
print("\n[7/7] Testing Engine A (ML Database Extension & Predictive Engine)...")
try:
    ml_db = MLDatabaseExtension()
    load_stats = ml_db.load_sap_data_from_csv(project_root / "Input Files")
    print(f"   ✅ SAP Tables Ingested into SQLite:")
    for tbl, count in sorted(load_stats.items()):
        print(f"      - {tbl.upper():<10}: {count:4d} rows")
    
    df_ml = ml_db.get_ml_ready_dataset()
    print(f"   ✅ ML Dataset generated: {len(df_ml)} rows, {len(df_ml.columns)} columns")
    
    pred_engine = PredictiveEngine(ml_db_extension=ml_db)
    trained = pred_engine.train_models(df_ml)
    print(f"   ✅ PredictiveEngine model training: {'SUCCESS' if trained else 'HEURISTICS FALLBACK'}")
    
    sample_order = df_ml.iloc[0]['order_id']
    sample_pred = pred_engine.predict_delivery_delay(sample_order)
    print(f"   ✅ Sample Prediction for Order {sample_order}:")
    print(f"      - Customer : {sample_pred['customer_name']} ({sample_pred['customer_tier']})")
    print(f"      - Delay Prob: {sample_pred['delay_probability']:.1%}")
    print(f"      - Delay Hrs : {sample_pred['delay_hours']:.1f} hrs")
    print(f"      - Risk ($)  : ${sample_pred['financial_risk_usd']:.2f}")
    print(f"      - Cause     : {sample_pred['root_cause']}")
    ml_db.close()
except Exception as e:
    print(f"   ❌ Engine A validation failed: {e}")
    import traceback
    traceback.print_exc()
    sys.exit(1)

print("\n" + "=" * 80)
print("🎉 ALL MODULES VALIDATED SUCCESSFULLY - ZERO ERRORS")
print("=" * 80)