#!/usr/bin/env python3
"""
RuralPoint - cPanel Setup Script
Run this script in cPanel's "Execute python script" feature
"""

import os
import sys
import subprocess
import django
import io
from django.contrib.auth import get_user_model
from django.contrib.auth.hashers import make_password
from decimal import Decimal

def run_command(command):
    """Run a command and print output"""
    print(f"Running: {command}")
    try:
        result = subprocess.run(command, shell=True, capture_output=True, text=True)
        print(f"Output: {result.stdout}")
        if result.stderr:
            print(f"Errors: {result.stderr}")
        return result.returncode == 0
    except Exception as e:
        print(f"Error running command: {e}")
        return False

def install_requirements():
    """Install requirements with better error handling"""
    print("\n📦 Installing requirements...")
    
    # First try installing from requirements.txt
    success = run_command("pip install -r requirements.txt")
    
    if not success:
        print("❌ Failed to install from requirements.txt. Installing packages individually...")
        
        # Essential packages for Django
        essential_packages = [
            "Django>=5.2.4",
            "Pillow>=10.0.0",
            "pymysql>=1.0.2",
            "python-magic>=0.4.27",
            "python-magic-bin>=0.4.14",
            "reportlab>=4.0.7",
            "openpyxl>=3.1.2",
            "xlsxwriter>=3.1.9",
            "django-phonenumber-field>=7.3.0",
            "phonenumbers>=8.13.25",
            "matplotlib>=3.7.0",  # Added for chart generation
            "seaborn>=0.12.0",  # Added for enhanced chart styling
            "numpy>=1.24.0",  # Added for numerical operations in reports
            "cryptography>=41.0.0",  # Added for security features
        ]
        
        # Optional packages (continue if these fail)
        optional_packages = [
            "python-dotenv>=1.0.0",
            "django-crispy-forms>=2.0",
            "crispy-tailwind>=0.5.0",
            "django-tailwind>=3.6.0",
            "django-browser-reload>=1.11.0",
            "whitenoise>=6.5.0",
            "python-dateutil>=2.8.2",
            "requests>=2.31.0",
        ]
        
        # Install essential packages
        failed_essential = []
        for package in essential_packages:
            if not run_command(f"pip install {package}"):
                failed_essential.append(package)
        
        if failed_essential:
            print(f"❌ Failed to install essential packages: {failed_essential}")
            print("⚠️ You may need to install these manually or contact your hosting provider")
        
        # Install optional packages
        for package in optional_packages:
            run_command(f"pip install {package}")
    
    # Special handling for python-magic on different systems
    print("\n🔧 Ensuring python-magic is properly installed...")
    run_command("pip install python-magic")
    run_command("pip install python-magic-bin")  # For Windows/some hosting environments
    
    print("✓ Requirements installation completed")

def create_production_settings():
    """Create production settings file"""
    print("\n⚙️ Creating production settings...")
    
    settings_content = """from .settings import *

# Security Settings
DEBUG = False
ALLOWED_HOSTS = ['ruralpoint.co.ke', 'www.ruralpoint.co.ke']

# Database Settings
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.mysql',
        'NAME': os.getenv('DB_NAME', 'acbptxvs_ruralpoint'),
        'USER': os.getenv('DB_USER', 'acbptxvs_ruralpoint'),
        'PASSWORD': os.getenv('DB_PASSWORD', '}bxqyR_xfHu7!]B'),
        'HOST': os.getenv('DB_HOST', 'localhost'),
        'PORT': os.getenv('DB_PORT', '3306'),
        'OPTIONS': {
            'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",
            'charset': 'utf8mb4',
        }
    }
}

# Security Settings
SECRET_KEY = os.getenv('SECRET_KEY', '(W6B1[bCEAu,1wA[dz7^')
SECURE_SSL_REDIRECT = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
SESSION_COOKIE_SECURE = True
CSRF_COOKIE_SECURE = True
SECURE_HSTS_SECONDS = 31536000  # 1 year
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_HSTS_PRELOAD = True

# Login URLs
LOGIN_URL = '/login/'
LOGIN_REDIRECT_URL = '/dashboard/'
LOGOUT_REDIRECT_URL = '/login/'

# Session Settings
SESSION_COOKIE_AGE = 3600  # 1 hour
SESSION_EXPIRE_AT_BROWSER_CLOSE = True
SESSION_SAVE_EVERY_REQUEST = True
SESSION_COOKIE_HTTPONLY = True

# Static and Media Files
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'

# Email Settings
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_HOST = os.getenv('EMAIL_HOST', 'smtp.ruralpoint.co.ke')
EMAIL_PORT = int(os.getenv('EMAIL_PORT', '587'))
EMAIL_USE_TLS = True
EMAIL_HOST_USER = os.getenv('EMAIL_USER', 'support@ruralpoint.co.ke')
EMAIL_HOST_PASSWORD = os.getenv('EMAIL_PASSWORD', '')
DEFAULT_FROM_EMAIL = 'support@ruralpoint.co.ke'

# Audit Logging
AUDIT_LOG_ENABLED = True
AUDIT_LOG_EXCLUDE_URLS = [
    '/static/',
    '/media/',
    '/favicon.ico',
]

# Tailwind Settings
TAILWIND_APP_NAME = 'theme'
INTERNAL_IPS = ['127.0.0.1']

# Crispy Forms
CRISPY_ALLOWED_TEMPLATE_PACKS = 'tailwind'
CRISPY_TEMPLATE_PACK = 'tailwind'

# Logging
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}',
            'style': '{',
        },
    },
    'handlers': {
        'file': {
            'level': 'ERROR',
            'class': 'logging.FileHandler',
            'filename': os.path.join(BASE_DIR, 'logs', 'django.log'),
            'formatter': 'verbose',
        },
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
        },
    },
    'loggers': {
        'django': {
            'handlers': ['file', 'mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    },
}
"""
    
    with open('ruralpoint_system/settings_production.py', 'w') as f:
        f.write(settings_content)
    print("✓ Production settings created")


def update_passenger_wsgi():
    """Update passenger_wsgi.py"""
    print("\n🔄 Updating passenger_wsgi.py...")
    
    content = """import os
import sys

# Add the project directory to the Python path
sys.path.insert(0, os.path.dirname(__file__))

# Set production settings
os.environ["DJANGO_SETTINGS_MODULE"] = "ruralpoint_system.settings_production"

# Import and create WSGI application
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
"""
    
    with open('passenger_wsgi.py', 'w') as f:
        f.write(content)
    print("✓ passenger_wsgi.py updated")

def verify_admin_user():
    """Create or verify admin user"""
    print("\n👤 Verifying admin user...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        User = get_user_model()
        email = 'admin@ruralpoint.co.ke'
        password = 'admin123'
        
        try:
            user = User.objects.get(email=email)
            print(f"Found existing user: {email}")
            # Update existing admin user with new role system
            user.password = make_password(password)
            user.is_staff = True
            user.is_superuser = True
            user.role = 'admin'  # Ensure role is set to admin
            user.status = 'active'
            user.is_phone_verified = True
            user.is_email_verified = True
            user.save()
            print("✓ Admin user updated with new role system")
        except User.DoesNotExist:
            User.objects.create_superuser(
                username='admin',
                email=email,
                password=password,
                role='admin',
                status='active',
                is_phone_verified=True,
                is_email_verified=True,
                first_name='System',
                last_name='Administrator'
            )
            print("✓ Admin user created")
    except Exception as e:
        print(f"❌ Error with admin user: {e}")

def setup_role_permissions():
    """Setup role-based permissions system"""
    print("\n🔐 Setting up role-based permissions...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        from users.models import RolePermission
        
        # Clear existing permissions to avoid duplicates
        existing_count = RolePermission.objects.count()
        if existing_count > 0:
            print(f"Found {existing_count} existing permissions, updating...")
            RolePermission.objects.all().delete()
        
        # Define default permissions for each role
        permissions = {
            'admin': {
                # Admin has all permissions
                'users': ['view', 'create', 'edit', 'delete', 'manage'],
                'loans': ['view', 'create', 'edit', 'delete', 'approve', 'reject', 'manage'],
                'reports': ['view', 'create', 'edit', 'delete', 'export', 'manage'],
                'documents': ['view', 'create', 'edit', 'delete', 'manage'],
                'settings': ['view', 'create', 'edit', 'delete', 'manage'],
                'notifications': ['view', 'create', 'edit', 'delete', 'manage'],
                'audit': ['view', 'create', 'edit', 'delete', 'export', 'manage'],
                'kyc': ['view', 'create', 'edit', 'delete', 'approve', 'reject', 'manage'],
                'payments': ['view', 'create', 'edit', 'delete', 'approve', 'reject', 'manage'],
                'communications': ['view', 'create', 'edit', 'delete', 'manage'],
            },
            'team_leader': {
                # Team Leader has broad permissions but can't manage system settings
                'users': ['view', 'create', 'edit'],  # Can't delete users
                'loans': ['view', 'create', 'edit', 'approve', 'reject'],
                'reports': ['view', 'create', 'edit', 'export'],
                'documents': ['view', 'create', 'edit'],
                'settings': ['view'],  # Can only view settings
                'notifications': ['view', 'create', 'edit'],
                'audit': ['view', 'export'],
                'kyc': ['view', 'create', 'edit', 'approve', 'reject'],
                'payments': ['view', 'create', 'edit', 'approve', 'reject'],
                'communications': ['view', 'create', 'edit'],
            },
            'loan_officer': {
                # Loan Officer focuses on loan processing and client management
                'users': ['view', 'create', 'edit'],  # Can manage clients
                'loans': ['view', 'create', 'edit', 'approve', 'reject'],
                'reports': ['view', 'export'],
                'documents': ['view', 'create', 'edit'],
                'settings': [],  # No access to settings
                'notifications': ['view', 'create'],
                'audit': ['view'],
                'kyc': ['view', 'create', 'edit', 'approve', 'reject'],
                'payments': ['view', 'create', 'edit'],
                'communications': ['view', 'create'],
            },
            'secretary': {
                # Secretary handles administrative tasks and document management
                'users': ['view', 'create', 'edit'],  # Can manage client records
                'loans': ['view', 'create', 'edit'],  # Can't approve/reject
                'reports': ['view', 'export'],
                'documents': ['view', 'create', 'edit'],
                'settings': [],  # No access to settings
                'notifications': ['view', 'create'],
                'audit': ['view'],
                'kyc': ['view', 'create', 'edit'],
                'payments': ['view', 'create'],
                'communications': ['view', 'create'],
            },
            'auditor': {
                # Auditor has read-only access to most modules
                'users': ['view'],
                'loans': ['view'],
                'reports': ['view', 'export'],
                'documents': ['view'],
                'settings': ['view'],
                'notifications': ['view'],
                'audit': ['view', 'export'],
                'kyc': ['view'],
                'payments': ['view'],
                'communications': ['view'],
            },
            'borrower': {
                # Borrowers have very limited access
                'users': ['view'],  # Can only view their own profile
                'loans': ['view'],  # Can only view their own loans
                'reports': [],  # No access to reports
                'documents': ['view'],  # Can only view their own documents
                'settings': [],  # No access to settings
                'notifications': ['view'],
                'audit': [],  # No access to audit logs
                'kyc': ['view', 'create', 'edit'],  # Can manage their own KYC
                'payments': ['view'],  # Can only view their own payments
                'communications': ['view'],
            }
        }
        
        # Create permissions
        created_count = 0
        for role, modules in permissions.items():
            for module, actions in modules.items():
                for action in actions:
                    RolePermission.objects.get_or_create(
                        role=role,
                        module=module,
                        action=action,
                        defaults={'is_allowed': True}
                    )
                    created_count += 1
        
        print(f"✓ Successfully set up {created_count} role permissions")
        
        # Display summary
        for role in permissions.keys():
            count = RolePermission.objects.filter(role=role).count()
            print(f"   {role.title()}: {count} permissions")
            
    except Exception as e:
        print(f"❌ Error setting up role permissions: {e}")

def create_default_settings():
    """Create default system settings"""
    print("\n⚙️ Creating default system settings...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        from utils.models import SystemSetting
        
        settings_data = [
            # Loan Settings
            {'key': 'max_loan_amount', 'value': '500000', 'category': 'loan', 'description': 'Maximum loan amount allowed'},
            {'key': 'min_loan_amount', 'value': '1000', 'category': 'loan', 'description': 'Minimum loan amount allowed'},
            {'key': 'default_interest_rate', 'value': '15.0', 'category': 'loan', 'description': 'Default interest rate percentage'},
            {'key': 'processing_fee_percentage', 'value': '3.0', 'category': 'loan', 'description': 'Processing fee percentage'},
            {'key': 'late_payment_penalty', 'value': '2.0', 'category': 'loan', 'description': 'Late payment penalty percentage'},
            
            # Document Settings
            {'key': 'max_file_size_mb', 'value': '10', 'category': 'document', 'description': 'Maximum file upload size in MB'},
            {'key': 'allowed_file_types', 'value': '.pdf,.doc,.docx,.jpg,.jpeg,.png', 'category': 'document', 'description': 'Allowed file types for uploads'},
            {'key': 'require_kyc_documents', 'value': 'true', 'category': 'document', 'description': 'Whether KYC documents are required'},
            
            # System Settings
            {'key': 'session_timeout_minutes', 'value': '60', 'category': 'system', 'description': 'Session timeout in minutes'},
            {'key': 'max_login_attempts', 'value': '5', 'category': 'system', 'description': 'Maximum login attempts before lockout'},
            
            # Notification Settings
            {'key': 'email_notifications_enabled', 'value': 'true', 'category': 'notification', 'description': 'Enable email notifications'},
            {'key': 'sms_notifications_enabled', 'value': 'true', 'category': 'notification', 'description': 'Enable SMS notifications'},
            {'key': 'push_notifications_enabled', 'value': 'false', 'category': 'notification', 'description': 'Enable push notifications'},
            
            # Auto-approval Settings
            {'key': 'auto_approval_min_score', 'value': '80', 'category': 'loan', 'description': 'Minimum credit score for auto-approval'},
            {'key': 'auto_approval_max_amount', 'value': '25000', 'category': 'loan', 'description': 'Maximum amount for auto-approval'},
            
            # Rollover Settings
            {'key': 'rollover_fee_percentage', 'value': '10.0', 'category': 'loan', 'description': 'Default rollover fee percentage'},
            {'key': 'max_rollover_count', 'value': '3', 'category': 'loan', 'description': 'Maximum number of rollovers allowed'},
            {'key': 'max_rollover_days', 'value': '30', 'category': 'loan', 'description': 'Maximum days for rollover extension'},
            {'key': 'rollover_approval_required', 'value': 'true', 'category': 'loan', 'description': 'Whether rollovers require manual approval'}
        ]
        
        created_count = 0
        for setting_data in settings_data:
            setting, created = SystemSetting.objects.get_or_create(
                key=setting_data['key'],
                defaults=setting_data
            )
            if created:
                created_count += 1
        
        print(f"✓ Created {created_count} default system settings")
        
    except Exception as e:
        print(f"❌ Error creating default settings: {e}")

def run_migrations_with_retry():
    """Run Django migrations with better error handling"""
    print("\n🗄️ Running Django migrations...")
    
    apps = ['users', 'loans', 'reports', 'utils']
    
    for app in apps:
        print(f"\n📱 Making migrations for {app}...")
        success = run_command(f"python manage.py makemigrations {app}")
        if not success:
            print(f"⚠️ Failed to make migrations for {app}, but continuing...")
    
    print("\n🔄 Running migrate command...")
    success = run_command("python manage.py migrate")
    if not success:
        print("⚠️ Migration failed, but continuing with setup...")
        # Try to migrate built-in apps first
        run_command("python manage.py migrate --run-syncdb")

def ensure_monthly_income_field():
    """Ensure monthly_income field exists in CustomUser model"""
    print("\n💰 Ensuring monthly_income field exists...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        from django.db import connection
        
        # Check if monthly_income column exists
        with connection.cursor() as cursor:
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'users' 
                AND COLUMN_NAME = 'monthly_income'
            """)
            result = cursor.fetchone()
            
            if not result:
                print("Adding monthly_income field to users table...")
                cursor.execute("""
                    ALTER TABLE users 
                    ADD COLUMN monthly_income DECIMAL(12,2) NULL,
                    ADD COLUMN employer VARCHAR(200) NULL
                """)
                print("✓ Added monthly_income and employer fields")
            else:
                print("✓ monthly_income field already exists")
                
    except Exception as e:
        print(f"❌ Error ensuring monthly_income field: {e}")

def ensure_document_fields():
    """Ensure all document fields exist in CustomUser model"""
    print("\n📄 Ensuring document fields exist...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        from django.db import connection
        
        # List of document fields to check/add
        document_fields = [
            ('logbook', 'VARCHAR(255) NULL'),
            ('title_deed', 'VARCHAR(255) NULL'),
            ('signature', 'VARCHAR(255) NULL'),
        ]
        
        with connection.cursor() as cursor:
            for field_name, field_type in document_fields:
                # Check if field exists
                cursor.execute(f"""
                    SELECT COLUMN_NAME 
                    FROM INFORMATION_SCHEMA.COLUMNS 
                    WHERE TABLE_SCHEMA = DATABASE() 
                    AND TABLE_NAME = 'users' 
                    AND COLUMN_NAME = '{field_name}'
                """)
                result = cursor.fetchone()
                
                if not result:
                    print(f"Adding {field_name} field to users table...")
                    cursor.execute(f"""
                        ALTER TABLE users 
                        ADD COLUMN {field_name} {field_type}
                    """)
                    print(f"✓ Added {field_name} field")
                else:
                    print(f"✓ {field_name} field already exists")
                    
    except Exception as e:
        print(f"❌ Error ensuring document fields: {e}")

def ensure_database_integrity():
    """Ensure all database tables and columns exist properly"""
    print("\n🗄️ Ensuring database integrity...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        from django.db import connection
        
        with connection.cursor() as cursor:
            # Check and fix loan_scoring table
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'loan_scoring' 
                AND COLUMN_NAME = 'user_id'
            """)
            if not cursor.fetchone():
                print("Adding user_id field to loan_scoring table...")
                cursor.execute("""
                    ALTER TABLE loan_scoring 
                    ADD COLUMN user_id CHAR(32) NULL,
                    ADD INDEX idx_loan_scoring_user_id (user_id)
                """)
                print("✓ Added user_id to loan_scoring table")
            
            # Check and fix utils_document table
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'utils_document' 
                AND COLUMN_NAME = 'uploaded_by_id'
            """)
            if not cursor.fetchone():
                print("Adding uploaded_by_id field to utils_document table...")
                cursor.execute("""
                    ALTER TABLE utils_document 
                    ADD COLUMN uploaded_by_id CHAR(32) NULL,
                    ADD INDEX idx_utils_document_uploaded_by (uploaded_by_id)
                """)
                print("✓ Added uploaded_by_id to utils_document table")
            
            # Check and fix utils_notification table
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'utils_notification' 
                AND COLUMN_NAME = 'user_id'
            """)
            if not cursor.fetchone():
                print("Adding user_id field to utils_notification table...")
                cursor.execute("""
                    ALTER TABLE utils_notification 
                    ADD COLUMN user_id CHAR(32) NULL,
                    ADD INDEX idx_utils_notification_user_id (user_id)
                """)
                print("✓ Added user_id to utils_notification table")
            
            # Check and fix utils_documentshare table
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'utils_documentshare' 
                AND COLUMN_NAME = 'shared_with_id'
            """)
            if not cursor.fetchone():
                print("Adding shared_with_id field to utils_documentshare table...")
                cursor.execute("""
                    ALTER TABLE utils_documentshare 
                    ADD COLUMN shared_with_id CHAR(32) NULL,
                    ADD INDEX idx_utils_documentshare_shared_with (shared_with_id)
                """)
                print("✓ Added shared_with_id to utils_documentshare table")
            
            # Check and fix utils_documentshare table for document_id
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'utils_documentshare' 
                AND COLUMN_NAME = 'document_id'
            """)
            if not cursor.fetchone():
                print("Adding document_id field to utils_documentshare table...")
                cursor.execute("""
                    ALTER TABLE utils_documentshare 
                    ADD COLUMN document_id INTEGER NULL,
                    ADD INDEX idx_utils_documentshare_document (document_id)
                """)
                print("✓ Added document_id to utils_documentshare table")
            
            # Check and fix users table for last_login_at field
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'users' 
                AND COLUMN_NAME = 'last_login_at'
            """)
            if not cursor.fetchone():
                print("Adding last_login_at field to users table...")
                cursor.execute("""
                    ALTER TABLE users 
                    ADD COLUMN last_login_at DATETIME(6) NULL
                """)
                print("✓ Added last_login_at to users table")
            
            # Check and fix users table for is_phone_verified and is_email_verified fields
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'users' 
                AND COLUMN_NAME = 'is_phone_verified'
            """)
            if not cursor.fetchone():
                print("Adding is_phone_verified field to users table...")
                cursor.execute("""
                    ALTER TABLE users 
                    ADD COLUMN is_phone_verified BOOLEAN DEFAULT FALSE,
                    ADD COLUMN is_email_verified BOOLEAN DEFAULT FALSE
                """)
                print("✓ Added verification fields to users table")
            
            # Check and fix users table for address field
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'users' 
                AND COLUMN_NAME = 'address'
            """)
            if not cursor.fetchone():
                print("Adding address field to users table...")
                cursor.execute("""
                    ALTER TABLE users 
                    ADD COLUMN address TEXT NULL
                """)
                print("✓ Added address field to users table")
            
            # Check and fix users table for county field
            cursor.execute("""
                SELECT COLUMN_NAME 
                FROM INFORMATION_SCHEMA.COLUMNS 
                WHERE TABLE_SCHEMA = DATABASE() 
                AND TABLE_NAME = 'users' 
                AND COLUMN_NAME = 'county'
            """)
            if not cursor.fetchone():
                print("Adding county field to users table...")
                cursor.execute("""
                    ALTER TABLE users 
                    ADD COLUMN county VARCHAR(100) NULL
                """)
                print("✓ Added county field to users table")
                
    except Exception as e:
        print(f"❌ Error ensuring database integrity: {e}")

def ensure_table_indexes():
    """Ensure proper indexes exist for performance"""
    print("\n📊 Ensuring database indexes...")
    
    try:
        # Setup Django
        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ruralpoint_system.settings_production')
        django.setup()
        
        from django.db import connection
        
        with connection.cursor() as cursor:
            # Add indexes for better performance
            indexes_to_add = [
                ("users", "idx_users_role", "role"),
                ("users", "idx_users_status", "status"),
                ("users", "idx_users_phone", "phone_number"),
                ("users", "idx_users_email", "email"),
                ("loan_scoring", "idx_loan_scoring_user", "user_id"),
                ("utils_document", "idx_document_uploaded_by", "uploaded_by_id"),
                ("utils_document", "idx_document_type", "document_type"),
                ("utils_notification", "idx_notification_user", "user_id"),
                ("utils_notification", "idx_notification_read", "read_at"),
                ("utils_documentshare", "idx_documentshare_shared_with", "shared_with_id"),
                ("utils_documentshare", "idx_documentshare_document", "document_id"),
            ]
            
            for table, index_name, column in indexes_to_add:
                try:
                    cursor.execute(f"""
                        CREATE INDEX {index_name} ON {table} ({column})
                    """)
                    print(f"✓ Added index {index_name} to {table}")
                except Exception as e:
                    if "Duplicate key name" in str(e):
                        print(f"✓ Index {index_name} already exists on {table}")
                    else:
                        print(f"⚠️ Could not add index {index_name}: {e}")
                        
    except Exception as e:
        print(f"❌ Error ensuring database indexes: {e}")

def create_document_upload_directories():
    """Create directories for document uploads"""
    print("\n📁 Creating document upload directories...")
    
    directories = [
        "media/kyc/logbooks",
        "media/kyc/title_deeds", 
        "media/kyc/signatures",
    ]
    
    for directory in directories:
        os.makedirs(directory, exist_ok=True)
        print(f"✓ Created {directory}")
    
    print("✓ Document upload directories created")


def setup_reports_environment():
    """Setup reports environment and ensure chart generation works properly"""
    print("\n📊 Setting up reports environment...")
    
    try:
        # Create reports cache directory
        reports_cache_dir = os.path.join(os.getcwd(), 'reports_cache')
        os.makedirs(reports_cache_dir, exist_ok=True)
        print(f"✓ Created reports cache directory: {reports_cache_dir}")
        
        # Set matplotlib backend to Agg for server environments
        import matplotlib
        matplotlib.use('Agg')
        print("✓ Configured matplotlib backend for server environment")
        
        # Test chart generation
        try:
            import matplotlib.pyplot as plt
            import numpy as np
            
            # Create a simple test chart
            fig, ax = plt.subplots(figsize=(6, 4))
            x = np.linspace(0, 10, 100)
            y = np.sin(x)
            ax.plot(x, y)
            ax.set_title('Test Chart')
            ax.set_xlabel('X')
            ax.set_ylabel('Y')
            
            # Save to buffer to test
            buffer = io.BytesIO()
            plt.savefig(buffer, format='png', dpi=100, bbox_inches='tight')
            buffer.seek(0)
            plt.close()
            
            print("✓ Chart generation test successful")
            
        except Exception as e:
            print(f"⚠️ Chart generation test failed: {e}")
            print("⚠️ Reports with charts may not work properly")
        
        # Ensure seaborn is available
        try:
            import seaborn as sns
            print("✓ Seaborn is available for enhanced chart styling")
        except ImportError:
            print("⚠️ Seaborn not available - charts will use basic styling")
        
    except Exception as e:
        print(f"❌ Error setting up reports environment: {e}")

def main():
    print("🚀 Starting RuralPoint setup...")
    
    # Step 1: Create necessary directories
    print("\n📁 Creating directories...")
    directories = [
        "logs",
        "media/kyc/bank_statements",
        "media/kyc/id_documents", 
        "media/kyc/selfies",
        "media/kyc/utility_bills",
        "media/kyc/business_licenses",
        "media/kyc/tax_certificates",
        "media/kyc/logbooks",
        "media/kyc/title_deeds",
        "media/kyc/signatures",
        "static",
        "staticfiles"
    ]
    
    for directory in directories:
        os.makedirs(directory, exist_ok=True)
        print(f"✓ Created {directory}")
    
    # Step 2: Install requirements
    install_requirements()
    
    # Step 3: Create production settings
    create_production_settings()
    
    # Step 4: Update passenger_wsgi.py
    update_passenger_wsgi()
    
    # Step 5: Run Django commands with better error handling
    run_migrations_with_retry()
    
    print("\n📂 Collecting static files...")
    run_command("python manage.py collectstatic --noinput")
    
    print("\n💾 Creating cache table...")
    run_command("python manage.py createcachetable")
    
    # Step 6: Ensure monthly_income field exists
    ensure_monthly_income_field()
    
    # Step 7: Ensure document fields exist
    ensure_document_fields()
    
    # Step 8: Ensure database integrity (fix missing columns)
    ensure_database_integrity()
    
    # Step 9: Ensure database indexes for performance
    ensure_table_indexes()
    
    # Step 10: Create document upload directories
    create_document_upload_directories()
    
    # Step 11: Setup role-based permissions system
    setup_role_permissions()
    
    # Step 12: Create default system settings
    create_default_settings()
    
    # Step 13: Verify admin user
    verify_admin_user()
    

    
    # Step 16: Create sample credit scores
    
    # Step 17: Setup reports environment
    setup_reports_environment()
    
    print("\n✅ Setup completed!")
    print("\n🔐 Role-Based Access Control (RBAC) System:")
    print("   • 6 User Roles: Admin, Team Leader, Loan Officer, Secretary, Auditor, Borrower")
    print("   • 160 Granular Permissions across 10 modules")
    print("   • Complete audit trail and access logging")
    print("   • Admin Management interface for user creation and permission management")
    
    print("\n📊 Enhanced Reporting & Analytics:")
    print("   • Advanced chart generation with matplotlib and seaborn")
    print("   • Interactive client popup charts with Chart.js CDN")
    print("   • Comprehensive loan analytics and risk assessment")
    print("   • PDF and Excel export capabilities")
    print("   • Real-time performance metrics and KPIs")
    
    print("\n👤 Enhanced Client Management:")
    print("   • Comprehensive client popup with selfie photo display")
    print("   • Interactive loan summary charts (Chart.js CDN)")
    print("   • Ordinal loan naming (First Loan, Second Loan, etc.)")
    print("   • Complete document management (10 document types)")
    print("   • Real-time document status tracking")
    print("   • Enhanced KYC document handling")
    print("   • Improved client profile visualization")
    
    print("\n🗄️ Database Integrity & Performance:")
    print("   • Fixed missing foreign key columns")
    print("   • Added proper database indexes")
    print("   • Ensured all required fields exist")
    print("   • Optimized query performance")
    print("   • Resolved production deployment issues")
    
    print("\nImportant Information:")
    print("🌐 Site URL: https://ruralpoint.co.ke/")
    print("👤 Admin Login:")
    print("   Email: admin@ruralpoint.co.ke")
    print("   Password: admin123")
    print("   Role: Admin (Full system access)")
    
    print("\n🔐 Available Staff Roles:")
    print("   • Admin: Full system access (58 permissions)")
    print("   • Team Leader: Broad access, no system settings management (34 permissions)")
    print("   • Loan Officer: Loan processing & client management (26 permissions)")
    print("   • Secretary: Administrative tasks & document management (21 permissions)")
    print("   • Auditor: Read-only access for compliance (12 permissions)")
    print("   • Borrower: Limited self-service access (9 permissions)")
    
    print("\n⚠️ Security Reminders:")
    print("1. Change the admin password after first login")
    print("2. Update SECRET_KEY in production settings")
    print("3. Store database credentials securely")
    print("4. Configure email settings in production")
    print("5. Review and customize role permissions as needed")
    
    print("\n📝 Logs Location: ./logs/django.log")
    print("🔍 Access Logs: Available in Admin Management → Access Logs")
    
    print("\n🔧 Next Steps:")
    print("1. Configure your domain in cPanel")
    print("2. Set up SSL certificate")
    print("3. Configure email settings")
    print("4. Set up database backups")
    print("5. Test all functionality in production")
    print("6. Create additional staff users via Admin Management")
    print("7. Customize role permissions for your organization")
    
    print("\n🔍 Troubleshooting:")
    if not os.path.exists('users/migrations'):
        print("⚠️ If migrations still fail, manually create migrations directories:")
        print("   mkdir -p users/migrations loans/migrations reports/migrations utils/migrations")
        print("   touch users/migrations/__init__.py loans/migrations/__init__.py")
        print("   touch reports/migrations/__init__.py utils/migrations/__init__.py")

if __name__ == "__main__":
    main()