/**
 * KPI Collector Service
 * 
 * Mục đích: Thu thập dữ liệu KPI từ Meteocontrol V2 API và lưu vào MongoDB
 * 
 * Chức năng:
 * - Kết nối với Meteocontrol V2 API (OAuth hoặc Basic Auth)
 * - Thu thập Production (E_Z_EVU), PR, Availability (VFG), Irradiation (G_M)
 * - Tính toán Specific Yield
 * - Lưu vào MongoDB KPI collection
 */

import { Project } from '../models/Project';
import { SystemSettings } from '../models/SystemSettings';
import { KPI } from '../models/KPI';
import mongoose from 'mongoose';

const V2_API_BASE_URL = 'https://api.meteocontrol.de/v2';
const WIDGET_API_BASE_URL = 'http://ws.meteocontrol.de/api';

interface MeteocontrolV2Credentials {
  username: string;
  password: string;
  apiKey: string;
}

// OAuth Token Cache
interface TokenCache {
  token: string;
  expiresAt: number; // Timestamp when token expires
}

let oauthTokenCache: TokenCache | null = null;
let tokenRequestPromise: Promise<string | null> | null = null; // Mutex to prevent concurrent token requests

interface MeteocontrolProjectConfig {
  siteKey: string;
  apiKey: string;
  systemKey: string;
  useV2API?: boolean;
}

interface KPIData {
  production: number; // kWh
  pr: number; // Performance Ratio (%)
  availability: number; // Availability (%)
  specificYield: number; // kWh/kWp
  irradiation?: number; // kWh/m² (optional)
}

export interface IKPIData {
  date: Date;
  production: number;
  pr: number;
  availability: number;
  specificYield: number;
  irradiation: number;
}

/**
 * Get OAuth access token from Meteocontrol V2 API
 * Uses caching to avoid excessive login requests (rate limit: 90 calls/minute)
 */
async function getOAuthToken(credentials: MeteocontrolV2Credentials): Promise<string | null> {
  try {
    // Check if we have a valid cached token
    const now = Date.now();
    if (oauthTokenCache && oauthTokenCache.expiresAt > now) {
      // Token is still valid, reuse it
      return oauthTokenCache.token;
    }

    // If there's already a token request in progress, wait for it instead of creating a new one
    if (tokenRequestPromise) {
      // console.log('[KPI Collector] Token request already in progress, waiting...');
      return await tokenRequestPromise;
    }

    // Token expired or doesn't exist, get a new one
    // Use a promise to prevent concurrent requests
    tokenRequestPromise = (async () => {
      try {
        // console.log('[KPI Collector] Requesting new OAuth token...');
        
        const response = await fetch(`${V2_API_BASE_URL}/login`, {
          method: 'POST',
          headers: {
            'Content-Type': 'application/x-www-form-urlencoded',
            'X-API-KEY': credentials.apiKey,
          },
          body: new URLSearchParams({
            grant_type: 'password',
            username: credentials.username,
            password: credentials.password,
            client_id: 'vcom-api',
            client_secret: 'AYB=~9_f-BvNoLt8+x=3maCq)>/?@Nom',
          }),
        });

        if (!response.ok) {
          const errorText = await response.text();
          // console.error(`[KPI Collector] OAuth failed: ${response.status} ${errorText}`);
          
          // If rate limited, wait a bit before retrying
          if (response.status === 429) {
            // console.warn('[KPI Collector] Rate limited, waiting 60 seconds before retry...');
            await new Promise(resolve => setTimeout(resolve, 60000));
            // Clear the promise so we can retry
            tokenRequestPromise = null;
            // Retry once
            return getOAuthToken(credentials);
          }
          
          tokenRequestPromise = null;
          return null;
        }

        const data = (await response.json()) as { access_token?: string; expires_in?: number };
        const accessToken = data.access_token;

        if (accessToken) {
          // Cache the token with expiration time (default: 6 days, but use 5 days to be safe)
          const expiresIn = data.expires_in ?? 518400; // Default from API
          const expiresAt = Date.now() + (expiresIn * 1000) - (60 * 60 * 1000); // Subtract 1 hour for safety
          
          oauthTokenCache = {
            token: accessToken,
            expiresAt,
          };
          
          // console.log(`[KPI Collector] OAuth token cached, expires in ${Math.floor((expiresAt - Date.now()) / 1000 / 60)} minutes`);
          tokenRequestPromise = null;
          return accessToken;
        }
        
        tokenRequestPromise = null;
        return null;
      } catch (error: any) {
        // console.error('[KPI Collector] Error getting OAuth token:', error.message);
        tokenRequestPromise = null;
        return null;
      }
    })();

    return await tokenRequestPromise;
  } catch (error: any) {
    // console.error('[KPI Collector] Error in getOAuthToken wrapper:', error.message);
    tokenRequestPromise = null;
    return null;
  }
}

/**
 * Call Meteocontrol V2 API with authentication
 */
async function callV2API(
  endpoint: string,
  credentials: MeteocontrolV2Credentials,
  useOAuth: boolean = true,
  providedToken?: string // Optional: use provided token instead of getting new one
): Promise<any> {
  let headers: Record<string, string> = {
    'X-API-KEY': credentials.apiKey,
  };

  if (useOAuth) {
    const token = providedToken || await getOAuthToken(credentials);
    if (!token) {
      throw new Error('Failed to get OAuth token');
    }
    headers['Authorization'] = `Bearer ${token}`;
  } else {
    // Basic Auth
    const basicAuth = Buffer.from(`${credentials.username}:${credentials.password}`).toString('base64');
    headers['Authorization'] = `Basic ${basicAuth}`;
  }

  const response = await fetch(`${V2_API_BASE_URL}${endpoint}`, {
    method: 'GET',
    headers,
  });

  if (!response.ok) {
    const errorText = await response.text();
    // Log detailed error for debugging
    if (response.status !== 404) { // 404 is expected for some dates
      // console.warn(`[KPI Collector] V2 API error ${response.status} for ${endpoint}: ${errorText.substring(0, 200)}`);
    }
    throw new Error(`V2 API error: ${response.status} ${errorText.substring(0, 100)}`);
  }

  const data = await response.json();
  
  // Debug: Log response structure for API calls (only for first batch to avoid spam)
  if (endpoint.includes('/basics/') || endpoint.includes('/calculations/')) {
    // Log full response structure for debugging (only first call)
    const isFirstCall = !endpoint.includes('?from=') || endpoint.includes('?from=') && Math.random() < 0.05;
    if (isFirstCall) {
      // console.log(`[KPI Collector] API Response for ${endpoint.substring(0, 100)}:`, {
      //   hasData: !!data?.data,
      //   dataType: typeof data?.data,
      //   isArray: Array.isArray(data?.data),
      //   isObject: typeof data?.data === 'object' && data?.data !== null,
      //   keys: data?.data && typeof data?.data === 'object' ? Object.keys(data?.data) : [],
      //   sample: JSON.stringify(data).substring(0, 1000),
      // });
    }
  }
  
  return data;
}

/**
 * Get KPI data from V2 API using provided token (to avoid multiple logins)
 */
async function getKPIDataFromV2APIWithToken(
  systemKey: string,
  date: Date,
  credentials: MeteocontrolV2Credentials,
  token: string
): Promise<KPIData | null> {
  try {
    // Ensure date is not in the future
    const today = new Date();
    today.setHours(23, 59, 59, 999);
    if (date > today) {
      // console.warn(`[KPI Collector] Date ${date.toISOString().split('T')[0]} is in the future, skipping`);
      return null;
    }
    
    const timezone = '+07:00'; // Vietnam timezone
    const dateStr = date.toISOString().split('T')[0];
    const from = `${dateStr}T00:00:00${timezone}`;
    const to = `${dateStr}T23:59:59${timezone}`;

    // Use Bulk API to get multiple measurements at once
    const basicsEndpoint = `/systems/${systemKey}/basics/measurements?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`;
    const calculationsEndpoint = `/systems/${systemKey}/calculations/measurements?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`;
    
    // Debug: Log endpoint for first few dates
    if (Math.random() < 0.05) { // Log 5% of requests
      // console.log(`[KPI Collector] Fetching data for ${dateStr} using systemKey: ${systemKey}`);
    }

    // Use provided token to avoid multiple logins
    const [basicsData, calculationsData] = await Promise.all([
      callV2API(basicsEndpoint, credentials, true, token).catch((err) => {
        // console.warn(`[KPI Collector] Basics API error for ${dateStr}:`, err.message);
        return null;
      }),
      callV2API(calculationsEndpoint, credentials, true, token).catch((err) => {
        // console.warn(`[KPI Collector] Calculations API error for ${dateStr}:`, err.message);
        return null;
      }),
    ]);

    // Debug: Log response structure for first date
    if (dateStr === date.toISOString().split('T')[0] && !basicsData && !calculationsData) {
      // console.log(`[KPI Collector] No data returned for ${dateStr} - basics: ${!!basicsData}, calculations: ${!!calculationsData}`);
    }

    // Return data even if only one source has data
    if (!basicsData && !calculationsData) {
      return null;
    }

    // Extract Production (E_Z_EVU) from basics
    let production = 0;
    if (basicsData?.data && Array.isArray(basicsData.data)) {
      const eZEVU = basicsData.data.find((item: any) => item.abbreviation === 'E_Z_EVU');
      if (eZEVU?.value !== null && eZEVU?.value !== undefined) {
        production = eZEVU.value;
      }
    } else if (basicsData?.data) {
      // Handle case where data might be in different format
      // console.warn(`[KPI Collector] Unexpected basics data format for ${dateStr}:`, typeof basicsData.data);
    }

    // Extract PR and Availability from calculations
    let pr = 0;
    let availability = 0;
    let irradiation = 0;
    
    if (calculationsData?.data && Array.isArray(calculationsData.data)) {
      const prData = calculationsData.data.find((item: any) => item.abbreviation === 'PR');
      const vfgData = calculationsData.data.find((item: any) => item.abbreviation === 'VFG');
      
      if (prData?.value !== null && prData?.value !== undefined) {
        pr = prData.value;
      }
      if (vfgData?.value !== null && vfgData?.value !== undefined) {
        availability = vfgData.value;
      }
    }

    // Extract Irradiation (G_M) from basics
    if (basicsData?.data && Array.isArray(basicsData.data)) {
      const gMData = basicsData.data.find((item: any) => item.abbreviation === 'G_M');
      if (gMData?.value !== null && gMData?.value !== undefined) {
        irradiation = gMData.value;
      }
    }

    // Return data even if all values are 0 (might be valid for some days)
    return {
      production,
      pr,
      availability,
      specificYield: 0, // Will be calculated later
      irradiation,
    };
  } catch (error: any) {
    // console.error(`[KPI Collector] Error getting KPI data from V2 API for ${date.toISOString().split('T')[0]}:`, error.message);
    return null;
  }
}

/**
 * Get KPI data from Meteocontrol V2 API using Bulk API
 */
async function getKPIDataFromV2API(
  systemKey: string,
  date: Date,
  credentials: MeteocontrolV2Credentials
): Promise<KPIData | null> {
  try {
    const timezone = '+07:00'; // Vietnam timezone
    const dateStr = date.toISOString().split('T')[0];
    const from = `${dateStr}T00:00:00${timezone}`;
    const to = `${dateStr}T23:59:59${timezone}`;

    // Use Bulk API to get multiple measurements at once
    const basicsEndpoint = `/systems/${systemKey}/basics/measurements?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`;
    const calculationsEndpoint = `/systems/${systemKey}/calculations/measurements?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`;

    const [basicsData, calculationsData] = await Promise.all([
      callV2API(basicsEndpoint, credentials).catch(() => null),
      callV2API(calculationsEndpoint, credentials).catch(() => null),
    ]);

    if (!basicsData && !calculationsData) {
      return null;
    }

    // Extract Production (E_Z_EVU) from basics
    let production = 0;
    if (basicsData?.data) {
      const eZEVU = basicsData.data.find((item: any) => item.abbreviation === 'E_Z_EVU');
      if (eZEVU?.value !== null && eZEVU?.value !== undefined) {
        production = eZEVU.value;
      }
    }

    // Extract PR and Availability from calculations
    let pr = 0;
    let availability = 0;
    let irradiation = 0;
    
    if (calculationsData?.data) {
      const prData = calculationsData.data.find((item: any) => item.abbreviation === 'PR');
      const vfgData = calculationsData.data.find((item: any) => item.abbreviation === 'VFG');
      
      if (prData?.value !== null && prData?.value !== undefined) {
        pr = prData.value;
      }
      if (vfgData?.value !== null && vfgData?.value !== undefined) {
        availability = vfgData.value;
      }
    }

    // Extract Irradiation (G_M) from basics
    if (basicsData?.data) {
      const gMData = basicsData.data.find((item: any) => item.abbreviation === 'G_M');
      if (gMData?.value !== null && gMData?.value !== undefined) {
        irradiation = gMData.value;
      }
    }

    // Calculate Specific Yield (kWh/kWp) - need capacity from project
    // This will be calculated in the main function

    return {
      production,
      pr,
      availability,
      specificYield: 0, // Will be calculated later
      irradiation,
    };
  } catch (error: any) {
    // console.error(`[KPI Collector] Error getting KPI data from V2 API:`, error.message);
    return null;
  }
}

/**
 * Get Production data from Widget API (fallback)
 */
async function getProductionFromWidgetAPI(
  siteKey: string,
  apiKey: string,
  date: Date
): Promise<number> {
  try {
    const dateStr = date.toISOString().split('T')[0];
    const response = await fetch(
      `${WIDGET_API_BASE_URL}/sites/${siteKey}/data/energygeneration?apiKey=${apiKey}&type=day&date=${dateStr}`
    );

    if (!response.ok) {
      return 0;
    }

    const data = (await response.json()) as { chartData?: { data?: unknown[] } };
    if (data.chartData?.data) {
      // Sum all non-null values
      const values = data.chartData.data
        .map((item: unknown) => (item as any[])[1])
        .filter((val: any) => val !== null && val !== undefined);
      
      return values.reduce((sum: number, val: number) => sum + val, 0);
    }

    return 0;
  } catch (error: any) {
    // console.error(`[KPI Collector] Error getting production from Widget API:`, error.message);
    return 0;
  }
}

/**
 * Get KPI data directly from V2 API using day resolution (efficient batch fetching)
 * Returns KPI data for a date range
 * Used by Dashboard to display real-time KPI data
 * 
 * Uses API endpoints with resolution=day to fetch multiple days at once (max 60 days per request)
 */
export async function getKPIsFromV2API(
  projectId: string,
  startDate: Date,
  endDate: Date
): Promise<IKPIData[]> {
  try {
    // Get project
    const project = await Project.findById(projectId);
    if (!project) {
      throw new Error(`Project not found: ${projectId}`);
    }

    // Check if project has Meteocontrol config
    if (!project.meteocontrol?.siteKey || !project.meteocontrol?.apiKey) {
      return [];
    }

    const config = project.meteocontrol;
    const systemKey = config.systemKey || config.siteKey;

    // Get V2 API credentials from SystemSettings
    const settings = await SystemSettings.findOne();
    if (!settings?.vcomApi?.username || !settings?.vcomApi?.password || !settings?.vcomApi?.apiKey) {
      throw new Error('V2 API credentials not configured in SystemSettings');
    }

    const v2Credentials: MeteocontrolV2Credentials = {
      username: settings.vcomApi.username,
      password: settings.vcomApi.password,
      apiKey: settings.vcomApi.apiKey,
    };

    const capacityMWp = project.capacityMWp || 0;
    const capacityKWp = capacityMWp * 1000;

    // Validate capacity
    if (capacityMWp <= 0) {
      // console.warn(`[KPI Live] Project ${projectId} has invalid capacity: ${capacityMWp} MWp. Specific Yield calculation may be incorrect.`);
    } else {
    }

    // Generate date range - ensure no future dates
    const today = new Date();
    today.setHours(23, 59, 59, 999);
    
    // Clamp dates to today if they're in the future
    let actualEndDate = endDate > today ? new Date(today) : new Date(endDate);
    let actualStartDate = startDate > today ? new Date(today) : new Date(startDate);
    
    // Ensure start is not after end
    if (actualStartDate > actualEndDate) {
      actualEndDate = new Date(today);
      actualStartDate = new Date(today);
      actualStartDate.setDate(actualStartDate.getDate() - 30);
      actualStartDate.setHours(0, 0, 0, 0);
    }
    
    // Final validation: ensure end date is exactly today or before
    if (actualEndDate > today) {
      actualEndDate = new Date(today);
    }
    if (actualStartDate > today) {
      actualStartDate = new Date(today);
      actualStartDate.setDate(actualStartDate.getDate() - 30);
      actualStartDate.setHours(0, 0, 0, 0);
    }
    

    // Get OAuth token once and reuse for all requests
    const token = await getOAuthToken(v2Credentials);
    if (!token) {
      throw new Error('Failed to get OAuth token');
    }

    // Calculate days difference
    const daysDiff = Math.ceil((actualEndDate.getTime() - actualStartDate.getTime()) / (1000 * 60 * 60 * 24)) + 1;
    
    // API limit: max 60 days per request for day resolution
    const MAX_DAYS_PER_REQUEST = 60;
    
    // Split into batches of max 60 days
    const batches: { start: Date; end: Date }[] = [];
    let currentStart = new Date(actualStartDate);
    currentStart.setHours(0, 0, 0, 0);
    
    while (currentStart <= actualEndDate) {
      const batchEnd = new Date(currentStart);
      batchEnd.setDate(batchEnd.getDate() + MAX_DAYS_PER_REQUEST - 1);
      if (batchEnd > actualEndDate) {
        batchEnd.setTime(actualEndDate.getTime());
      }
      batchEnd.setHours(23, 59, 59, 999);
      
      batches.push({
        start: new Date(currentStart),
        end: new Date(batchEnd),
      });
      
      currentStart.setDate(currentStart.getDate() + MAX_DAYS_PER_REQUEST);
    }


    const kpiDataArray: IKPIData[] = [];
    const timezone = '+07:00'; // Vietnam timezone

    // Process each batch
    for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
      const batch = batches[batchIdx];
      const batchStartStr = batch.start.toISOString().split('T')[0];
      const batchEndStr = batch.end.toISOString().split('T')[0];
      

      try {
        // Format dates for API (YYYY-MM-DD format, API will use system timezone)
        const from = batchStartStr;
        const to = batchEndStr;

        // Use abbreviations endpoint with resolution=day to get multiple days at once
        // Note: When using multiple abbreviations, we need to call each separately or use bulk endpoint
        // For now, we'll call each abbreviation separately to ensure correct parsing
        const basicsEZEVEndpoint = `/systems/${systemKey}/basics/abbreviations/E_Z_EVU/measurements?from=${from}&to=${to}&resolution=day`;
        const basicsGMEndpoint = `/systems/${systemKey}/basics/abbreviations/G_M/measurements?from=${from}&to=${to}&resolution=day`;
        const basicsGM0Endpoint = `/systems/${systemKey}/basics/abbreviations/G_M0/measurements?from=${from}&to=${to}&resolution=day`; // Fallback for G_M
        const calculationsPREndpoint = `/systems/${systemKey}/calculations/abbreviations/PR/measurements?from=${from}&to=${to}&resolution=day`;
        const calculationsVFGEndpoint = `/systems/${systemKey}/calculations/abbreviations/VFG/measurements?from=${from}&to=${to}&resolution=day`;

        // Fetch all endpoints in parallel
        const [basicsEZEVRResponse, basicsGMResponse, basicsGM0Response, calculationsPRResponse, calculationsVFGResponse] = await Promise.all([
          callV2API(basicsEZEVEndpoint, v2Credentials, true, token).catch((err) => {
            // console.warn(`[KPI Live] Basics E_Z_EVU API error for batch ${batchStartStr} to ${batchEndStr}:`, err.message);
            return null;
          }),
          callV2API(basicsGMEndpoint, v2Credentials, true, token).catch((err) => {
            // G_M might not be available, try G_M0 as fallback
            return null;
          }),
          callV2API(basicsGM0Endpoint, v2Credentials, true, token).catch((err) => {
            // console.warn(`[KPI Live] Basics G_M0 API error for batch ${batchStartStr} to ${batchEndStr}:`, err.message);
            return null;
          }),
          callV2API(calculationsPREndpoint, v2Credentials, true, token).catch((err) => {
            // console.warn(`[KPI Live] Calculations PR API error for batch ${batchStartStr} to ${batchEndStr}:`, err.message);
            return null;
          }),
          callV2API(calculationsVFGEndpoint, v2Credentials, true, token).catch((err) => {
            // console.warn(`[KPI Live] Calculations VFG API error for batch ${batchStartStr} to ${batchEndStr}:`, err.message);
            return null;
          }),
        ]);

        // Parse basics data (E_Z_EVU and G_M)
        const basicsDataMap: Map<string, { production: number; irradiation: number }> = new Map();
        
        // Parse E_Z_EVU (production) response
        // Response format: { data: { E_Z_EVU: [{ timestamp, value }] } }
        // Or: { data: [{ timestamp, value }] } when using single abbreviation
        if (basicsEZEVRResponse?.data) {
          let eZEVUData: any[] = [];
          
          // Handle different response formats
          // Format 1: { data: { E_Z_EVU: [...] } }
          if (basicsEZEVRResponse.data.E_Z_EVU) {
            eZEVUData = Array.isArray(basicsEZEVRResponse.data.E_Z_EVU) 
              ? basicsEZEVRResponse.data.E_Z_EVU 
              : [];
          } else if (basicsEZEVRResponse.data['E_Z_EVU']) {
            eZEVUData = Array.isArray(basicsEZEVRResponse.data['E_Z_EVU']) 
              ? basicsEZEVRResponse.data['E_Z_EVU'] 
              : [];
          } 
          // Format 2: { data: [...] } - direct array (single abbreviation endpoint)
          else if (Array.isArray(basicsEZEVRResponse.data)) {
            eZEVUData = basicsEZEVRResponse.data;
          }
          // Format 3: Check if data is an object with nested structure
          else if (typeof basicsEZEVRResponse.data === 'object') {
            // Try to find any array property
            const keys = Object.keys(basicsEZEVRResponse.data);
            for (const key of keys) {
              if (Array.isArray(basicsEZEVRResponse.data[key])) {
                eZEVUData = basicsEZEVRResponse.data[key];
                break;
              }
            }
          }
          
          // Parse E_Z_EVU (production)
          let parsedCount = 0;
          eZEVUData.forEach((item: any) => {
            if (item && item.timestamp) {
              try {
                const dateStr = new Date(item.timestamp).toISOString().split('T')[0];
                const value = typeof item.value === 'number' ? item.value : parseFloat(item.value) || 0;
                const existing = basicsDataMap.get(dateStr) || { production: 0, irradiation: 0 };
                existing.production = value;
                basicsDataMap.set(dateStr, existing);
                parsedCount++;
              } catch (e) {
                // console.warn(`[KPI Live] Error parsing E_Z_EVU timestamp:`, item.timestamp, e);
              }
            }
          });
          
          if (batchIdx === 0) {
          }
        } else if (batchIdx === 0) {
          // console.warn(`[KPI Live] Basics E_Z_EVU response has no data field`);
        }
        
        // Parse G_M or G_M0 (irradiation) response - try G_M first, then G_M0 as fallback
        let irradiationParsed = false;
        
        // Try G_M first
        if (basicsGMResponse?.data) {
          let gMData: any[] = [];
          
          // Handle different response formats
          // Format 1: { data: { G_M: [...] } }
          if (basicsGMResponse.data.G_M) {
            gMData = Array.isArray(basicsGMResponse.data.G_M) 
              ? basicsGMResponse.data.G_M 
              : [];
          } else if (basicsGMResponse.data['G_M']) {
            gMData = Array.isArray(basicsGMResponse.data['G_M']) 
              ? basicsGMResponse.data['G_M'] 
              : [];
          } 
          // Format 2: { data: [...] } - direct array (single abbreviation endpoint)
          else if (Array.isArray(basicsGMResponse.data)) {
            gMData = basicsGMResponse.data;
          }
          // Format 3: Check if data is an object with nested structure
          else if (typeof basicsGMResponse.data === 'object') {
            // Try to find any array property
            const keys = Object.keys(basicsGMResponse.data);
            for (const key of keys) {
              if (Array.isArray(basicsGMResponse.data[key])) {
                gMData = basicsGMResponse.data[key];
                break;
              }
            }
          }
          
          // Parse G_M (irradiation)
          let parsedCount = 0;
          gMData.forEach((item: any) => {
            if (item && item.timestamp) {
              try {
                const dateStr = new Date(item.timestamp).toISOString().split('T')[0];
                const value = typeof item.value === 'number' ? item.value : parseFloat(item.value) || 0;
                const existing = basicsDataMap.get(dateStr) || { production: 0, irradiation: 0 };
                existing.irradiation = value;
                basicsDataMap.set(dateStr, existing);
                parsedCount++;
                irradiationParsed = true;
              } catch (e) {
                // console.warn(`[KPI Live] Error parsing G_M timestamp:`, item.timestamp, e);
              }
            }
          });
          
          if (batchIdx === 0) {
          }
        }
        
        // Fallback to G_M0 if G_M not available
        if (!irradiationParsed && basicsGM0Response?.data) {
          let gM0Data: any[] = [];
          
          // Handle different response formats
          if (basicsGM0Response.data.G_M0) {
            gM0Data = Array.isArray(basicsGM0Response.data.G_M0) 
              ? basicsGM0Response.data.G_M0 
              : [];
          } else if (basicsGM0Response.data['G_M0']) {
            gM0Data = Array.isArray(basicsGM0Response.data['G_M0']) 
              ? basicsGM0Response.data['G_M0'] 
              : [];
          } else if (Array.isArray(basicsGM0Response.data)) {
            gM0Data = basicsGM0Response.data;
          } else if (typeof basicsGM0Response.data === 'object') {
            const keys = Object.keys(basicsGM0Response.data);
            for (const key of keys) {
              if (Array.isArray(basicsGM0Response.data[key])) {
                gM0Data = basicsGM0Response.data[key];
                break;
              }
            }
          }
          
          // Parse G_M0 (irradiation fallback)
          let parsedCount = 0;
          gM0Data.forEach((item: any) => {
            if (item && item.timestamp) {
              try {
                const dateStr = new Date(item.timestamp).toISOString().split('T')[0];
                const value = typeof item.value === 'number' ? item.value : parseFloat(item.value) || 0;
                const existing = basicsDataMap.get(dateStr) || { production: 0, irradiation: 0 };
                // Only set if not already set from G_M
                if (existing.irradiation === 0) {
                  existing.irradiation = value;
                  basicsDataMap.set(dateStr, existing);
                  parsedCount++;
                  irradiationParsed = true;
                }
              } catch (e) {
                // console.warn(`[KPI Live] Error parsing G_M0 timestamp:`, item.timestamp, e);
              }
            }
          });
          
          if (batchIdx === 0 && parsedCount > 0) {
          }
        }
        
        if (!irradiationParsed && batchIdx === 0) {
          // console.warn(`[KPI Live] No irradiation data found (tried G_M and G_M0). System may not have irradiation sensor configured.`);
        }

        // Parse calculations data (PR and VFG)
        const calculationsDataMap: Map<string, { pr: number; availability: number }> = new Map();
        
        // Parse PR (performance ratio) response
        // Response format: { data: { PR: [{ timestamp, value }] } }
        // Or: { data: [{ timestamp, value }] } when using single abbreviation
        if (calculationsPRResponse?.data) {
          let prData: any[] = [];
          
          // Handle different response formats
          // Format 1: { data: { PR: [...] } }
          if (calculationsPRResponse.data.PR) {
            prData = Array.isArray(calculationsPRResponse.data.PR) 
              ? calculationsPRResponse.data.PR 
              : [];
          } else if (calculationsPRResponse.data['PR']) {
            prData = Array.isArray(calculationsPRResponse.data['PR']) 
              ? calculationsPRResponse.data['PR'] 
              : [];
          } 
          // Format 2: { data: [...] } - direct array (single abbreviation endpoint)
          else if (Array.isArray(calculationsPRResponse.data)) {
            prData = calculationsPRResponse.data;
          }
          // Format 3: Check if data is an object with nested structure
          else if (typeof calculationsPRResponse.data === 'object') {
            // Try to find any array property
            const keys = Object.keys(calculationsPRResponse.data);
            for (const key of keys) {
              if (Array.isArray(calculationsPRResponse.data[key])) {
                prData = calculationsPRResponse.data[key];
                break;
              }
            }
          }
          
          // Parse PR (performance ratio)
          let parsedCount = 0;
          prData.forEach((item: any) => {
            if (item && item.timestamp) {
              try {
                const dateStr = new Date(item.timestamp).toISOString().split('T')[0];
                const value = typeof item.value === 'number' ? item.value : parseFloat(item.value) || 0;
                const existing = calculationsDataMap.get(dateStr) || { pr: 0, availability: 0 };
                existing.pr = value;
                calculationsDataMap.set(dateStr, existing);
                parsedCount++;
              } catch (e) {
                // console.warn(`[KPI Live] Error parsing PR timestamp:`, item.timestamp, e);
              }
            }
          });
          
          if (batchIdx === 0) {
          }
        } else if (batchIdx === 0) {
          // console.warn(`[KPI Live] Calculations PR response has no data field`);
        }
        
        // Parse VFG (availability) response
        // Response format: { data: { VFG: [{ timestamp, value }] } }
        // Or: { data: [{ timestamp, value }] } when using single abbreviation
        if (calculationsVFGResponse?.data) {
          let vfgData: any[] = [];
          
          // Handle different response formats
          // Format 1: { data: { VFG: [...] } }
          if (calculationsVFGResponse.data.VFG) {
            vfgData = Array.isArray(calculationsVFGResponse.data.VFG) 
              ? calculationsVFGResponse.data.VFG 
              : [];
          } else if (calculationsVFGResponse.data['VFG']) {
            vfgData = Array.isArray(calculationsVFGResponse.data['VFG']) 
              ? calculationsVFGResponse.data['VFG'] 
              : [];
          } 
          // Format 2: { data: [...] } - direct array (single abbreviation endpoint)
          else if (Array.isArray(calculationsVFGResponse.data)) {
            vfgData = calculationsVFGResponse.data;
          }
          // Format 3: Check if data is an object with nested structure
          else if (typeof calculationsVFGResponse.data === 'object') {
            // Try to find any array property
            const keys = Object.keys(calculationsVFGResponse.data);
            for (const key of keys) {
              if (Array.isArray(calculationsVFGResponse.data[key])) {
                vfgData = calculationsVFGResponse.data[key];
                break;
              }
            }
          }
          
          // Parse VFG (availability)
          let parsedCount = 0;
          vfgData.forEach((item: any) => {
            if (item && item.timestamp) {
              try {
                const dateStr = new Date(item.timestamp).toISOString().split('T')[0];
                const value = typeof item.value === 'number' ? item.value : parseFloat(item.value) || 0;
                const existing = calculationsDataMap.get(dateStr) || { pr: 0, availability: 0 };
                existing.availability = value;
                calculationsDataMap.set(dateStr, existing);
                parsedCount++;
              } catch (e) {
                // console.warn(`[KPI Live] Error parsing VFG timestamp:`, item.timestamp, e);
              }
            }
          });
          
          if (batchIdx === 0) {
            // Parsed VFG data
          }
        } else if (batchIdx === 0) {
          // console.warn(`[KPI Live] Calculations VFG response has no data field`);
        }

        // Combine data for all dates in batch
        const allDates = new Set<string>();
        basicsDataMap.forEach((_, date) => allDates.add(date));
        calculationsDataMap.forEach((_, date) => allDates.add(date));

        // Generate all dates in batch range
        const batchStart = new Date(batch.start);
        batchStart.setHours(0, 0, 0, 0);
        const batchEnd = new Date(batch.end);
        batchEnd.setHours(23, 59, 59, 999);
        
        const currentDate = new Date(batchStart);
        while (currentDate <= batchEnd && currentDate <= today) {
          const dateStr = currentDate.toISOString().split('T')[0];
          allDates.add(dateStr);
          currentDate.setDate(currentDate.getDate() + 1);
        }

        // Create KPI data for each date
        let daysWithData = 0;
        let daysWithProduction = 0;
        let daysWithPR = 0;
        let daysWithAvailability = 0;
        let daysWithIrradiation = 0;
        
        Array.from(allDates).sort().forEach((dateStr) => {
          const dateObj = new Date(dateStr + 'T00:00:00');
          if (dateObj > today) return; // Skip future dates
          
          const basics = basicsDataMap.get(dateStr) || { production: 0, irradiation: 0 };
          const calculations = calculationsDataMap.get(dateStr) || { pr: 0, availability: 0 };
          
          // Ensure all values are numbers (not null/undefined)
          const production = typeof basics.production === 'number' ? basics.production : 0;
          const pr = typeof calculations.pr === 'number' ? calculations.pr : 0;
          const availability = typeof calculations.availability === 'number' ? calculations.availability : 0;
          const irradiation = typeof basics.irradiation === 'number' ? basics.irradiation : 0;
          
          // Calculate Specific Yield (kWh/kWp)
          // Formula: Specific Yield = Production (kWh) / Capacity (kWp)
          // Note: Production from API (E_Z_EVU) is in kWh, Capacity is converted from MWp to kWp
          const specificYield = capacityKWp > 0 ? production / capacityKWp : 0;
          
          const kpiRecord = {
            date: dateObj,
            production,
            pr,
            availability,
            specificYield,
            irradiation,
          };
          
          kpiDataArray.push(kpiRecord);
          
          // Track statistics (include 0 values for availability if they exist)
          if (production > 0) daysWithProduction++;
          if (pr > 0) daysWithPR++;
          if (availability >= 0 && availability !== null && availability !== undefined) daysWithAvailability++; // Count even if 0
          if (irradiation > 0) daysWithIrradiation++;
          if (production > 0 || pr > 0 || (availability >= 0 && availability !== null) || irradiation > 0) {
            daysWithData++;
          }
          
        });

        
        // Small delay between batches to respect rate limits
        if (batchIdx < batches.length - 1) {
          await new Promise(resolve => setTimeout(resolve, 1000));
        }
      } catch (error: any) {
        // console.error(`[KPI Live] Error processing batch ${batchIdx + 1}:`, error.message);
        // Continue with next batch
      }
    }

    return kpiDataArray.sort((a, b) => a.date.getTime() - b.date.getTime());
  } catch (error: any) {
    // console.error(`[KPI Live] Error getting KPIs from V2 API:`, error.message);
    throw error;
  }
}

/**
 * Collect KPI data for a single project
 */
export async function collectKPIForProject(
  projectId: string,
  targetDate: Date = new Date()
): Promise<boolean> {
  try {
    // Get project
    const project = await Project.findById(projectId);
    if (!project) {
      // console.error(`[KPI Collector] Project not found: ${projectId}`);
      return false;
    }

    // Check if project has Meteocontrol config
    if (!project.meteocontrol?.siteKey || !project.meteocontrol?.apiKey) {
      // console.log(`[KPI Collector] Project ${projectId} does not have Meteocontrol config, skipping`);
      return false;
    }

    const config = project.meteocontrol;
    const systemKey = config.systemKey ?? config.siteKey;
    if (!systemKey) {
      // console.log(`[KPI Collector] Project ${projectId} has no systemKey/siteKey, skipping`);
      return false;
    }

    // Get V2 API credentials from SystemSettings
    const settings = await SystemSettings.findOne();
    if (!settings?.vcomApi?.username || !settings?.vcomApi?.password || !settings?.vcomApi?.apiKey) {
      // console.error(`[KPI Collector] V2 API credentials not configured in SystemSettings`);
      return false;
    }

    const v2Credentials: MeteocontrolV2Credentials = {
      username: settings.vcomApi.username,
      password: settings.vcomApi.password,
      apiKey: settings.vcomApi.apiKey,
    };

    // Try to get KPI data from V2 API (preferred)
    let kpiData: KPIData | null = null;
    const useV2API = config.useV2API !== false; // Default to true

    if (useV2API) {
      kpiData = await getKPIDataFromV2API(systemKey, targetDate, v2Credentials);
    }

    // Fallback to Widget API if V2 API fails or not configured
    if (!kpiData || kpiData.production === 0) {
      // console.log(`[KPI Collector] Falling back to Widget API for project ${projectId}`);
      const siteKey = config.siteKey ?? config.apiKey;
      if (!siteKey || !config.apiKey) return false;
      const production = await getProductionFromWidgetAPI(siteKey, config.apiKey, targetDate);
      if (production > 0) {
        kpiData = {
          production,
          pr: 0, // Not available from Widget API
          availability: 0, // Not available from Widget API
          specificYield: 0,
        };
      }
    }

    if (!kpiData || kpiData.production === 0) {
      // console.log(`[KPI Collector] No KPI data available for project ${projectId} on ${targetDate.toISOString().split('T')[0]}`);
      return false;
    }

    // Calculate Specific Yield (kWh/kWp)
    // Formula: Specific Yield = Production (kWh) / Capacity (kWp)
    // Note: Production from Widget API is in kWh, Capacity is converted from MWp to kWp
    const capacityMWp = project.capacityMWp || 0;
    const capacityKWp = capacityMWp * 1000;
    if (capacityKWp > 0) {
      const calculatedSpecificYield = kpiData.production / capacityKWp;
      kpiData.specificYield = calculatedSpecificYield;
      
      // Log calculation for debugging
      // console.log(`[KPI Collector] Calculated Specific Yield for ${targetDate.toISOString().split('T')[0]}:`, {
      //   production: `${kpiData.production.toFixed(2)} kWh`,
      //   capacity: `${capacityKWp.toFixed(2)} kWp (${capacityMWp.toFixed(2)} MWp)`,
      //   specificYield: `${calculatedSpecificYield.toFixed(4)} kWh/kWp`,
      //   calculation: `specificYield = ${kpiData.production.toFixed(2)} / ${capacityKWp.toFixed(2)} = ${calculatedSpecificYield.toFixed(4)}`,
      // });
    } else {
      // console.warn(`[KPI Collector] Cannot calculate Specific Yield: capacity is 0 (capacityMWp: ${capacityMWp})`);
      kpiData.specificYield = 0;
    }

    // Check if KPI already exists for this date
    const targetDateStart = new Date(targetDate);
    targetDateStart.setHours(0, 0, 0, 0);
    const targetDateEnd = new Date(targetDate);
    targetDateEnd.setHours(23, 59, 59, 999);

    const existingKPI = await KPI.findOne({
      siteId: projectId,
      date: {
        $gte: targetDateStart,
        $lt: targetDateEnd,
      },
    });

    const kpiDataToSave = {
      siteId: new mongoose.Types.ObjectId(projectId),
      date: targetDateStart,
      production: kpiData.production,
      pr: kpiData.pr,
      availability: kpiData.availability,
      specificYield: kpiData.specificYield,
      irradiation: kpiData.irradiation || 0,
    };

    if (existingKPI) {
      // Update existing KPI
      await KPI.findByIdAndUpdate(existingKPI._id, kpiDataToSave);
      // console.log(`[KPI Collector] Updated KPI for project ${projectId} on ${targetDate.toISOString().split('T')[0]}`);
    } else {
      // Create new KPI
      await KPI.create(kpiDataToSave);
      // console.log(`[KPI Collector] Created KPI for project ${projectId} on ${targetDate.toISOString().split('T')[0]}`);
    }

    return true;
  } catch (error: any) {
    // console.error(`[KPI Collector] Error collecting KPI for project ${projectId}:`, error.message);
    return false;
  }
}

/**
 * Collect KPI data for all projects with Meteocontrol config
 */
export async function collectKPIForAllProjects(targetDate: Date = new Date()): Promise<void> {
  try {
    // console.log(`[KPI Collector] Starting KPI collection for date: ${targetDate.toISOString().split('T')[0]}`);

    // Get all projects with Meteocontrol config
    const projects = await Project.find({
      'meteocontrol.siteKey': { $exists: true, $ne: '' },
      'meteocontrol.apiKey': { $exists: true, $ne: '' },
    });

    // console.log(`[KPI Collector] Found ${projects.length} projects with Meteocontrol config`);

    let successCount = 0;
    let failCount = 0;

    for (const project of projects) {
      const success = await collectKPIForProject(project._id.toString(), targetDate);
      if (success) {
        successCount++;
      } else {
        failCount++;
      }

      // Add small delay to respect rate limits (90 calls/minute)
      await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay
    }

    // console.log(`[KPI Collector] Completed: ${successCount} success, ${failCount} failed`);
  } catch (error: any) {
    // console.error('[KPI Collector] Error in collectKPIForAllProjects:', error.message);
    throw error;
  }
}