nxxmdata/backend/utils/query-optimizer.js

/**
 * 数据库查询优化器
 * @file query-optimizer.js
 * @description 监控和优化SQL查询性能
 */
const { sequelize } = require('../config/database-pool');
const { QueryTypes } = require('sequelize');

// 查询性能日志
class QueryPerformanceLog {
  constructor() {
    this.logs = [];
    this.maxLogs = 200; // 最多保存200条日志
    this.slowQueryThreshold = 500; // 慢查询阈值（毫秒）
    this.queryPatterns = new Map(); // 存储查询模式及其统计信息
  }

  // 添加日志
  add(query, duration, params = {}) {
    const log = {
      query,
      duration,
      params,
      timestamp: new Date(),
      isSlow: duration > this.slowQueryThreshold
    };

    this.logs.unshift(log); // 添加到开头
    
    // 保持日志数量不超过最大值
    if (this.logs.length > this.maxLogs) {
      this.logs.pop();
    }

    // 如果是慢查询，输出警告
    if (log.isSlow) {
      console.warn(`慢查询 (${duration}ms): ${query}`);
      console.warn('参数:', params);
    }

    // 更新查询模式统计
    this.updateQueryPatternStats(query, duration);

    return log;
  }

  // 更新查询模式统计
  updateQueryPatternStats(query, duration) {
    // 简化查询，移除具体参数值，保留查询结构
    const patternQuery = this.simplifyQuery(query);
    
    if (!this.queryPatterns.has(patternQuery)) {
      this.queryPatterns.set(patternQuery, {
        count: 0,
        totalDuration: 0,
        avgDuration: 0,
        minDuration: duration,
        maxDuration: duration,
        lastSeen: new Date()
      });
    }
    
    const stats = this.queryPatterns.get(patternQuery);
    stats.count++;
    stats.totalDuration += duration;
    stats.avgDuration = stats.totalDuration / stats.count;
    stats.minDuration = Math.min(stats.minDuration, duration);
    stats.maxDuration = Math.max(stats.maxDuration, duration);
    stats.lastSeen = new Date();
  }

  // 简化查询，移除具体参数值
  simplifyQuery(query) {
    return query
      .replace(/('([^']*)'|"([^"]*)")/g, '?') // 替换字符串
      .replace(/\b\d+\b/g, '?') // 替换数字
      .replace(/\s+/g, ' ') // 规范化空白字符
      .trim();
  }

  // 获取所有日志
  getLogs() {
    return this.logs;
  }

  // 获取慢查询日志
  getSlowLogs() {
    return this.logs.filter(log => log.isSlow);
  }

  // 获取查询模式统计
  getQueryPatternStats() {
    return Array.from(this.queryPatterns.entries()).map(([pattern, stats]) => ({
      pattern,
      ...stats
    }));
  }

  // 获取最常见的查询模式
  getMostFrequentQueries(limit = 10) {
    return this.getQueryPatternStats()
      .sort((a, b) => b.count - a.count)
      .slice(0, limit);
  }

  // 获取平均执行时间最长的查询模式
  getSlowestQueries(limit = 10) {
    return this.getQueryPatternStats()
      .sort((a, b) => b.avgDuration - a.avgDuration)
      .slice(0, limit);
  }

  // 清除日志
  clear() {
    this.logs = [];
    this.queryPatterns.clear();
  }

  // 设置慢查询阈值
  setSlowQueryThreshold(threshold) {
    this.slowQueryThreshold = threshold;
    return this.slowQueryThreshold;
  }

  // 获取慢查询阈值
  getSlowQueryThreshold() {
    return this.slowQueryThreshold;
  }
}

// 创建性能日志实例
const performanceLog = new QueryPerformanceLog();

// 查询优化器
class QueryOptimizer {
  constructor(sequelize, performanceLog) {
    this.sequelize = sequelize;
    this.performanceLog = performanceLog;
    this.indexSuggestions = new Map(); // 存储索引建议
    this.setupLogging();
  }

  // 设置查询日志记录
  setupLogging() {
    // 如果已经在开发环境中启用了benchmark，则不需要额外设置
    if (!this.sequelize.options.benchmark) {
      const originalQuery = this.sequelize.query.bind(this.sequelize);
      
      this.sequelize.query = async function (...args) {
        const start = Date.now();
        try {
          const result = await originalQuery(...args);
          const duration = Date.now() - start;
          
          // 记录查询性能
          performanceLog.add(args[0], duration, args[1]);
          
          return result;
        } catch (error) {
          const duration = Date.now() - start;
          performanceLog.add(`ERROR: ${args[0]}`, duration, { ...args[1], error: error.message });
          throw error;
        }
      };
    }
  }

  // 分析表结构并提供优化建议
  async analyzeTableStructure(tableName) {
    try {
      // 获取表结构
      const tableInfo = await this.getTableInfo(tableName);
      
      // 获取索引信息
      const indexInfo = await this.getIndexInfo(tableName);
      
      // 获取表数据统计
      const tableStats = await this.getTableStats(tableName);
      
      // 分析并生成建议
      const suggestions = this.generateOptimizationSuggestions(
        tableName, 
        tableInfo, 
        indexInfo, 
        tableStats
      );
      
      return {
        tableName,
        structure: tableInfo,
        indexes: indexInfo,
        stats: tableStats,
        suggestions
      };
    } catch (error) {
      console.error(`分析表 ${tableName} 结构失败:`, error);
      return { error: error.message };
    }
  }

  // 生成优化建议
  generateOptimizationSuggestions(tableName, tableInfo, indexInfo, tableStats) {
    const suggestions = [];
    
    // 检查是否有主键
    const hasPrimaryKey = indexInfo.some(idx => idx.Key_name === 'PRIMARY');
    if (!hasPrimaryKey) {
      suggestions.push({
        type: 'missing_primary_key',
        importance: 'high',
        message: `表 ${tableName} 缺少主键，建议添加主键以提高性能`
      });
    }
    
    // 检查大型TEXT/BLOB字段
    const largeTextFields = tableInfo.structure.filter(
      field => field.Type.includes('text') || field.Type.includes('blob')
    );
    if (largeTextFields.length > 0) {
      suggestions.push({
        type: 'large_text_fields',
        importance: 'medium',
        message: `表 ${tableName} 包含 ${largeTextFields.length} 个大型TEXT/BLOB字段，考虑将不常用的大型字段移至单独的表中`
      });
    }
    
    // 检查表大小
    if (tableStats && tableStats.data_length > 100 * 1024 * 1024) { // 大于100MB
      suggestions.push({
        type: 'large_table',
        importance: 'medium',
        message: `表 ${tableName} 较大 (${Math.round(tableStats.data_length / (1024 * 1024))}MB)，考虑分区或归档旧数据`
      });
    }
    
    // 从查询日志中分析可能需要的索引
    const suggestedIndexes = this.suggestIndexesFromQueries(tableName);
    suggestedIndexes.forEach(suggestion => {
      suggestions.push({
        type: 'suggested_index',
        importance: 'high',
        message: `建议在表 ${tableName} 的 ${suggestion.columns.join(', ')} 列上创建索引，可能提高查询性能`,
        details: suggestion
      });
    });
    
    return suggestions;
  }

  // 从查询日志中分析可能需要的索引
  suggestIndexesFromQueries(tableName) {
    // 获取与该表相关的慢查询
    const tableQueries = this.performanceLog.getSlowLogs()
      .filter(log => log.query.includes(tableName));
    
    // 简单的索引建议逻辑 - 实际实现会更复杂
    const suggestions = [];
    
    // 检查WHERE子句中频繁使用的列
    const wherePattern = new RegExp(`${tableName}\\s+WHERE\\s+([\\w\\s,=<>!]+)`, 'i');
    
    tableQueries.forEach(log => {
      const match = log.query.match(wherePattern);
      if (match && match[1]) {
        const whereClause = match[1];
        // 提取列名 - 这是一个简化的实现
        const columnMatches = whereClause.match(/\b(\w+)\b\s*[=<>!]/g);
        if (columnMatches) {
          const columns = columnMatches.map(col => col.trim().replace(/[=<>!\s]/g, ''));
          
          // 检查这些列是否已经在建议中
          const key = columns.sort().join(',');
          if (!this.indexSuggestions.has(key)) {
            this.indexSuggestions.set(key, {
              tableName,
              columns,
              queryCount: 1,
              avgDuration: log.duration
            });
          } else {
            const suggestion = this.indexSuggestions.get(key);
            suggestion.queryCount++;
            suggestion.avgDuration = (suggestion.avgDuration * (suggestion.queryCount - 1) + log.duration) / suggestion.queryCount;
          }
        }
      }
    });
    
    // 返回针对该表的索引建议
    return Array.from(this.indexSuggestions.values())
      .filter(suggestion => suggestion.tableName === tableName)
      .sort((a, b) => b.queryCount - a.queryCount);
  }

  // 获取表统计信息
  async getTableStats(tableName) {
    try {
      const stats = await this.sequelize.query(
        'SHOW TABLE STATUS LIKE ?',
        {
          replacements: [tableName],
          type: QueryTypes.SELECT
        }
      );
      
      return stats[0] || null;
    } catch (error) {
      console.error(`获取表 ${tableName} 统计信息失败:`, error);
      return null;
    }
  }

  // 分析和优化表
  async analyzeAndOptimizeTable(tableName) {
    try {
      // 分析表
      await this.sequelize.query(`ANALYZE TABLE ${tableName}`, { type: QueryTypes.RAW });
      
      // 优化表
      const optimizeResult = await this.sequelize.query(`OPTIMIZE TABLE ${tableName}`, { type: QueryTypes.RAW });
      
      return optimizeResult[0];
    } catch (error) {
      console.error(`分析和优化表 ${tableName} 失败:`, error);
      return { error: error.message };
    }
  }

  // 获取表的索引信息
  async getIndexInfo(tableName) {
    try {
      const indexInfo = await this.sequelize.query(
        'SHOW INDEX FROM ??',
        {
          replacements: [tableName],
          type: QueryTypes.SELECT
        }
      );
      
      return indexInfo;
    } catch (error) {
      console.error(`获取表 ${tableName} 的索引信息失败:`, error);
      return [];
    }
  }

  // 获取表信息
  async getTableInfo(tableName) {
    try {
      // 获取表状态
      const tableStatus = await this.sequelize.query(
        'SHOW TABLE STATUS LIKE ?',
        {
          replacements: [tableName],
          type: QueryTypes.SELECT
        }
      );
      
      // 获取表结构
      const tableStructure = await this.sequelize.query(
        'DESCRIBE ??',
        {
          replacements: [tableName],
          type: QueryTypes.SELECT
        }
      );
      
      return {
        status: tableStatus[0] || {},
        structure: tableStructure
      };
    } catch (error) {
      console.error(`获取表 ${tableName} 信息失败:`, error);
      return { error: error.message };
    }
  }

  // 解释查询计划
  async explainQuery(query, params = {}) {
    try {
      // 执行EXPLAIN
      const explainResult = await this.sequelize.query(
        `EXPLAIN ${query}`,
        {
          replacements: params.replacements || [],
          type: QueryTypes.SELECT
        }
      );
      
      return explainResult;
    } catch (error) {
      console.error('解释查询计划失败:', error);
      return [];
    }
  }

  // 识别慢查询
  async identifySlowQueries(threshold = this.performanceLog.slowQueryThreshold) {
    try {
      // 从性能日志中获取慢查询
      const slowLogs = this.performanceLog.getSlowLogs();
      
      // 如果性能日志中没有足够的数据，则从数据库中查询
      if (slowLogs.length < 5) {
        const dbSlowQueries = await this.sequelize.query(
          'SELECT * FROM information_schema.PROCESSLIST WHERE TIME > ?',
          {
            replacements: [threshold / 1000], // 转换为秒
            type: QueryTypes.SELECT
          }
        );
        
        return dbSlowQueries;
      }
      
      return slowLogs;
    } catch (error) {
      console.error('识别慢查询失败:', error);
      return [];
    }
  }

  // 获取数据库状态
  async getDatabaseStatus() {
    try {
      // 获取全局状态
      const globalStatus = await this.sequelize.query(
        'SHOW GLOBAL STATUS',
        { type: QueryTypes.SELECT }
      );
      
      // 转换为对象格式
      const status = {};
      globalStatus.forEach(item => {
        if (item.Variable_name && item.Value) {
          status[item.Variable_name] = item.Value;
        }
      });
      
      // 提取关键指标
      return {
        connections: {
          max_used: status.Max_used_connections,
          current: status.Threads_connected,
          running: status.Threads_running,
          created: status.Threads_created,
          cached: status.Threads_cached
        },
        queries: {
          total: status.Questions,
          slow: status.Slow_queries,
          qps: status.Queries
        },
        buffer_pool: {
          size: status.Innodb_buffer_pool_pages_total,
          free: status.Innodb_buffer_pool_pages_free,
          dirty: status.Innodb_buffer_pool_pages_dirty,
          reads: status.Innodb_buffer_pool_reads,
          read_requests: status.Innodb_buffer_pool_read_requests,
          hit_rate: status.Innodb_buffer_pool_read_requests && status.Innodb_buffer_pool_reads
            ? (1 - parseInt(status.Innodb_buffer_pool_reads) / parseInt(status.Innodb_buffer_pool_read_requests)) * 100
            : 0
        },
        performance: {
          slow_queries_count: this.performanceLog.getSlowLogs().length,
          total_queries_logged: this.performanceLog.getLogs().length,
          query_patterns: this.performanceLog.getQueryPatternStats().length
        }
      };
    } catch (error) {
      console.error('获取数据库状态失败:', error);
      return { error: error.message };
    }
  }

  // 获取所有查询日志
  getAllQueries() {
    return this.performanceLog.getLogs();
  }

  // 获取慢查询
  getSlowQueries() {
    return this.performanceLog.getSlowLogs();
  }

  // 获取查询模式统计
  getQueryPatternStats() {
    return this.performanceLog.getQueryPatternStats();
  }

  // 获取最常见的查询
  getMostFrequentQueries(limit = 10) {
    return this.performanceLog.getMostFrequentQueries(limit);
  }

  // 获取最慢的查询
  getSlowestQueries(limit = 10) {
    return this.performanceLog.getSlowestQueries(limit);
  }

  // 设置慢查询阈值
  setSlowQueryThreshold(threshold) {
    return this.performanceLog.setSlowQueryThreshold(threshold);
  }

  // 清除性能日志
  clearPerformanceLogs() {
    this.performanceLog.clear();
    this.indexSuggestions.clear();
    return { success: true, message: '性能日志已清除' };
  }
}

// 创建查询优化器实例
const queryOptimizer = new QueryOptimizer(sequelize, performanceLog);

module.exports = queryOptimizer;