/** * 数据库查询优化器 * @file query-optimizer.js * @description 监控和优化SQL查询性能 */ const { sequelize } = require('../config/database-pool'); const { QueryTypes } = require('sequelize'); // 查询性能日志 class QueryPerformanceLog { constructor() { this.logs = []; this.maxLogs = 200; // 最多保存200条日志 this.slowQueryThreshold = 500; // 慢查询阈值(毫秒) this.queryPatterns = new Map(); // 存储查询模式及其统计信息 } // 添加日志 add(query, duration, params = {}) { const log = { query, duration, params, timestamp: new Date(), isSlow: duration > this.slowQueryThreshold }; this.logs.unshift(log); // 添加到开头 // 保持日志数量不超过最大值 if (this.logs.length > this.maxLogs) { this.logs.pop(); } // 如果是慢查询,输出警告 if (log.isSlow) { console.warn(`慢查询 (${duration}ms): ${query}`); console.warn('参数:', params); } // 更新查询模式统计 this.updateQueryPatternStats(query, duration); return log; } // 更新查询模式统计 updateQueryPatternStats(query, duration) { // 简化查询,移除具体参数值,保留查询结构 const patternQuery = this.simplifyQuery(query); if (!this.queryPatterns.has(patternQuery)) { this.queryPatterns.set(patternQuery, { count: 0, totalDuration: 0, avgDuration: 0, minDuration: duration, maxDuration: duration, lastSeen: new Date() }); } const stats = this.queryPatterns.get(patternQuery); stats.count++; stats.totalDuration += duration; stats.avgDuration = stats.totalDuration / stats.count; stats.minDuration = Math.min(stats.minDuration, duration); stats.maxDuration = Math.max(stats.maxDuration, duration); stats.lastSeen = new Date(); } // 简化查询,移除具体参数值 simplifyQuery(query) { return query .replace(/('([^']*)'|"([^"]*)")/g, '?') // 替换字符串 .replace(/\b\d+\b/g, '?') // 替换数字 .replace(/\s+/g, ' ') // 规范化空白字符 .trim(); } // 获取所有日志 getLogs() { return this.logs; } // 获取慢查询日志 getSlowLogs() { return this.logs.filter(log => log.isSlow); } // 获取查询模式统计 getQueryPatternStats() { return Array.from(this.queryPatterns.entries()).map(([pattern, stats]) => ({ pattern, ...stats })); } // 获取最常见的查询模式 getMostFrequentQueries(limit = 10) { return this.getQueryPatternStats() .sort((a, b) => b.count - a.count) .slice(0, limit); } // 获取平均执行时间最长的查询模式 getSlowestQueries(limit = 10) { return this.getQueryPatternStats() .sort((a, b) => b.avgDuration - a.avgDuration) .slice(0, limit); } // 清除日志 clear() { this.logs = []; this.queryPatterns.clear(); } // 设置慢查询阈值 setSlowQueryThreshold(threshold) { this.slowQueryThreshold = threshold; return this.slowQueryThreshold; } // 获取慢查询阈值 getSlowQueryThreshold() { return this.slowQueryThreshold; } } // 创建性能日志实例 const performanceLog = new QueryPerformanceLog(); // 查询优化器 class QueryOptimizer { constructor(sequelize, performanceLog) { this.sequelize = sequelize; this.performanceLog = performanceLog; this.indexSuggestions = new Map(); // 存储索引建议 this.setupLogging(); } // 设置查询日志记录 setupLogging() { // 如果已经在开发环境中启用了benchmark,则不需要额外设置 if (!this.sequelize.options.benchmark) { const originalQuery = this.sequelize.query.bind(this.sequelize); this.sequelize.query = async function (...args) { const start = Date.now(); try { const result = await originalQuery(...args); const duration = Date.now() - start; // 记录查询性能 performanceLog.add(args[0], duration, args[1]); return result; } catch (error) { const duration = Date.now() - start; performanceLog.add(`ERROR: ${args[0]}`, duration, { ...args[1], error: error.message }); throw error; } }; } } // 分析表结构并提供优化建议 async analyzeTableStructure(tableName) { try { // 获取表结构 const tableInfo = await this.getTableInfo(tableName); // 获取索引信息 const indexInfo = await this.getIndexInfo(tableName); // 获取表数据统计 const tableStats = await this.getTableStats(tableName); // 分析并生成建议 const suggestions = this.generateOptimizationSuggestions( tableName, tableInfo, indexInfo, tableStats ); return { tableName, structure: tableInfo, indexes: indexInfo, stats: tableStats, suggestions }; } catch (error) { console.error(`分析表 ${tableName} 结构失败:`, error); return { error: error.message }; } } // 生成优化建议 generateOptimizationSuggestions(tableName, tableInfo, indexInfo, tableStats) { const suggestions = []; // 检查是否有主键 const hasPrimaryKey = indexInfo.some(idx => idx.Key_name === 'PRIMARY'); if (!hasPrimaryKey) { suggestions.push({ type: 'missing_primary_key', importance: 'high', message: `表 ${tableName} 缺少主键,建议添加主键以提高性能` }); } // 检查大型TEXT/BLOB字段 const largeTextFields = tableInfo.structure.filter( field => field.Type.includes('text') || field.Type.includes('blob') ); if (largeTextFields.length > 0) { suggestions.push({ type: 'large_text_fields', importance: 'medium', message: `表 ${tableName} 包含 ${largeTextFields.length} 个大型TEXT/BLOB字段,考虑将不常用的大型字段移至单独的表中` }); } // 检查表大小 if (tableStats && tableStats.data_length > 100 * 1024 * 1024) { // 大于100MB suggestions.push({ type: 'large_table', importance: 'medium', message: `表 ${tableName} 较大 (${Math.round(tableStats.data_length / (1024 * 1024))}MB),考虑分区或归档旧数据` }); } // 从查询日志中分析可能需要的索引 const suggestedIndexes = this.suggestIndexesFromQueries(tableName); suggestedIndexes.forEach(suggestion => { suggestions.push({ type: 'suggested_index', importance: 'high', message: `建议在表 ${tableName} 的 ${suggestion.columns.join(', ')} 列上创建索引,可能提高查询性能`, details: suggestion }); }); return suggestions; } // 从查询日志中分析可能需要的索引 suggestIndexesFromQueries(tableName) { // 获取与该表相关的慢查询 const tableQueries = this.performanceLog.getSlowLogs() .filter(log => log.query.includes(tableName)); // 简单的索引建议逻辑 - 实际实现会更复杂 const suggestions = []; // 检查WHERE子句中频繁使用的列 const wherePattern = new RegExp(`${tableName}\\s+WHERE\\s+([\\w\\s,=<>!]+)`, 'i'); tableQueries.forEach(log => { const match = log.query.match(wherePattern); if (match && match[1]) { const whereClause = match[1]; // 提取列名 - 这是一个简化的实现 const columnMatches = whereClause.match(/\b(\w+)\b\s*[=<>!]/g); if (columnMatches) { const columns = columnMatches.map(col => col.trim().replace(/[=<>!\s]/g, '')); // 检查这些列是否已经在建议中 const key = columns.sort().join(','); if (!this.indexSuggestions.has(key)) { this.indexSuggestions.set(key, { tableName, columns, queryCount: 1, avgDuration: log.duration }); } else { const suggestion = this.indexSuggestions.get(key); suggestion.queryCount++; suggestion.avgDuration = (suggestion.avgDuration * (suggestion.queryCount - 1) + log.duration) / suggestion.queryCount; } } } }); // 返回针对该表的索引建议 return Array.from(this.indexSuggestions.values()) .filter(suggestion => suggestion.tableName === tableName) .sort((a, b) => b.queryCount - a.queryCount); } // 获取表统计信息 async getTableStats(tableName) { try { const stats = await this.sequelize.query( 'SHOW TABLE STATUS LIKE ?', { replacements: [tableName], type: QueryTypes.SELECT } ); return stats[0] || null; } catch (error) { console.error(`获取表 ${tableName} 统计信息失败:`, error); return null; } } // 分析和优化表 async analyzeAndOptimizeTable(tableName) { try { // 分析表 await this.sequelize.query(`ANALYZE TABLE ${tableName}`, { type: QueryTypes.RAW }); // 优化表 const optimizeResult = await this.sequelize.query(`OPTIMIZE TABLE ${tableName}`, { type: QueryTypes.RAW }); return optimizeResult[0]; } catch (error) { console.error(`分析和优化表 ${tableName} 失败:`, error); return { error: error.message }; } } // 获取表的索引信息 async getIndexInfo(tableName) { try { const indexInfo = await this.sequelize.query( 'SHOW INDEX FROM ??', { replacements: [tableName], type: QueryTypes.SELECT } ); return indexInfo; } catch (error) { console.error(`获取表 ${tableName} 的索引信息失败:`, error); return []; } } // 获取表信息 async getTableInfo(tableName) { try { // 获取表状态 const tableStatus = await this.sequelize.query( 'SHOW TABLE STATUS LIKE ?', { replacements: [tableName], type: QueryTypes.SELECT } ); // 获取表结构 const tableStructure = await this.sequelize.query( 'DESCRIBE ??', { replacements: [tableName], type: QueryTypes.SELECT } ); return { status: tableStatus[0] || {}, structure: tableStructure }; } catch (error) { console.error(`获取表 ${tableName} 信息失败:`, error); return { error: error.message }; } } // 解释查询计划 async explainQuery(query, params = {}) { try { // 执行EXPLAIN const explainResult = await this.sequelize.query( `EXPLAIN ${query}`, { replacements: params.replacements || [], type: QueryTypes.SELECT } ); return explainResult; } catch (error) { console.error('解释查询计划失败:', error); return []; } } // 识别慢查询 async identifySlowQueries(threshold = this.performanceLog.slowQueryThreshold) { try { // 从性能日志中获取慢查询 const slowLogs = this.performanceLog.getSlowLogs(); // 如果性能日志中没有足够的数据,则从数据库中查询 if (slowLogs.length < 5) { const dbSlowQueries = await this.sequelize.query( 'SELECT * FROM information_schema.PROCESSLIST WHERE TIME > ?', { replacements: [threshold / 1000], // 转换为秒 type: QueryTypes.SELECT } ); return dbSlowQueries; } return slowLogs; } catch (error) { console.error('识别慢查询失败:', error); return []; } } // 获取数据库状态 async getDatabaseStatus() { try { // 获取全局状态 const globalStatus = await this.sequelize.query( 'SHOW GLOBAL STATUS', { type: QueryTypes.SELECT } ); // 转换为对象格式 const status = {}; globalStatus.forEach(item => { if (item.Variable_name && item.Value) { status[item.Variable_name] = item.Value; } }); // 提取关键指标 return { connections: { max_used: status.Max_used_connections, current: status.Threads_connected, running: status.Threads_running, created: status.Threads_created, cached: status.Threads_cached }, queries: { total: status.Questions, slow: status.Slow_queries, qps: status.Queries }, buffer_pool: { size: status.Innodb_buffer_pool_pages_total, free: status.Innodb_buffer_pool_pages_free, dirty: status.Innodb_buffer_pool_pages_dirty, reads: status.Innodb_buffer_pool_reads, read_requests: status.Innodb_buffer_pool_read_requests, hit_rate: status.Innodb_buffer_pool_read_requests && status.Innodb_buffer_pool_reads ? (1 - parseInt(status.Innodb_buffer_pool_reads) / parseInt(status.Innodb_buffer_pool_read_requests)) * 100 : 0 }, performance: { slow_queries_count: this.performanceLog.getSlowLogs().length, total_queries_logged: this.performanceLog.getLogs().length, query_patterns: this.performanceLog.getQueryPatternStats().length } }; } catch (error) { console.error('获取数据库状态失败:', error); return { error: error.message }; } } // 获取所有查询日志 getAllQueries() { return this.performanceLog.getLogs(); } // 获取慢查询 getSlowQueries() { return this.performanceLog.getSlowLogs(); } // 获取查询模式统计 getQueryPatternStats() { return this.performanceLog.getQueryPatternStats(); } // 获取最常见的查询 getMostFrequentQueries(limit = 10) { return this.performanceLog.getMostFrequentQueries(limit); } // 获取最慢的查询 getSlowestQueries(limit = 10) { return this.performanceLog.getSlowestQueries(limit); } // 设置慢查询阈值 setSlowQueryThreshold(threshold) { return this.performanceLog.setSlowQueryThreshold(threshold); } // 清除性能日志 clearPerformanceLogs() { this.performanceLog.clear(); this.indexSuggestions.clear(); return { success: true, message: '性能日志已清除' }; } } // 创建查询优化器实例 const queryOptimizer = new QueryOptimizer(sequelize, performanceLog); module.exports = queryOptimizer;