517 lines
15 KiB
JavaScript
517 lines
15 KiB
JavaScript
/**
|
||
* 数据库查询优化器
|
||
* @file query-optimizer.js
|
||
* @description 监控和优化SQL查询性能
|
||
*/
|
||
const { sequelize } = require('../config/database-pool');
|
||
const { QueryTypes } = require('sequelize');
|
||
|
||
// 查询性能日志
|
||
class QueryPerformanceLog {
|
||
constructor() {
|
||
this.logs = [];
|
||
this.maxLogs = 200; // 最多保存200条日志
|
||
this.slowQueryThreshold = 500; // 慢查询阈值（毫秒）
|
||
this.queryPatterns = new Map(); // 存储查询模式及其统计信息
|
||
}
|
||
|
||
// 添加日志
|
||
add(query, duration, params = {}) {
|
||
const log = {
|
||
query,
|
||
duration,
|
||
params,
|
||
timestamp: new Date(),
|
||
isSlow: duration > this.slowQueryThreshold
|
||
};
|
||
|
||
this.logs.unshift(log); // 添加到开头
|
||
|
||
// 保持日志数量不超过最大值
|
||
if (this.logs.length > this.maxLogs) {
|
||
this.logs.pop();
|
||
}
|
||
|
||
// 如果是慢查询，输出警告
|
||
if (log.isSlow) {
|
||
console.warn(`慢查询 (${duration}ms): ${query}`);
|
||
console.warn('参数:', params);
|
||
}
|
||
|
||
// 更新查询模式统计
|
||
this.updateQueryPatternStats(query, duration);
|
||
|
||
return log;
|
||
}
|
||
|
||
// 更新查询模式统计
|
||
updateQueryPatternStats(query, duration) {
|
||
// 简化查询，移除具体参数值，保留查询结构
|
||
const patternQuery = this.simplifyQuery(query);
|
||
|
||
if (!this.queryPatterns.has(patternQuery)) {
|
||
this.queryPatterns.set(patternQuery, {
|
||
count: 0,
|
||
totalDuration: 0,
|
||
avgDuration: 0,
|
||
minDuration: duration,
|
||
maxDuration: duration,
|
||
lastSeen: new Date()
|
||
});
|
||
}
|
||
|
||
const stats = this.queryPatterns.get(patternQuery);
|
||
stats.count++;
|
||
stats.totalDuration += duration;
|
||
stats.avgDuration = stats.totalDuration / stats.count;
|
||
stats.minDuration = Math.min(stats.minDuration, duration);
|
||
stats.maxDuration = Math.max(stats.maxDuration, duration);
|
||
stats.lastSeen = new Date();
|
||
}
|
||
|
||
// 简化查询，移除具体参数值
|
||
simplifyQuery(query) {
|
||
return query
|
||
.replace(/('([^']*)'|"([^"]*)")/g, '?') // 替换字符串
|
||
.replace(/\b\d+\b/g, '?') // 替换数字
|
||
.replace(/\s+/g, ' ') // 规范化空白字符
|
||
.trim();
|
||
}
|
||
|
||
// 获取所有日志
|
||
getLogs() {
|
||
return this.logs;
|
||
}
|
||
|
||
// 获取慢查询日志
|
||
getSlowLogs() {
|
||
return this.logs.filter(log => log.isSlow);
|
||
}
|
||
|
||
// 获取查询模式统计
|
||
getQueryPatternStats() {
|
||
return Array.from(this.queryPatterns.entries()).map(([pattern, stats]) => ({
|
||
pattern,
|
||
...stats
|
||
}));
|
||
}
|
||
|
||
// 获取最常见的查询模式
|
||
getMostFrequentQueries(limit = 10) {
|
||
return this.getQueryPatternStats()
|
||
.sort((a, b) => b.count - a.count)
|
||
.slice(0, limit);
|
||
}
|
||
|
||
// 获取平均执行时间最长的查询模式
|
||
getSlowestQueries(limit = 10) {
|
||
return this.getQueryPatternStats()
|
||
.sort((a, b) => b.avgDuration - a.avgDuration)
|
||
.slice(0, limit);
|
||
}
|
||
|
||
// 清除日志
|
||
clear() {
|
||
this.logs = [];
|
||
this.queryPatterns.clear();
|
||
}
|
||
|
||
// 设置慢查询阈值
|
||
setSlowQueryThreshold(threshold) {
|
||
this.slowQueryThreshold = threshold;
|
||
return this.slowQueryThreshold;
|
||
}
|
||
|
||
// 获取慢查询阈值
|
||
getSlowQueryThreshold() {
|
||
return this.slowQueryThreshold;
|
||
}
|
||
}
|
||
|
||
// 创建性能日志实例
|
||
const performanceLog = new QueryPerformanceLog();
|
||
|
||
// 查询优化器
|
||
class QueryOptimizer {
|
||
constructor(sequelize, performanceLog) {
|
||
this.sequelize = sequelize;
|
||
this.performanceLog = performanceLog;
|
||
this.indexSuggestions = new Map(); // 存储索引建议
|
||
this.setupLogging();
|
||
}
|
||
|
||
// 设置查询日志记录
|
||
setupLogging() {
|
||
// 如果已经在开发环境中启用了benchmark，则不需要额外设置
|
||
if (!this.sequelize.options.benchmark) {
|
||
const originalQuery = this.sequelize.query.bind(this.sequelize);
|
||
|
||
this.sequelize.query = async function (...args) {
|
||
const start = Date.now();
|
||
try {
|
||
const result = await originalQuery(...args);
|
||
const duration = Date.now() - start;
|
||
|
||
// 记录查询性能
|
||
performanceLog.add(args[0], duration, args[1]);
|
||
|
||
return result;
|
||
} catch (error) {
|
||
const duration = Date.now() - start;
|
||
performanceLog.add(`ERROR: ${args[0]}`, duration, { ...args[1], error: error.message });
|
||
throw error;
|
||
}
|
||
};
|
||
}
|
||
}
|
||
|
||
// 分析表结构并提供优化建议
|
||
async analyzeTableStructure(tableName) {
|
||
try {
|
||
// 获取表结构
|
||
const tableInfo = await this.getTableInfo(tableName);
|
||
|
||
// 获取索引信息
|
||
const indexInfo = await this.getIndexInfo(tableName);
|
||
|
||
// 获取表数据统计
|
||
const tableStats = await this.getTableStats(tableName);
|
||
|
||
// 分析并生成建议
|
||
const suggestions = this.generateOptimizationSuggestions(
|
||
tableName,
|
||
tableInfo,
|
||
indexInfo,
|
||
tableStats
|
||
);
|
||
|
||
return {
|
||
tableName,
|
||
structure: tableInfo,
|
||
indexes: indexInfo,
|
||
stats: tableStats,
|
||
suggestions
|
||
};
|
||
} catch (error) {
|
||
console.error(`分析表 ${tableName} 结构失败:`, error);
|
||
return { error: error.message };
|
||
}
|
||
}
|
||
|
||
// 生成优化建议
|
||
generateOptimizationSuggestions(tableName, tableInfo, indexInfo, tableStats) {
|
||
const suggestions = [];
|
||
|
||
// 检查是否有主键
|
||
const hasPrimaryKey = indexInfo.some(idx => idx.Key_name === 'PRIMARY');
|
||
if (!hasPrimaryKey) {
|
||
suggestions.push({
|
||
type: 'missing_primary_key',
|
||
importance: 'high',
|
||
message: `表 ${tableName} 缺少主键，建议添加主键以提高性能`
|
||
});
|
||
}
|
||
|
||
// 检查大型TEXT/BLOB字段
|
||
const largeTextFields = tableInfo.structure.filter(
|
||
field => field.Type.includes('text') || field.Type.includes('blob')
|
||
);
|
||
if (largeTextFields.length > 0) {
|
||
suggestions.push({
|
||
type: 'large_text_fields',
|
||
importance: 'medium',
|
||
message: `表 ${tableName} 包含 ${largeTextFields.length} 个大型TEXT/BLOB字段，考虑将不常用的大型字段移至单独的表中`
|
||
});
|
||
}
|
||
|
||
// 检查表大小
|
||
if (tableStats && tableStats.data_length > 100 * 1024 * 1024) { // 大于100MB
|
||
suggestions.push({
|
||
type: 'large_table',
|
||
importance: 'medium',
|
||
message: `表 ${tableName} 较大 (${Math.round(tableStats.data_length / (1024 * 1024))}MB)，考虑分区或归档旧数据`
|
||
});
|
||
}
|
||
|
||
// 从查询日志中分析可能需要的索引
|
||
const suggestedIndexes = this.suggestIndexesFromQueries(tableName);
|
||
suggestedIndexes.forEach(suggestion => {
|
||
suggestions.push({
|
||
type: 'suggested_index',
|
||
importance: 'high',
|
||
message: `建议在表 ${tableName} 的 ${suggestion.columns.join(', ')} 列上创建索引，可能提高查询性能`,
|
||
details: suggestion
|
||
});
|
||
});
|
||
|
||
return suggestions;
|
||
}
|
||
|
||
// 从查询日志中分析可能需要的索引
|
||
suggestIndexesFromQueries(tableName) {
|
||
// 获取与该表相关的慢查询
|
||
const tableQueries = this.performanceLog.getSlowLogs()
|
||
.filter(log => log.query.includes(tableName));
|
||
|
||
// 简单的索引建议逻辑 - 实际实现会更复杂
|
||
const suggestions = [];
|
||
|
||
// 检查WHERE子句中频繁使用的列
|
||
const wherePattern = new RegExp(`${tableName}\\s+WHERE\\s+([\\w\\s,=<>!]+)`, 'i');
|
||
|
||
tableQueries.forEach(log => {
|
||
const match = log.query.match(wherePattern);
|
||
if (match && match[1]) {
|
||
const whereClause = match[1];
|
||
// 提取列名 - 这是一个简化的实现
|
||
const columnMatches = whereClause.match(/\b(\w+)\b\s*[=<>!]/g);
|
||
if (columnMatches) {
|
||
const columns = columnMatches.map(col => col.trim().replace(/[=<>!\s]/g, ''));
|
||
|
||
// 检查这些列是否已经在建议中
|
||
const key = columns.sort().join(',');
|
||
if (!this.indexSuggestions.has(key)) {
|
||
this.indexSuggestions.set(key, {
|
||
tableName,
|
||
columns,
|
||
queryCount: 1,
|
||
avgDuration: log.duration
|
||
});
|
||
} else {
|
||
const suggestion = this.indexSuggestions.get(key);
|
||
suggestion.queryCount++;
|
||
suggestion.avgDuration = (suggestion.avgDuration * (suggestion.queryCount - 1) + log.duration) / suggestion.queryCount;
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
// 返回针对该表的索引建议
|
||
return Array.from(this.indexSuggestions.values())
|
||
.filter(suggestion => suggestion.tableName === tableName)
|
||
.sort((a, b) => b.queryCount - a.queryCount);
|
||
}
|
||
|
||
// 获取表统计信息
|
||
async getTableStats(tableName) {
|
||
try {
|
||
const stats = await this.sequelize.query(
|
||
'SHOW TABLE STATUS LIKE ?',
|
||
{
|
||
replacements: [tableName],
|
||
type: QueryTypes.SELECT
|
||
}
|
||
);
|
||
|
||
return stats[0] || null;
|
||
} catch (error) {
|
||
console.error(`获取表 ${tableName} 统计信息失败:`, error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
// 分析和优化表
|
||
async analyzeAndOptimizeTable(tableName) {
|
||
try {
|
||
// 分析表
|
||
await this.sequelize.query(`ANALYZE TABLE ${tableName}`, { type: QueryTypes.RAW });
|
||
|
||
// 优化表
|
||
const optimizeResult = await this.sequelize.query(`OPTIMIZE TABLE ${tableName}`, { type: QueryTypes.RAW });
|
||
|
||
return optimizeResult[0];
|
||
} catch (error) {
|
||
console.error(`分析和优化表 ${tableName} 失败:`, error);
|
||
return { error: error.message };
|
||
}
|
||
}
|
||
|
||
// 获取表的索引信息
|
||
async getIndexInfo(tableName) {
|
||
try {
|
||
const indexInfo = await this.sequelize.query(
|
||
'SHOW INDEX FROM ??',
|
||
{
|
||
replacements: [tableName],
|
||
type: QueryTypes.SELECT
|
||
}
|
||
);
|
||
|
||
return indexInfo;
|
||
} catch (error) {
|
||
console.error(`获取表 ${tableName} 的索引信息失败:`, error);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
// 获取表信息
|
||
async getTableInfo(tableName) {
|
||
try {
|
||
// 获取表状态
|
||
const tableStatus = await this.sequelize.query(
|
||
'SHOW TABLE STATUS LIKE ?',
|
||
{
|
||
replacements: [tableName],
|
||
type: QueryTypes.SELECT
|
||
}
|
||
);
|
||
|
||
// 获取表结构
|
||
const tableStructure = await this.sequelize.query(
|
||
'DESCRIBE ??',
|
||
{
|
||
replacements: [tableName],
|
||
type: QueryTypes.SELECT
|
||
}
|
||
);
|
||
|
||
return {
|
||
status: tableStatus[0] || {},
|
||
structure: tableStructure
|
||
};
|
||
} catch (error) {
|
||
console.error(`获取表 ${tableName} 信息失败:`, error);
|
||
return { error: error.message };
|
||
}
|
||
}
|
||
|
||
// 解释查询计划
|
||
async explainQuery(query, params = {}) {
|
||
try {
|
||
// 执行EXPLAIN
|
||
const explainResult = await this.sequelize.query(
|
||
`EXPLAIN ${query}`,
|
||
{
|
||
replacements: params.replacements || [],
|
||
type: QueryTypes.SELECT
|
||
}
|
||
);
|
||
|
||
return explainResult;
|
||
} catch (error) {
|
||
console.error('解释查询计划失败:', error);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
// 识别慢查询
|
||
async identifySlowQueries(threshold = this.performanceLog.slowQueryThreshold) {
|
||
try {
|
||
// 从性能日志中获取慢查询
|
||
const slowLogs = this.performanceLog.getSlowLogs();
|
||
|
||
// 如果性能日志中没有足够的数据，则从数据库中查询
|
||
if (slowLogs.length < 5) {
|
||
const dbSlowQueries = await this.sequelize.query(
|
||
'SELECT * FROM information_schema.PROCESSLIST WHERE TIME > ?',
|
||
{
|
||
replacements: [threshold / 1000], // 转换为秒
|
||
type: QueryTypes.SELECT
|
||
}
|
||
);
|
||
|
||
return dbSlowQueries;
|
||
}
|
||
|
||
return slowLogs;
|
||
} catch (error) {
|
||
console.error('识别慢查询失败:', error);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
// 获取数据库状态
|
||
async getDatabaseStatus() {
|
||
try {
|
||
// 获取全局状态
|
||
const globalStatus = await this.sequelize.query(
|
||
'SHOW GLOBAL STATUS',
|
||
{ type: QueryTypes.SELECT }
|
||
);
|
||
|
||
// 转换为对象格式
|
||
const status = {};
|
||
globalStatus.forEach(item => {
|
||
if (item.Variable_name && item.Value) {
|
||
status[item.Variable_name] = item.Value;
|
||
}
|
||
});
|
||
|
||
// 提取关键指标
|
||
return {
|
||
connections: {
|
||
max_used: status.Max_used_connections,
|
||
current: status.Threads_connected,
|
||
running: status.Threads_running,
|
||
created: status.Threads_created,
|
||
cached: status.Threads_cached
|
||
},
|
||
queries: {
|
||
total: status.Questions,
|
||
slow: status.Slow_queries,
|
||
qps: status.Queries
|
||
},
|
||
buffer_pool: {
|
||
size: status.Innodb_buffer_pool_pages_total,
|
||
free: status.Innodb_buffer_pool_pages_free,
|
||
dirty: status.Innodb_buffer_pool_pages_dirty,
|
||
reads: status.Innodb_buffer_pool_reads,
|
||
read_requests: status.Innodb_buffer_pool_read_requests,
|
||
hit_rate: status.Innodb_buffer_pool_read_requests && status.Innodb_buffer_pool_reads
|
||
? (1 - parseInt(status.Innodb_buffer_pool_reads) / parseInt(status.Innodb_buffer_pool_read_requests)) * 100
|
||
: 0
|
||
},
|
||
performance: {
|
||
slow_queries_count: this.performanceLog.getSlowLogs().length,
|
||
total_queries_logged: this.performanceLog.getLogs().length,
|
||
query_patterns: this.performanceLog.getQueryPatternStats().length
|
||
}
|
||
};
|
||
} catch (error) {
|
||
console.error('获取数据库状态失败:', error);
|
||
return { error: error.message };
|
||
}
|
||
}
|
||
|
||
// 获取所有查询日志
|
||
getAllQueries() {
|
||
return this.performanceLog.getLogs();
|
||
}
|
||
|
||
// 获取慢查询
|
||
getSlowQueries() {
|
||
return this.performanceLog.getSlowLogs();
|
||
}
|
||
|
||
// 获取查询模式统计
|
||
getQueryPatternStats() {
|
||
return this.performanceLog.getQueryPatternStats();
|
||
}
|
||
|
||
// 获取最常见的查询
|
||
getMostFrequentQueries(limit = 10) {
|
||
return this.performanceLog.getMostFrequentQueries(limit);
|
||
}
|
||
|
||
// 获取最慢的查询
|
||
getSlowestQueries(limit = 10) {
|
||
return this.performanceLog.getSlowestQueries(limit);
|
||
}
|
||
|
||
// 设置慢查询阈值
|
||
setSlowQueryThreshold(threshold) {
|
||
return this.performanceLog.setSlowQueryThreshold(threshold);
|
||
}
|
||
|
||
// 清除性能日志
|
||
clearPerformanceLogs() {
|
||
this.performanceLog.clear();
|
||
this.indexSuggestions.clear();
|
||
return { success: true, message: '性能日志已清除' };
|
||
}
|
||
}
|
||
|
||
// 创建查询优化器实例
|
||
const queryOptimizer = new QueryOptimizer(sequelize, performanceLog);
|
||
|
||
module.exports = queryOptimizer; |