最近要调优SQL语句，于是运维倒了一份SQL日志给我，我需要知道那一些是查询慢，更新多，总之就是哪些语句执行多。
1、需要将MySQL日志解析：例如：
2、需要实现成：
思路：
1、SQL语句分类：SELECT、UPDATE、INSERT、DELETE(增、删，改，查)
2、针对更新语句、插入语句单独处理
3、正则匹配、字符串Split
4、Dictionary<string, int>键值组合
5、异步读写
开始代码：
1、正则表达式：
/// <summary>
/// 关键的运算付
/// </summary>
string[] keys = { "<", ">", "=", "!=" };
/// <summary>
/// 匹配Lik和in
/// </summary>
Regex patLike = new Regex(@"(\b)(like|in)(\b)", RegexOptions.IgnoreCase);
//时间
Regex patTime = new Regex(@"(\d{2}|\d{4})(?:\-)?([0]{1}\d{1}|[1]{1}[0-2]{1})(?:\-)?([0-2]{1}\d{1}|[3]{1}[0-1]{1})(?:\s)?([0-1]{1}\d{1}|[2]{1}[0-3]{1})(?::)?([0-5]{1}\d{1})(?::)?([0-5]{1}\d{1})|([\d+]{4}\/[\d+]{1,2}\/[\d+]{1,2} [\d+]{2}\:[\d+]{2}\:[\d+]{2})", RegexOptions.IgnoreCase);
//带SQL运算符
Regex patSql = new Regex(@"(=|<|>|<=|>=|!=|<>|like|in\s+\(|in\()(\s+\S+)|(=|<|>|<=|>=|!=|<>|like|in\s+\(|in\()(\S+)", RegexOptions.IgnoreCase);
/// <summary>
/// 获取单词
/// </summary>
Regex patArry = new Regex(@"([a-z_]+)");
/// <summary>
/// 字符集合
/// </summary>
MatchCollection matchsMade;
2、读取文件：
1 //resultFile 文件全路径
2 StreamReader sr = new StreamReader(resultFile, Encoding.UTF8);
3 int i = 0;
4 String line;
5 string strSQL = string.Empty;
6 int IsNumberCount = 0;//
7 int lineNumber = 0;//本行行号
8 string Number = string.Empty;
9
10 string[] KeysArry = { "init", "connect", "quit", "statistics", "begin", "show", "commit" };
11
12 bool isKeys = false;
13 while ((line = sr.ReadLine()) != null)
14 {
15 i++;
16 line = line.Trim();//去除首位空白
17 line = line.ToLower();
18 isKeys = false;
19 foreach (string s in KeysArry)
20 {
21 if (line.IndexOf(s) > -1)
22 {
23 isKeys = true;
24 continue;
25 }
26 }
27 if (isKeys)
28 {
29 continue;
30 }
31 line = line.Replace("\t", "");
32
33 if (line.IndexOf("query") > -1)
34 {
35 IsNumberCount += 1;
36 if (lineNumber == 1 && IsNumberCount == 2)
37 {
38 analyzeSQLString(strSQL);
39 IsNumberCount = 1;
40 strSQL = "";
41 }
42 if (lineNumber != 1 && IsNumberCount == 1)
43 {
44 if (!string.IsNullOrEmpty(strSQL))
45 {
46 analyzeSQLString(strSQL);
47 strSQL = "";
48 }
49 }
50 lineNumber = 1;
51 Number = line.Split(‘ ‘).Length > 0 ? line.Split(‘ ‘)[0] : "";
52 strSQL = line.ToString();
53 strSQL = strSQL.Substring(Number.Length);
54 strSQL = strSQL.Replace("query", "");
55 strSQL = strSQL.Trim();
56
57 }
58 else
59 {
60 strSQL += " " + line.ToString();
61 lineNumber++;
62 IsNumberCount = 0;
63 }
64 bgWorker.ReportProgress(i);
65 if (bgWorker.CancellationPending)
66 {
67 e.Cancel = true;
68 return;
69 }
70 Thread.Sleep(SleepMis);
71
72 }
73 if (strSQL != "")
74 {
75 analyzeSQLString(strSQL);
76 }
77 CountLine = i;
78 sr.Close();
79 sr.Dispose();
3、SQL解析-----analyzeSQLString(string strSQL)
#region SQL解析
private void analyzeSQLString(string strSQL)
{
int t = 1;
int idx = 0;
if (strSQL.IndexOf("select") > -1)
{
t = 1;
idx = strSQL.IndexOf("select");
}
else if (strSQL.IndexOf("insert") > -1)
{
t = 2;
idx = strSQL.IndexOf("insert");
}
else if (strSQL.IndexOf("update") > -1)
{
t = 3;
idx = strSQL.IndexOf("update");
}
else if (strSQL.IndexOf("delete") > -1)
{
t = 4;
idx = strSQL.IndexOf("delete");
}
else
{
t = 5;
}
if (idx != 0)
{
strSQL = strSQL.Substring(idx);
}
switch (t)
{
case 1://查询
{
//过滤参数
if (ckIgnore.Checked)
{
//过滤limit
if (strSQL.LastIndexOf("limit") > 0)
{
strSQL = strSQL.Substring(0, strSQL.LastIndexOf("limit"));
strSQL = strSQL.Trim();
}
//过滤order by
if (strSQL.Contains("order by"))
{
strSQL = strSQL.Substring(0, strSQL.LastIndexOf("order by"));
strSQL = strSQL.Trim();
}
strSQL = RegexString(strSQL);
}
if (Seldic.ContainsKey(strSQL))
{
Seldic[strSQL] = Convert.ToInt32(Seldic[strSQL]) + 1;
}
else
{
Seldic.Add(strSQL, 1);
}
}; break;
case 2://插入
{
//过滤参数
if (ckIgnore.Checked)
{
if (strSQL.LastIndexOf("values") > 0)
{
strSQL = strSQL.Substring(0, strSQL.LastIndexOf("values"));
}
}
if (Insertdic.ContainsKey(strSQL))
{
Insertdic[strSQL] = Convert.ToInt32(Insertdic[strSQL]) + 1;
}
else
{
Insertdic.Add(strSQL, 1);
}
}; break;
case 3://更新
{
//过滤参数
if (ckIgnore.Checked)
{
matchsMade = patArry.Matches(strSQL);
strSQL = "";
for (int i = 0, len = matchsMade.Count; i < len; i++)
{
if (i < 3)
{
strSQL += matchsMade[i].Value + " ";
}
else
{
if (matchsMade[i].Value != "where")
{
strSQL += matchsMade[i].Value + "=? ";
}
else
{
strSQL += matchsMade[i].Value + " ";
}
}
}
strSQL = strSQL.TrimEnd();
#region 老
//string strTable = string.Empty, strWhere = string.Empty;
//int idxwhere = strSQL.LastIndexOf("where");
//if (idxwhere > 0)
//{
// strWhere = RegexString(strSQL.Substring(idxwhere));
// strSQL = strSQL.Substring(0, idxwhere);
// strSQL = strSQL.Trim();
//}
//strTable = strSQL.Substring(0, strSQL.IndexOf("set"));
//strSQL = strSQL.Substring(strSQL.IndexOf("set") + 3);
//strSQL = RegRepac(strSQL).Trim();
////解析
//foreach (string s in strSQL.Split(‘B‘))
//{
// if (s.Split(‘A‘).Length > 1)
// {
// strSQL = strSQL.Replace(s, s.Split(‘A‘)[0] + "[email protected]");
// }
//}
//strSQL = strSQL.Replace("B", ",");
//strSQL = strTable + "set " + strSQL + (string.IsNullOrEmpty(strWhere) ? "" : " " + strWhere);
#endregion 老
}
if (Updic.ContainsKey(strSQL))
{
Updic[strSQL] = Convert.ToInt32(Updic[strSQL]) + 1;
}
else
{
Updic.Add(strSQL, 1);
}
}; break;
case 4://删除
{
//过滤参数
if (ckIgnore.Checked)
{
strSQL = RegexString(strSQL);
}
if (Deletedic.ContainsKey(strSQL))
{
Deletedic[strSQL] = Convert.ToInt32(Deletedic[strSQL]) + 1;
}
else
{
Deletedic.Add(strSQL, 1);
}
}; break;
default://其它
{
if (Othdic.ContainsKey(strSQL))
{
Othdic[strSQL] = Convert.ToInt32(Othdic[strSQL]) + 1;
}
else
{
Othdic.Add(strSQL, 1);
}
}; break;
}
}
#endregion
4、写入文件：
1 private void Write()
2 {
3 FileStream fs = new FileStream(System.Environment.CurrentDirectory + "\\sql.txt", FileMode.Create);
4 StreamWriter sw = new StreamWriter(fs, Encoding.UTF8);
5 //开始写入
6 Dictionary<string, int> sortdic = new Dictionary<string, int>();
7 sw.Write("查询操作\r\n");
8 sw.Write("-----------------------------------------------------------\r\n");
9 WriteLog(Seldic, sortdic, sw, 0);
10
11 sw.Write("插入操作\r\n");
12 sw.Write("-----------------------------------------------------------\r\n");
13 WriteLog(Insertdic, sortdic, sw, 1);
14
15 sw.Write("更新操作\r\n");
16 sw.Write("-----------------------------------------------------------\r\n");
17 WriteLog(Updic, sortdic, sw, 2);
18
19
20 sw.Write("删除操作\r\n");
21 sw.Write("-----------------------------------------------------------\r\n");
22 WriteLog(Deletedic, sortdic, sw, 3);
23
24 sw.Write("其它操作\r\n");
25 sw.Write("-----------------------------------------------------------\r\n");
26 WriteLog(Othdic, sortdic, sw, 4);
27
28 //清空缓冲区
29 sw.Flush();
30 //关闭流
31 sw.Close();
32 fs.Close();
33 }
34
35
36 private void WriteLog(Dictionary<String, int> data, Dictionary<String, int> sort, StreamWriter sw, int i)
37 {
38 if (data.Count > 0)
39 {
40 sort = data.OrderByDescending(o => o.Value).ToDictionary(o => o.Key, p => p.Value);
41 foreach (KeyValuePair<string, int> de in sort)
42 {
43 sw.Write(de.Value.ToString().PadRight(7, ‘ ‘) + " " + de.Key + "\r\n");
44 }
45 bgWaritWorke.ReportProgress(i);
46 Thread.Sleep(SleepMis);
47 }
48 }
5、异步读写
6、最关键的正则解析
#region 正则过滤参数
/// <summary>
/// 正则过滤参数
/// </summary>
/// <param name="s">带解析字符串</param>
/// <returns></returns>
public string RegexString(string strSQL)
{
// "<", ">", "<=", ">=", "!=", "<>", "="
//(\b|\b\s+|\‘)(,)
//(\b|\b\s+)(=)
//(\=\w+\,)|(\=\s+\w+\,)|(\=\‘\w+\‘\,)|(\=\s\‘\w+\‘\,)|(\‘\,)
strSQL = patTime.Replace(strSQL, "@p"); //时间
strSQL = patSql.Replace(strSQL, "@p"); //参数
#region 老
//int r = 0, len = 0;
//temp = key = "";
////带SQL运算符
//matchsMade = patSql.Matches(strSQL);
//for (r = 0, len = matchsMade.Count; r < len; r++)
//{
// //取出前缀关键字
// temp = matchsMade[r].Value;
// key = "";
// if (patLike.IsMatch(temp))
// {
// key = patLike.Matches(temp)[0].Value;
// }
// else
// {
// foreach (string t in keys)
// {
// if (temp.Replace(t, "").Length < temp.Length)
// {
// temp = temp.Replace(t, "");
// key += t;
// }
// }
// }
// //重新赋值
// temp = matchsMade[r].Value;
// if (temp.IndexOf(key) > 0)
// {
// temp = temp.Substring(0, temp.IndexOf(key));
// }
// strSQL = strSQL.Replace(matchsMade[r].Value, temp + " " + key + "@p");
// key = "";
//}
#endregion
return strSQL;
}
/// <summary>
/// 替换
/// </summary>
/// <param name="strSQL"></param>
/// <param name="newValue"></param>
/// <returns></returns>
public string RegRepac(string strSQL)
{
//(\b|\b\s+|\‘)(,)
//(\b|\b\s+)(=)
Regex pat = new Regex(@"(\b|\b\s+|`)(=)");
strSQL = pat.Replace(strSQL, "A");
pat = new Regex(@"(\b|\b\s+|\‘)(,)");
strSQL = pat.Replace(strSQL, "B");
return strSQL;
}
#endregion
完整的实例：
=================================================================================================================================================
所遇问题：
1、解析速度偏慢（应该是正则没有写好）。
2、关于更新语句的处理效果太烂了。
3、很多SQL的运算付没有保留下来。
4、如果需要保留关键的运算付，就使用标记了 "老" 字的代码
列举一些列子：
select state as `status`, round(sum(duration),7) as `duration`, concat(round(sum(duration)/0.002432*100,3), ‘%‘) as `percentage` from information_schema.profiling
最后感言：
1、还有很多情况没有考虑到
2、不知道有没有谁可以推荐C#版的SQL语句解析的工具或者DLL
3、历时3天
这几天心情不好，唉。想回家休息个把月，哈哈
下载连接
http://pan.baidu.com/s/1mglImpU
时间： 2024-12-30 22:33:16