由于之前一直遇到一些关于并行进行数据处理的时效果往往不好，不管是c#还是java程序都是一样，但是在Python中通过mutilprocess实现同样的功能时，却发现确实可以提高程序运行的性能，及服务器资源使用提高。python具体性能及mutilprocess用法，请参考：《Python：使用pymssql批量插入csv文件到数据库测试》
如有转载请标明原文地址：https://i.cnblogs.com/EditPosts.aspx?postid=7228337
很久之前就设想如何在c#中实现多进程的方案，如今终于设计出了C#中动态实现多进程的方案：
具体代码：
1 using Microsoft.CSharp;
2 using System;
3 using System.CodeDom;
4 using System.CodeDom.Compiler;
5 using System.Collections.Generic;
6 using System.Linq;
7 using System.Reflection;
8 using System.Text;
9
10 namespace MutilProcessDemo
11 {
12 public class Program
13 {
14 static System.Timers.Timer timer = null;
15
16 public static void Main(string[] args)
17 {
18 GenerateExe();
19
20 timer = new System.Timers.Timer();
21 timer.AutoReset = true;
22 timer.Enabled = true;
23 timer.Interval = 5000;
24 timer.Elapsed += Timer_Elapsed;
25
26
27 Console.WriteLine("Press Enter key to exit...");
28 Console.ReadKey();
29 }
30
31 private static void DynamicCompiler()
32 {
33 // 1.CSharpCodePrivoder
34 CSharpCodeProvider objCSharpCodePrivoder = new CSharpCodeProvider();
35
36 // 2.ICodeComplier
37 ICodeCompiler objICodeCompiler = objCSharpCodePrivoder.CreateCompiler();
38
39 // 3.CompilerParameters
40 CompilerParameters objCompilerParameters = new CompilerParameters();
41 objCompilerParameters.ReferencedAssemblies.Add("System.dll");
42 objCompilerParameters.GenerateExecutable = false;
43 objCompilerParameters.GenerateInMemory = true;
44
45 // 4.CompilerResults
46 CompilerResults cr = objICodeCompiler.CompileAssemblyFromSource(objCompilerParameters, GenerateCode());
47
48 if (cr.Errors.HasErrors)
49 {
50 Console.WriteLine("编译错误：");
51 foreach (CompilerError err in cr.Errors)
52 {
53 Console.WriteLine(err.ErrorText);
54 }
55 }
56 else
57 {
58 // 通过反射，调用HelloWorld的实例
59 Assembly objAssembly = cr.CompiledAssembly;
60 object objHelloWorld = objAssembly.CreateInstance("DynamicCodeGenerate.HelloWorld");
61 MethodInfo objMI = objHelloWorld.GetType().GetMethod("OutPut");
62
63 Console.WriteLine(objMI.Invoke(objHelloWorld, null));
64 }
65
66 Console.ReadLine();
67 }
68
69 static string GenerateCode()
70 {
71 StringBuilder sb = new StringBuilder();
72 sb.Append("using System;");
73 sb.Append(Environment.NewLine);
74 sb.Append("namespace DynamicCodeGenerate");
75 sb.Append(Environment.NewLine);
76 sb.Append("{");
77 sb.Append(Environment.NewLine);
78 sb.Append(" public class HelloWorld");
79 sb.Append(Environment.NewLine);
80 sb.Append(" {");
81 sb.Append(Environment.NewLine);
82 sb.Append(" public string OutPut()");
83 sb.Append(Environment.NewLine);
84 sb.Append(" {");
85 sb.Append(Environment.NewLine);
86 sb.Append(" return \"Hello world!\";");
87 sb.Append(Environment.NewLine);
88 sb.Append(" }");
89 sb.Append(Environment.NewLine);
90 sb.Append(" }");
91 sb.Append(Environment.NewLine);
92 sb.Append("}");
93
94 string code = sb.ToString();
95 Console.WriteLine(code);
96 Console.WriteLine();
97
98 return code;
99 }
100
101
102 private static void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
103 {
104 System.Diagnostics.Process[] processes = System.Diagnostics.Process.GetProcessesByName("HelloWorld.exe");
105
106 int maxProcessCount = 8;
107 // 如果超过了最大允许任务数据不再触发新的进程任务
108 if (processes.Length >= maxProcessCount)
109 return;
110
111 List<int> tasks = new List<int>();
112 Random random = new Random(1000);
113 for (int i = 0; i < maxProcessCount - processes.Length; i++)
114 {
115 tasks.Add(random.Next(1000));
116 }
117
118
119 System.Threading.Tasks.Task[] taskItems = new System.Threading.Tasks.Task[tasks.Count];
120
121 for (int i = 0; i < tasks.Count; i++)
122 {
123 //TaskBase taskSubmit = new ParseMrToHdfsFileItemTask();
124 //taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew(taskSubmit.Submit, new ParseMrToHdfsFileItemTaskArgument() { Task = tasks[i], ComputeNode = this.ComputeNode }, TaskCreationOptions.PreferFairness);
125 taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew((object state) =>
126 {
127 System.Diagnostics.Process process = new System.Diagnostics.Process();
128 //System.Diagnostics.Process process = System.Diagnostics.Process.Start("HelloWorld.exe", state.ToString());
129 process.StartInfo = new System.Diagnostics.ProcessStartInfo();
130 process.StartInfo.CreateNoWindow = true;
131 process.StartInfo.UseShellExecute = false;
132 process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
133 process.StartInfo.Arguments = state.ToString();
134 process.StartInfo.FileName = "HelloWorld.exe";
135 process.Start();
136 process.WaitForExit();
137 process.Close();
138 process.Dispose();
139 }, tasks[i]);
140 }
141
142 System.Threading.Tasks.Task.WaitAll(taskItems);
143 }
144
145 private static void GenerateExe()
146 {
147 // 创建编译器对象
148 CSharpCodeProvider p = new CSharpCodeProvider();
149 ICodeCompiler cc = p.CreateCompiler();
150
151 // 设置编译参数
152 CompilerParameters options = new CompilerParameters();
153 options.ReferencedAssemblies.Add("System.dll");
154 options.ReferencedAssemblies.Add("MutilProcessDemo.exe");
155 options.GenerateExecutable = true;
156 options.OutputAssembly = "HelloWorld.exe";
157
158 //options.ReferencedAssemblies.Add("System.Windows.Forms.dll");
159 //options.EmbeddedResources.Add("Data.xml"); // 添加内置资源
160 //options.CompilerOptions += " /target:winexe";
161 //options.CompilerOptions += " /res:Resource1.res";
162 //options.CompilerOptions += " /win32icon:test.ico";
163
164 // 创建源码
165
166 // 1. 使用CodeDom创建源码
167 //CodeCompileUnit cu = new CodeCompileUnit();
168 //CodeNamespace Samples = new CodeNamespace("Samples");
169 //cu.Namespaces.Add(Samples);
170 //Samples.Imports.Add(new CodeNamespaceImport("System"));
171 //CodeTypeDeclaration Class1 = new CodeTypeDeclaration("Class1");
172 //Samples.Types.Add(Class1);
173 //CodeEntryPointMethod Start = new CodeEntryPointMethod();
174 //CodeMethodInvokeExpression cs1 = new CodeMethodInvokeExpression(
175 // new CodeTypeReferenceExpression("System.Console"), "WriteLine",
176 // new CodePrimitiveExpression("Hello World!")
177 // );
178 //Start.Statements.Add(new CodeExpressionStatement(cs1));
179 //Class1.Members.Add(Start);
180
181 // 2. 直接指定源码字符串
182 string code = @"
183 using System;
184 using MutilProcessDemo;
185
186 namespace Samples
187 {
188 public class Class1
189 {
190 static void Main(string[] args)
191 {
192 Console.WriteLine(""Hello, World!"");
193 MutilProcessDemo.Program.DoMethod(args);
194 Console.WriteLine(DateTime.Now.ToString());
195 }
196 }
197 }";
198 CodeSnippetCompileUnit codeSnippetCompileUnit = new CodeSnippetCompileUnit(code);
199
200 // 开始编译
201 CompilerResults compilerResults = cc.CompileAssemblyFromDom(options, codeSnippetCompileUnit);
202
203 // 显示编译信息
204 if (compilerResults.Errors.Count == 0)
205 Console.WriteLine("{0}compiled ok!", compilerResults.CompiledAssembly.Location);
206 else
207 {
208 Console.WriteLine("Complie Error:");
209 foreach (CompilerError error in compilerResults.Errors)
210 Console.WriteLine(" {0}", error);
211 }
212 }
213
214 public static void DoMethod(string[] args)
215 {
216 System.Console.WriteLine("begin ..." + args[0]);
217
218 for (int i = 0; i < int.Parse(args[0]); i++)
219 {
220 System.Threading.Thread.Sleep(20);
221 }
222
223 System.Console.WriteLine("end..." + args[0]);
224 }
225 }
226 }
上边的程序运行之后会在\MutilProcessDemo\bin\Debug\下出现两个可执行的.exe：HelloWorld.exe、MutilProcessDemo.exe。
参考文章：
c#动态编译，程序集的动态创建（http://www.360doc.com/content/14/1014/11/5054188_416763069.shtml）
时间： 2024-10-07 12:45:26