最近项目需要部署Scrapy爬虫，采用最简单的Scrapyd服务进行部署，基于.net core 进行了客户端的封装。
1）Scrapyd API文档：http://scrapyd.readthedocs.io/en/latest/api.html
2）使用HttpClient进行交互
比较麻烦的是部署爬虫所需要的使用 HttpClient 上传文件的代码，使用了 “MultipartFormDataContent” 对象进行上传，添加字符串内容，及StreamContent，直接可以将MVC中的File对象传入，不用自行处理临时文件。
var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));
content.Add(new StringContent(project), "\"project\"");
content.Add(new StringContent(version), "\"version\"");
content.Add(new StreamContent(eggStream), "\"egg\"", "\"" + project + ".egg" + "\"");
var resp = client.PostAsync("addversion.json", content).Result;
var strResult = resp.Content.ReadAsStringAsync().Result;
return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);
完整代码
1 using System;
2 using System.Collections.Generic;
3 using System.IO;
4 using System.Linq;
5 using System.Net.Http;
6 using System.Text;
7 using Newtonsoft.Json;
8 using Scrapyd.Client.Response;
9
10 namespace Scrapyd.Client
11 {
12 /// <summary>
13 /// A Client used to communicate with the Scrapyd Daemon
14 /// </summary>
15 public class ScrapydClient
16 {
17 private readonly string _host;
18 private readonly int _port;
19
20 public ScrapydClient(string host, int port)
21 {
22 _host = host;
23 _port = port;
24 }
25
26 /// <summary>
27 /// 查询状态
28 /// </summary>
29 /// <returns></returns>
30 public DaemonStatusResponse QueryStatus()
31 {
32 HttpClient client = new HttpClient()
33 {
34 BaseAddress = new Uri($"http://{_host}:{_port}")
35 };
36
37 try
38 {
39 var strResult = client.GetStringAsync("daemonstatus.json").Result;
40
41 var jresult = JsonConvert.DeserializeObject<DaemonStatusResponse>(strResult);
42 return jresult;
43 }
44 catch (Exception e)
45 {
46 return new DaemonStatusResponse { status = "failed", message = e.Message };
47 }
48 }
49
50 public AddVersionResponse AddVersion(string project, string version, Stream eggStream)
51 {
52 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
53 if (eggStream == null) throw new ArgumentNullException(nameof(eggStream));
54
55 var client = new HttpClient()
56 {
57 BaseAddress = new Uri($"http://{_host}:{_port}")
58 };
59
60 try
61 {
62 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));
63
64 content.Add(new StringContent(project), "\"project\"");
65 content.Add(new StringContent(version), "\"version\"");
66 content.Add(new StreamContent(eggStream), "\"egg\"", "\"" + project + ".egg" + "\"");
67
68 var resp = client.PostAsync("addversion.json", content).Result;
69
70 var strResult = resp.Content.ReadAsStringAsync().Result;
71
72 return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);
73 }
74 catch (Exception e)
75 {
76 return new AddVersionResponse { status = "failed", message = e.Message };
77 }
78 }
79
80 public ScheduleResponse ScheduleJob(string project, string spider,
81 string setting = "",
82 string jobid = "",
83 string version = "",
84 Dictionary<string, string> arguments = null)
85 {
86 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
87 if (string.IsNullOrEmpty(spider)) throw new ArgumentNullException(nameof(spider));
88
89
90 var client = new HttpClient()
91 {
92 BaseAddress = new Uri($"http://{_host}:{_port}")
93 };
94
95 try
96 {
97 var data = new Dictionary<string, string>()
98 {
99 {"project", project},
100 {"spider", spider}
101 };
102
103 if (!string.IsNullOrEmpty(jobid))
104 {
105 data.Add("jobid", jobid);
106 }
107 if (!string.IsNullOrEmpty(setting))
108 {
109 data.Add("setting", setting);
110 }
111
112 if (!string.IsNullOrEmpty(version))
113 {
114 data.Add("_version", version);
115 }
116
117 if (arguments != null && arguments.Count > 0)
118 {
119 foreach (var argument in arguments)
120 {
121 data.Add(argument.Key, argument.Value);
122 }
123 }
124
125 var strResult =
126 client.PostAsync("schedule.json", new FormUrlEncodedContent(data))
127 .Result.Content.ReadAsStringAsync()
128 .Result;
129
130 return JsonConvert.DeserializeObject<ScheduleResponse>(strResult);
131
132 }
133 catch (Exception e)
134 {
135 return new ScheduleResponse { status = "failed", message = e.Message };
136 }
137 }
138
139 public CancelResponse CancelJob(string project, string jobid)
140 {
141 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
142 if (string.IsNullOrEmpty(jobid)) throw new ArgumentNullException(nameof(jobid));
143
144
145 var client = new HttpClient()
146 {
147 BaseAddress = new Uri($"http://{_host}:{_port}")
148 };
149
150 try
151 {
152 var content = new FormUrlEncodedContent(new Dictionary<string, string>()
153 {
154 {"project", project},
155 {"job", jobid}
156 });
157 var strResult = client.PostAsync($"/cancel.json", content).Result.Content.ReadAsStringAsync().Result;
158
159 return JsonConvert.DeserializeObject<CancelResponse>(strResult);
160
161 }
162 catch (Exception e)
163 {
164 return new CancelResponse { status = "failed", message = e.Message };
165 }
166 }
167
168 public ListProjectsResponse ListProjects()
169 {
170 try
171 {
172 var client = new HttpClient
173 {
174 BaseAddress = new Uri($"http://{_host}:{_port}")
175 };
176 var result = client.GetStringAsync("listprojects.json").Result;
177 return JsonConvert.DeserializeObject<ListProjectsResponse>(result);
178 }
179 catch (Exception e)
180 {
181 return new ListProjectsResponse { status = "failed", message = e.Message };
182 }
183 }
184
185 public ListVersionsResponse ListVersions(string project)
186 {
187 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
188
189 try
190 {
191 var client = new HttpClient
192 {
193 BaseAddress = new Uri($"http://{_host}:{_port}")
194 };
195
196 var result = client.GetStringAsync($"listversions.json?project={project}").Result;
197 return JsonConvert.DeserializeObject<ListVersionsResponse>(result);
198 }
199 catch (Exception e)
200 {
201 return new ListVersionsResponse { status = "failed", message = e.Message };
202 }
203 }
204
205 public ListSpidersResponse ListSpiders(string project, string version = "")
206 {
207 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
208
209 try
210 {
211 var client = new HttpClient
212 {
213 BaseAddress = new Uri($"http://{_host}:{_port}")
214 };
215
216 var result = client.GetStringAsync($"listspiders.json?project={project}&_version={version}").Result;
217 return JsonConvert.DeserializeObject<ListSpidersResponse>(result);
218 }
219 catch (Exception e)
220 {
221 return new ListSpidersResponse { status = "failed", message = e.Message };
222 }
223 }
224
225 public ListJobsResponse ListJobs(string project)
226 {
227 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
228
229 try
230 {
231 var client = new HttpClient
232 {
233 BaseAddress = new Uri($"http://{_host}:{_port}")
234 };
235
236 var result = client.GetStringAsync($"listjobs.json?project={project}").Result;
237 return JsonConvert.DeserializeObject<ListJobsResponse>(result);
238 }
239 catch (Exception e)
240 {
241 return new ListJobsResponse { status = "failed", message = e.Message };
242 }
243 }
244
245 public ScrapydResponse DeleteVersion(string project, string version)
246 {
247 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
248 if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version));
249
250 var client = new HttpClient()
251 {
252 BaseAddress = new Uri($"http://{_host}:{_port}")
253 };
254
255 try
256 {
257 var data = new Dictionary<string, string>()
258 {
259 {"project", project},
260 {"version", version}
261 };
262
263 var strResult =
264 client.PostAsync("delversion.json", new FormUrlEncodedContent(data))
265 .Result.Content.ReadAsStringAsync()
266 .Result;
267
268 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);
269
270 }
271 catch (Exception e)
272 {
273 return new ScrapydResponse { status = "failed", message = e.Message };
274 }
275 }
276
277 public ScrapydResponse DeleteProject(string project)
278 {
279 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
280
281
282 var client = new HttpClient()
283 {
284 BaseAddress = new Uri($"http://{_host}:{_port}")
285 };
286
287 try
288 {
289 var data = new Dictionary<string, string>()
290 {
291 {"project", project}
292 };
293
294 var strResult =
295 client.PostAsync("delproject.json", new FormUrlEncodedContent(data))
296 .Result.Content.ReadAsStringAsync()
297 .Result;
298
299 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);
300
301 }
302 catch (Exception e)
303 {
304 return new ScrapydResponse { status = "failed", message = e.Message };
305 }
306 }
307
308 }
309
310 }
博客地址 : http://www.cnblogs.com/mobwiz/
时间： 2024-08-12 14:48:14