- 引言
在C#中，我们通常会使用HttpWebRequest来访问url资源，例子如下：
public static string GetContentFromUrl(string url)
{
HttpWebResponse response = null;
WebRequest request;
try
{
request = WebRequest.Create(url);
// make sure it accept gzip
request.Headers.Set("Accept-Encoding", "gzip, deflate");
request.Timeout = 100000;
response = (HttpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
string sXML = null;
Stream unzipStream = null;
StreamReader outSr = null;
switch(response.ContentEncoding)
{
case "gzip":
// get the gzip stream and unzipped
unzipStream = new GZipStream(responseStream, CompressionMode.Decompress);
outSr = new StreamReader(unzipStream);
break;
case "deflate":
unzipStream = new DeflateStream(responseStream, CompressionMode.Decompress);
outSr = new StreamReader(unzipStream);
break;
default:
outSr = new StreamReader(responseStream);
break;
}
sXML = outSr.ReadToEnd();
response.Close();
}
catch (WebException ex)
{
Console.WriteLine(ex.Message);
}
return null;
}
当遇到HttpStatusCode>400时，通常会被认为是WebException，错误消息会带上StatusCode如
The remote server returned an error: (401) Unauthorized.
有时候，当401发生时，服务器会在Response里告诉你具体的错误信息，如
<XOIException><StatusCode>40100</StatusCode><StatusMessage>Unknown login error</StatusMessage></XOIException>
因为HttpWebRequest类会把HttpStatusCode>=400当做WebException，这样的话我们就无法获取到Response Stream中的内容啦。
- 怎样才能获取具体的错误信息，然后根据错误信息做具体的处理呢？
我们可以通过Socket来访问url资源，下面是具体实现的类
RequestHeader
namespace Com.Morningstar.EquityData.XOIAccessor.Http
{
public static class RequestHeader
{
public const string Host = "Host";
public const string AcceptEncoding = "Accept-Encoding";
public const string AcceptLanguage = "Accept-Language";
public const string Accept = "Accept";
public const string Connection = "Connection";
public const string Cookie = "Cookie";
public const string UserAgent = "User-Agent";
public const string ContentType = "Content-Type";
public const string ContentLength = "Content-Length";
}
public static class ResponseHeader
{
public static string ContentLength = "Content-Length";
public static string ContentType = "Content-Type";
public static string ContentEncoding = "Content-Encoding";
public static string SetCookie = "Set-Cookie";
}
public static class Connection
{
public static string KeepAlive = "Keep-Alive";
public static string Close = "Close";
}
}
HttpMethod
namespace Com.Morningstar.EquityData.XOIAccessor.Http
{
public enum HttpMethod
{
GET,POST
}
}
HttpException
namespace Com.Morningstar.EquityData.XOIAccessor.Http
{
public class HttpException:Exception
{
public HttpException(string message) : base(message) { }
public HttpException(string message, Exception innerException) : base(message, innerException) { }
}
}
HttpRequest:建立Http请求，并且返回HttpResponse
namespace Com.Morningstar.EquityData.XOIAccessor.Http
{
public class HttpRequest
{
internal HttpRequest()
{
}
public string Host { set; get; }
private int port = 80;
public int Port
{
set
{
if (port > 0)
{
port = value;
}
}
get
{
return port;
}
}
private HttpMethod method = HttpMethod.GET;
public HttpMethod Method
{
set
{
method = value;
}
get
{
return method;
}
}
private string path = "/";
public string Path
{
set
{
path = value;
}
get
{
return path;
}
}
private NameValueCollection headers = new NameValueCollection();
public NameValueCollection Headers
{
set
{
headers = value;
}
get
{
return headers;
}
}
public void AddHeader(string name, string value)
{
headers[name] = value;
}
public string Body { set; get; }
/// <summary>
/// Millseconds to wait response
/// </summary>
private int timeout = -1;//Never time out
public int Timeout
{
set
{
if (timeout < -1)
{
throw new ArgumentOutOfRangeException("Timeout is less than -1");
}
timeout = value;
}
get
{
return timeout;
}
}
private void CheckReqiredParameters()
{
if (string.IsNullOrEmpty(Host))
{
throw new ArgumentException("Host is blank");
}
}
public string BuilSocketRequest()
{
StringBuilder requestBuilder = new StringBuilder();
FillHeader();
BuildRequestLine(requestBuilder);
BuildRequestHeader(requestBuilder);
BuildRequestBody(requestBuilder);
return requestBuilder.ToString();
}
private void FillHeader()
{
if (Method.Equals(HttpMethod.POST))
{
if (string.IsNullOrEmpty(Headers[RequestHeader.ContentType]))
{
Headers[RequestHeader.ContentType] = "application/x-www-form-urlencoded";
}
if (!string.IsNullOrEmpty(Body) && string.IsNullOrEmpty(Headers[RequestHeader.ContentLength]))
{
Headers[RequestHeader.ContentLength] = Encoding.Default.GetBytes(Body).Length.ToString();
}
}
if (!string.IsNullOrEmpty(Headers[RequestHeader.Connection]))
{
Headers[RequestHeader.Connection] = Connection.Close;
}
if (string.IsNullOrEmpty(Headers[RequestHeader.Accept]))
{
Headers[RequestHeader.Accept] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
}
if (string.IsNullOrEmpty(Headers[RequestHeader.UserAgent]))
{
Headers[RequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 6.1; IE 9.0)";
}
if (string.IsNullOrEmpty(Headers[RequestHeader.AcceptEncoding]))
{
Headers[RequestHeader.AcceptEncoding] = "gzip, deflate";
}
if (string.IsNullOrEmpty(Headers[RequestHeader.Host]))
{
Headers[RequestHeader.Host] = Host;
}
}
private void BuildRequestLine(StringBuilder requestBuilder)
{
if (Method.Equals(HttpMethod.POST))
{
requestBuilder.AppendLine(string.Format("POST {0} HTTP/1.1", Path));
}
else
{
requestBuilder.AppendLine(string.Format("GET {0} HTTP/1.1", Path));
}
}
private void BuildRequestHeader(StringBuilder requestBuilder)
{
foreach (string name in Headers)
{
requestBuilder.AppendLine(string.Format("{0}: {1}", name, Headers[name]));
}
}
private void BuildRequestBody(StringBuilder requestBuilder)
{
requestBuilder.AppendLine();
if (!string.IsNullOrEmpty(Body))
{
requestBuilder.Append(Body);
}
}
public HttpResponse GetResponse()
{
CheckReqiredParameters();
HttpResponse httpResponse = new HttpResponse();
string socketRequest = BuilSocketRequest();
byte[] requestBytes = Encoding.ASCII.GetBytes(socketRequest);
try
{
using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp))
{
socket.ReceiveTimeout = Timeout;
socket.Connect(Host, Port);
if (socket.Connected)
{
socket.Send(requestBytes);
ParseResponseLine(socket, httpResponse);
ParseResponseHeader(socket, httpResponse);
ParseResponseBody(socket, httpResponse);
socket.Close();
}
}
}
catch (Exception e)
{
throw new HttpException("Get response failure. Host:" + Host + ", Port:" + Port + ",RequestString:" + socketRequest, e);
}
return httpResponse;
}
private void ParseResponseLine(Socket socket, HttpResponse response)
{
string responseLine = ReceiveCharBytes(socket, "\r\n");
responseLine = responseLine.Replace("\r\n", "");
string[] fields = responseLine.Split(‘ ‘);
if (fields.Length >= 3)
{
response.StatusCode = fields[1];
response.StatusDescription = responseLine.Substring(responseLine.IndexOf(fields[1]) + fields[1].Length + 1);
}
else
{
throw new HttpException("The response line:‘" + responseLine + "‘ has the wrong format.");
}
}
private void ParseResponseHeader(Socket socket, HttpResponse response)
{
string responseHeader = ReceiveCharBytes(socket, "\r\n\r\n");
string[] headerArry = Regex.Split(responseHeader, "\r\n");
if (headerArry != null)
{
foreach (string header in headerArry)
{
if (!string.IsNullOrEmpty(header))
{
int start = header.IndexOf(":");
if (start > 0)
{
string name = header.Substring(0, start);
string value = "";
if(header.Length>start+2){
value = header.Substring(start + 2);
}
response.AddHeader(name, value);
}
}
}
}
}
private string ReceiveCharBytes(Socket socket, string breakFlag)
{
StringBuilder builder = new StringBuilder();
while (true)
{
byte[] buff = new byte[1];
int read = socket.Receive(buff, SocketFlags.None);
if (read > 0)
{
builder.Append((char)buff[0]);
}
if (builder.ToString().EndsWith(breakFlag))
{
break;
}
}
return builder.ToString();
}
private void ParseResponseBody(Socket socket, HttpResponse response)
{
string contentLen = response.GetHeader(ResponseHeader.ContentLength);
bool bodyDone = false;
if (!string.IsNullOrEmpty(contentLen))
{
int len = Convert.ToInt32(contentLen);
if (len > 0)
{
byte[] contentBytes = new byte[len];
if (socket.Receive(contentBytes) > 0)
{
response.Body = contentBytes;
}
bodyDone = true;
}
}
if (!bodyDone)
{
List<byte[]> readsList = new List<byte[]>();
int totalLength = 0;
while (true)
{
byte[] buff = new byte[1024];
int readLen = socket.Receive(buff);
if (readLen > 0)
{
totalLength += readLen;
byte[] reads = new byte[readLen];
Array.Copy(buff, 0, reads, 0, readLen);
readsList.Add(reads);
}
else
{
break;
}
}
byte[] fullBytes = new byte[totalLength];
int index = 0;
foreach (byte[] reads in readsList)
{
Array.Copy(reads, 0, fullBytes, index, reads.Length);
index += reads.Length;
}
response.Body = fullBytes;
}
}
private string GetResponseHeader(Socket socket)
{
StringBuilder builder = new StringBuilder();
while (true)
{
byte[] buff = new byte[1];
int read = socket.Receive(buff, SocketFlags.None);
if (read > 0)
{
builder.Append((char)buff[0]);
}
if (builder.ToString().Contains("\r\n\r\n"))
{
break;
}
}
return builder.ToString();
}
public static HttpRequest Create(string url)
{
Uri uri = new Uri(url);
HttpRequest request = new HttpRequest();
request.Host = uri.Host;
request.Port = uri.Port;
request.Path = uri.PathAndQuery;
return request;
}
}
}
HttpResponse：对Http的响应流进行封装
namespace Com.Morningstar.EquityData.XOIAccessor.Http
{
public class HttpResponse
{
internal HttpResponse()
{
}
#region Response Line
public string StatusCode { internal set; get; }
public string StatusDescription{ internal set;get; }
#endregion
#region Response Headers
private NameValueCollection headers = new NameValueCollection();
public NameValueCollection Headers { get { return headers; } }
internal void AddHeader(string name, string value)
{
headers[name] = value;
}
public string GetHeader(string name)
{
return headers[name];
}
public long? ContentLength
{
get
{
if(!string.IsNullOrEmpty(GetHeader(ResponseHeader.ContentLength)))
{
return Convert.ToInt64(GetHeader(ResponseHeader.ContentLength));
}
return null;
}
}
public string ContentEncoding
{
get
{
return GetHeader(ResponseHeader.ContentEncoding);
}
}
#endregion
public byte[] Body { internal set; get; }
public Stream GetBodyStream()
{
if (Body != null)
{
return new MemoryStream(Body);
}
return null;
}
}
}
- 如何使用HttpRequest类
public string GetContent(string url)
{
Login();
HttpRequest request = HttpRequest.Create(url);
request.Method = HttpMethod.GET;
request.AddHeader(RequestHeader.AcceptEncoding, "gzip, deflate");
request.AddHeader(RequestHeader.Cookie, AuthCookie);
request.Timeout = Timeout;
HttpResponse resp = request.GetResponse();
string xoiErrorCode = resp.GetHeader("X-XOI-ErrorCode");
if (!string.IsNullOrEmpty(xoiErrorCode))
{
if (!xoiErrorCode.Equals(XOIErrorCode.XOI_EC_40401))
{
XOIException xoiException = new XOIException("Get content fail. Url:" + url);
string errorContent = ReadContent(resp);
XmlDocument doc = new XmlDocument();
doc.LoadXml(errorContent);
XmlNode statusCodeNode = doc.SelectSingleNode(@"/XOIException/StatusCode");
XmlNode statusMessageNode = doc.SelectSingleNode(@"/XOIException/StatusMessage");
if (statusCodeNode != null)
{
xoiException.XOIErrorCode = statusCodeNode.Value;
}
if (statusMessageNode != null)
{
xoiException.XOIErrorInfo = statusMessageNode.Value;
}
throw xoiException;
}
else
{
return string.Empty;
}
}
return ReadContent(resp);
}
protected string ReadContent(HttpResponse resp)
{
StreamReader reader = null;
try
{
switch (resp.ContentEncoding)
{
case "gzip":
reader = new StreamReader(new GZipStream(resp.GetBodyStream(), CompressionMode.Decompress));
break;
case "deflate":
reader = new StreamReader(new DeflateStream(resp.GetBodyStream(), CompressionMode.Decompress));
break;
default:
reader = new StreamReader(resp.GetBodyStream());
break;
}
return reader.ReadToEnd();
}
finally
{
if (reader != null)
{
reader.Close();
}
}
}
c# 如何通过Socket访问网站资源,布布扣,bubuko.com
时间： 2024-12-23 05:29:39