ASP.NET抓取网页内容的实现方法

这篇文章主要介绍了ASP.NET抓取网页内容的实现方法,涉及使用HttpWebRequest及WebResponse抓取网页内容的技巧,需要的朋友可以参考下

 一、ASP.NET 使用HttpWebRequest抓取网页内容

 /// <summary>方法一：比较推荐
 /// 用HttpWebRequest取得网页源码
 /// 对于带BOM的网页很有效，不管是什么编码都能正确识别
 /// </summary>
 /// <param name="url">网页地址" </param>
 /// <returns>返回网页源文件</returns>
 public static string GetHtmlSource2(string url)
 {
     //处理内容
     string html = "";
     HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
     request.Accept = "*/*"; //接受任意文件
     request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)"; //
     request.AllowAutoRedirect = true;//是否允许302
     //request.CookieContainer = new CookieContainer();//cookie容器，
     request.Referer = url; //当前页面的引用
     HttpWebResponse response = (HttpWebResponse)request.GetResponse();
     Stream stream = response.GetResponseStream();
     StreamReader reader = new StreamReader(stream, Encoding.Default);  //这里如果出现乱码可以使用 Encoding.GetEncoding("utf-8")处理

html = reader.ReadToEnd(); stream.Close(); return html; }

二、ASP.NET 使用 WebResponse 抓取网页内容

public static string GetHttpData2(string Url)
 {
     string sException = null;
     string sRslt = null;
     WebResponse oWebRps = null;
     WebRequest oWebRqst = WebRequest.Create(Url);
     oWebRqst.Timeout = 50000;
     try
     {
         oWebRps = oWebRqst.GetResponse();
     }
     catch (WebException e)
     {
         sException = e.Message.ToString();
     }
     catch (Exception e)
     {
         sException = e.ToString();
     }
     finally
     {
         if (oWebRps != null)
         {
             StreamReader oStreamRd = new StreamReader(oWebRps.GetResponseStream(), Encoding.GetEncoding("utf-8"));
             sRslt = oStreamRd.ReadToEnd();
             oStreamRd.Close();
             oWebRps.Close();
         }
     }
     return sRslt;
 }

原文：http://www.jb51.net/article/60740.htm