asp.net 利用HttpWebRequest自动获取网页编码并获取网页源代码

     /// <summary>
    /// 获取源代码
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    public static string GetHtml(string url, Encoding encoding)
    {
        HttpWebRequest request = null;
        HttpWebResponse response = null;
        StreamReader reader = null;
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            request.Timeout = 20000;
            request.AllowAutoRedirect = false;

            response = (HttpWebResponse)request.GetResponse();
            if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
            {
                if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                    reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding);
                else
                    reader = new StreamReader(response.GetResponseStream(), encoding);
                string html = reader.ReadToEnd();

                return html;
            }
        }
        catch
        {
        }
        finally
        {

            if (response != null)
            {
                response.Close();
                response = null;
            }
            if (reader != null)
                reader.Close();

            if (request != null)
                request = null;

        }

        return string.Empty;
    }

    public static string GetEncoding(string url)
    {
        HttpWebRequest request = null;
        HttpWebResponse response = null;
        StreamReader reader = null;
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            request.Timeout = 20000;
            request.AllowAutoRedirect = false;

            response = (HttpWebResponse)request.GetResponse();
            if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
            {
                if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                    reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));
                else
                    reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII);

                string html = reader.ReadToEnd();

                Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)");
                if (reg_charset.IsMatch(html))
                {
                    return reg_charset.Match(html).Groups["charset"].Value;
                }
                else if (response.CharacterSet != string.Empty)
                {
                    return response.CharacterSet;
                }
                else
                    return Encoding.Default.BodyName;
            }
        }
        catch
        {
        }
        finally
        {

            if (response != null)
            {
                response.Close();
                response = null;
            }
            if (reader != null)
                reader.Close();

            if (request != null)
                request = null;

        }
    }

 using System;
 using System.Net;
 using System.Text;
 using System.Text.RegularExpressions; 

 class Program
 {
   // 获取网页的HTML内容，根据网页的charset自动判断Encoding
   static string GetHtml(string url)
   {
     return GetHtml(url, null);
   } 

   // 获取网页的HTML内容，指定Encoding
   static string GetHtml(string url, Encoding encoding)
   {
     byte[] buf = new WebClient().DownloadData(url);
     if (encoding != null) return encoding.GetString(buf);
     string html = Encoding.UTF8.GetString(buf);
     encoding = GetEncoding(html);
     if (encoding == null || encoding == Encoding.UTF8) return html;
     return encoding.GetString(buf);
   } 

   // 根据网页的HTML内容提取网页的Encoding
   static Encoding GetEncoding(string html)
   {
     string pattern = @"(?i)\bcharset=(? <charset>[-a-zA-Z_0-9]+)";
     string charset = Regex.Match(html, pattern).Groups["charset"].Value;
     try { return Encoding.GetEncoding(charset); }
     catch (ArgumentException) { return null; }
   } 

   // 根据网页的HTML内容提取网页的Title
   static string GetTitle(string html)
   {
     string pattern = @"(?si) <title(?:\s+(?:""[^""]*""|‘[^‘]*‘|[^""‘>])*)?>(? <title>.*?) </title>";
     return Regex.Match(html, pattern).Groups["title"].Value.Trim();
   } 

   // 打印网页的Encoding和Title
   static void PrintEncodingAndTitle(string url)
   {
     string html = GetHtml(url);
     Console.WriteLine("[{0}] [{1}]", GetEncoding(html), GetTitle(html));
   } 

   // 程序入口
   static void Main()
   {
     PrintEncodingAndTitle("http://www.msdn.net/");
     PrintEncodingAndTitle("http://www.cnblogs.com/");
     PrintEncodingAndTitle("http://www.cnblogs.com/skyiv/");
     PrintEncodingAndTitle("http://www.csdn.net/");
     PrintEncodingAndTitle("http://news.163.com/");
   }
 }
 /* 程序输出：
 [] [MSDN: Microsoft Developer Network]
 [System.Text.UTF8Encoding] [博客园 - 程序员的网上家园]
 [System.Text.UTF8Encoding] [空间/IV - 博客园]
 [System.Text.UTF8Encoding] [CSDN.NET - 中国最大的IT技术社区，为IT专业技术人员提供最全面的信息传播和服务平台]
 [System.Text.DBCSCodePageEncoding] [新闻中心_网易新闻]
 */