SEOKeywordSearch/HttpHelper.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Authentication;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

namespace CityCraft
{
    public enum HttpMethod
    {
        GET,
        POST
    }
    public class HttpArgs
    {
        public string Url { get; set; }
        public string Host { get; set; }
        public int Port { get; set; }
        public string Accept { get; set; }
        public string Referer { get; set; }
        public string Cookie { get; set; }
        public string Data { get; set; }
        public string UA { get; set; }
        public HttpMethod Method { get; set; }
    }
    public enum HttpReadyState
    {
        未初始化,
        载入,
        载入完成,
        交互,
        完成
    }
    public class HttpHelper
    {
        public HttpReadyState readyState = HttpReadyState.未初始化;
        public int Status = 0;
        public string responseBody = "";
        public string responseText = "";
        public byte[] responseByte = null;
        public HttpArgs args = new HttpArgs();
        public string ErrMsg = string.Empty;
        /// <summary>
        /// 提交方法
        /// </summary>
        #region HttpWebRequest & HttpWebResponse

        /// <summary>
        /// Get方法
        /// </summary>
        /// <param name="geturl">请求地址</param>
        /// <param name="cookieser">Cookies存储器</param>
        /// <returns>请求返回的Stream</returns>
        public void Send(HttpMethod method, string url)
        {
            readyState = HttpReadyState.载入;
            ParseURL(url);
            args.Method = method;
            new Thread(new ThreadStart(ReciveData)).Start();
        }

        public void ReciveData()
        {
            responseBody = InternalSocketHttp();
            readyState = HttpReadyState.完成;
        }
        /// <summary>
        /// Post方法
        /// </summary>
        /// <param name="posturl">请求地址</param>
        /// <param name="bytes">Post数据</param>
        /// <param name="cookieser">Cllkies存储器</param>
        /// <returns>请求返回的流</returns>
        public string Post(string url,
            byte[] bytes,
            CookieContainer cookies,
            Encoding encoding)
        {
            return null;
        }

        /// <summary>
        /// 根据Url得到host
        /// </summary>
        /// <param name="strUrl">url字符串</param>
        /// <returns>host字符串</returns>
        private void ParseURL(string strUrl)
        {
            if (args == null)
                args = new HttpArgs();

            args.Host = "";
            args.Port = 80;
            args.Referer = "";
            args.Cookie = "";
            args.Url = "";
            args.Accept = "text/html";//,application/xhtml+xml,application/xml,application/json;";
            args.UA = "Mozilla/5.0+(Compatible;+Baiduspider/2.0;++http://www.baidu.com/search/spider.html)";

            //http://www.alibaba.com/products/Egg_Laying_Block_Machine/1.html
            int iIndex = strUrl.IndexOf(@"//");
            if (iIndex <= 0)
                args = null;
            //www.alibaba.com:80/products/Egg_Laying_Block_Machine/1.html
            string nohttpurl = strUrl.Substring(iIndex + 2);
            string address = nohttpurl;
            iIndex = nohttpurl.IndexOf(@"/");
            if (iIndex > 0)
            {
                //www.alibaba.com:80
                address = nohttpurl.Substring(0, iIndex);
                args.Url = nohttpurl.Substring(iIndex);
            }
            iIndex = nohttpurl.IndexOf(@":");
            if (iIndex > 0)
            {
                string[] tempargs = address.Trim().Split(char.Parse(":"));
                args.Host = tempargs[0];
                args.Port = int.Parse(tempargs[1]);
            }
            else
            {
                //www.alibaba.com:80
                args.Host = address;
                args.Port = 80;
            }
        }
        #endregion

        #region Socket

        string InternalSocketHttp()
        {
            using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp))
            {
                try
                {
                    socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, 1000);
                    socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, 5000);
                    socket.Connect(args.Host, args.Port);
                    if (socket.Connected)
                    {
                        byte[] buff = ParseHttpArgs();
                        if (socket.Send(buff) > 0)
                        {
                            List<byte> responseBytes = new List<byte>();
                            byte[] buffer = new byte[1024];
                            int iNumber = socket.Receive(buffer, buffer.Length, SocketFlags.None);
                            while (iNumber > 0)//使用了Connection: Close 所以判断长度为0 时停止接受
                            {
                                responseBytes.AddRange(new List<byte>(buffer));//添加数据到List
                                iNumber = socket.Receive(buffer, buffer.Length, SocketFlags.None);//继续接收数据
                            }
                            responseByte = responseBytes.ToArray();
                            readyState = HttpReadyState.载入完成;
                            return ParseResponse(responseByte);
                        }
                    }
                }
                catch (Exception e)
                {
                    ErrMsg = e.Message;
                }
                return string.Empty;
            }
        }

        private string ParseResponse(byte[] responseBytes)
        {
            string responseStr = Encoding.UTF8.GetString(responseBytes);
            int splitindex = responseStr.IndexOf("\r\n\r\n");
            if (splitindex > 0)
            {
                string responseHeader = responseStr.Substring(0, splitindex);
                string responseBody = responseStr.Substring(splitindex + 4);
                if (responseHeader.StartsWith("HTTP/1.1 400"))
                {
                    Status = 400;
                    return string.Empty;
                }
                else if (responseHeader.StartsWith("HTTP/1.1 404"))
                {
                    Status = 404;
                    return string.Empty;
                }
                else if (responseHeader.StartsWith("HTTP/1.1 302") || responseHeader.StartsWith("HTTP/1.1 301"))
                {
                    Status = 302;
                    int start = responseHeader.ToUpper().IndexOf("LOCATION");
                    if (start > 0)
                    {
                        string temp = responseHeader.Substring(start, responseHeader.Length - start);
                        string[] sArry = Regex.Split(temp, "\r\n");
                        args.Url = sArry[0].Remove(0, 10);
                        if (args.Url == "")
                            return string.Empty;
                        return InternalSocketHttp();  //注意：302协议需要重定向
                    }
                }
                else if (responseHeader.StartsWith("HTTP/1.1 200"))  //读取内容
                {
                    Status = 200;
                    //解压
                    DecompressWebPage(ref responseBytes, responseHeader);
                    //转码
                    responseBody = DecodeWebStringByHttpHeader(responseBytes, responseHeader);
                    responseBody = DecodeWebStringByHtmlPageInfo(responseBytes, responseBody);
                }
                splitindex = responseBody.IndexOf("\r\n\r\n");
                if (splitindex > 0)
                    responseBody = responseBody.Substring(splitindex + 4);
                else
                    responseBody = string.Empty;
                return responseBody;
            }
            return string.Empty;
        }
        #endregion

        #region  Helper

        /// <summary>
        /// 解压网页
        /// </summary>
        /// <param name="responseBytes">网页字节数组含http头</param>
        /// <param name="iTotalCount">数组长度</param>
        /// <param name="strHeader">Http头字符串</param>
        /// <param name="iStart">网页正文开始位置</param>
        private void DecompressWebPage(ref byte[] responseBytes, string strHeader)
        {
            Regex regZip = new Regex(@"Content-Encoding:\s+gzip[^\n]*\r\n", RegexOptions.IgnoreCase);

            if (regZip.IsMatch(strHeader))
            {
                responseBytes = Decompress(responseBytes);
            }
        }

        /// <summary>
        /// 解压gzip网页
        /// </summary>
        /// <param name="szSource">压缩过的字符串字节数组</param>
        /// <returns>解压后的字节数组</returns>
        private byte[] Decompress(byte[] szSource)
        {
            MemoryStream msSource = new MemoryStream(szSource);
            //DeflateStream  也可以这儿
            GZipStream stream = new GZipStream(msSource, CompressionMode.Decompress);
            byte[] szTotal = new byte[40 * 1024];
            long lTotal = 0;
            byte[] buffer = new byte[8];
            int iCount = 0;
            do
            {
                iCount = stream.Read(buffer, 0, 8);
                if (szTotal.Length <= lTotal + iCount) //放大数组
                {
                    byte[] temp = new byte[szTotal.Length * 10];
                    szTotal.CopyTo(temp, 0);
                    szTotal = temp;
                }
                buffer.CopyTo(szTotal, lTotal);
                lTotal += iCount;
            } while (iCount != 0);
            byte[] szDest = new byte[lTotal];
            Array.Copy(szTotal, 0, szDest, 0, lTotal);
            return szDest;
        }

        /// <summary>
        /// 根据Http头标记里面的字符编码解析字符串
        /// </summary>
        /// <param name="responseBytes">网页内容字节数组(除http头以外的内容)</param>
        /// <param name="iTotalCount">网页内容字节数组长度</param>
        /// <param name="strHeader">http头的字符串</param>
        /// <returns>转好的字符串</returns>
        private string DecodeWebStringByHttpHeader(byte[] responseBytes, string strHeader)
        {
            string strResponse = "";
            if (strHeader.Contains("charset=GBK") || strHeader.Contains("charset=gb2312"))
            {
                strResponse = Encoding.GetEncoding("GBK").GetString(responseBytes);
            }
            else
                strResponse = Encoding.UTF8.GetString(responseBytes);
            return strResponse;
        }

        /// <summary>
        /// 根据网页meta标记里面的字符编码解析字符串
        /// </summary>
        /// <param name="responseBytes">网页内容字节数组(除http头以外的内容)</param>
        /// <param name="iTotalCount">网页内容字节数组长度</param>
        /// <param name="strResponse">网页内容字符串, 可能已经根据其它转码要求转换过的字符串</param>
        /// <returns>转好的字符串</returns>
        private string DecodeWebStringByHtmlPageInfo(byte[] responseBytes, string strResponse)
        {
            Regex regGB2312 = new Regex(@"<meta[^>]+Content-Type[^>]+gb2312[^>]*>", RegexOptions.IgnoreCase);
            Regex regGBK = new Regex(@"<meta[^>]+Content-Type[^>]+gbk[^>]*>", RegexOptions.IgnoreCase);
            Regex regBig5 = new Regex(@"<meta[^>]+Content-Type[^>]+Big5[^>]*>", RegexOptions.IgnoreCase);
            if (regGB2312.IsMatch(strResponse) || regGBK.IsMatch(strResponse))
                strResponse = Encoding.GetEncoding("GBK").GetString(responseBytes);
            if (regBig5.IsMatch(strResponse))
                strResponse = Encoding.GetEncoding("Big5").GetString(responseBytes);
            return strResponse;
        }

        private byte[] ParseHttpArgs()
        {
            StringBuilder bulider = new StringBuilder();
            if (args.Method == HttpMethod.POST)
            {
                bulider.AppendLine(string.Format("POST {0} HTTP/1.1", args.Url));
                bulider.AppendLine("Content-Type: application/x-www-form-urlencoded");
            }
            else
            {
                bulider.AppendLine(string.Format("GET {0} HTTP/1.1", args.Url));
            }

            bulider.AppendLine(string.Format("Host: {0}:{1}", args.Host, args.Port));

            bulider.AppendLine("User-Agent: " + args.UA);
            //"User-Agent: Mozilla/5.0+(Compatible;+Baiduspider/2.0;++http://www.baidu.com/search/spider.html)";Mozilla/5.0 (Windows NT 6.1; IE 9.0)
            if (!string.IsNullOrEmpty(args.Referer))
                bulider.AppendLine(string.Format("Referer: {0}", args.Referer));

            //bulider.AppendLine("Connection: close");

            bulider.AppendLine("Connection: Close");

            if (!string.IsNullOrEmpty(args.Accept))
                bulider.AppendLine(string.Format("Accept: {0}", args.Accept));

            if (!string.IsNullOrEmpty(args.Cookie))
                bulider.AppendLine(string.Format("Cookie: {0}", args.Cookie));

            if (args.Method == HttpMethod.POST)
            {
                bulider.AppendLine(string.Format("Content-Length: {0}\r\n", Encoding.Default.GetBytes(args.Data).Length));
                bulider.Append(args.Data);
            }
            else
            {
                bulider.Append("\r\n");
            }

            string header = bulider.ToString();
            return Encoding.Default.GetBytes(header);
        }
        #endregion

    }

    public class MilliTimer
    {
        private static double times { get; set; }
        public static void start()
        {
            times = getTotalMilliseconds();
        }

        public static double getTimes()
        {
            return getTotalMilliseconds() - times;
        }


        public static double getTotalMilliseconds()
        {
            return DateTime.Now.Subtract(DateTime.Parse("1970-1-1")).TotalMilliseconds;
        }
    }
}