502647092
/
SEOKeywordSearch
mirror of https://e.coding.net/circlecloud/SEOKeywordSearch.git
1
0
Fork 0
Code Releases Activity

修改HttpHelper获取数据到线程 不阻塞UI...

master
j502647092 2015-07-30 20:39:07 +08:00
parent be2b84f56b
commit 5858a6323b
4 changed files with 85 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Frm_Main.Designer.cs generated
View File

@ -1,4 +1,4 @@
namespace EnAliKeywordSearch namespace SEOKeywordSearch
{ {
partial class Frm_Main partial class Frm_Main
{ {

16
Frm_Main.cs
View File

@ -9,8 +9,9 @@ using AliKeywordSearch;
using System.Web; using System.Web;
using System.Reflection; using System.Reflection;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using CityCraft;
namespace EnAliKeywordSearch namespace SEOKeywordSearch
{ {
public partial class Frm_Main : Form public partial class Frm_Main : Form
{ {
@ -104,17 +105,23 @@ namespace EnAliKeywordSearch
string url = String.Format(UrlEnum.GetUrl(s), HttpUtility.UrlEncode(key), (s.ToString() == "搜狗" ? i : i - 1));//s搜狗的页面值不同 string url = String.Format(UrlEnum.GetUrl(s), HttpUtility.UrlEncode(key), (s.ToString() == "搜狗" ? i : i - 1));//s搜狗的页面值不同
this.ResultView.Rows[index].Cells[s.ToString() + "r"].Value = "查询第" + i + "页..."; this.ResultView.Rows[index].Cells[s.ToString() + "r"].Value = "查询第" + i + "页...";
Application.DoEvents(); Application.DoEvents();
string htmldoc = httpHelper.Get(url); httpHelper.Send(HttpMethod.GET, url);
while (HttpHelper.readyState != HttpReadyState.完成)
{
Application.DoEvents();
}
string htmldoc = HttpHelper.responseBody;
if (string.IsNullOrEmpty(htmldoc)) if (string.IsNullOrEmpty(htmldoc))
{ {
state.ForeColor = Color.Red; state.ForeColor = Color.Red;
state.Text = "关键词 " + key + " 在 " + s.ToString() + " 第 " + i + " 页 网页抓取失败 错误:" + HttpHelper.ErrMsg; state.Text = "关键词 " + key + " 在 " + s.ToString() + " 第 " + i + " 页 网页抓取失败 错误:" + HttpHelper.ErrMsg;
this.ResultView.Rows[index].Cells[s.ToString() + "r"].Style.ForeColor = Color.Red;
maybe = true; maybe = true;
continue; continue;
} }
if (htmldoc == "0") if (htmldoc.StartsWith("0"))
{ {
this.ResultView.Rows[index].Cells[s.ToString() + "r"].Value = "已被屏蔽"; pageinfo = "可能被屏蔽";
this.ResultView.Rows[index].Cells[s.ToString() + "r"].Style.ForeColor = Color.Red; this.ResultView.Rows[index].Cells[s.ToString() + "r"].Style.ForeColor = Color.Red;
break; break;
} }
@ -150,7 +157,6 @@ namespace EnAliKeywordSearch
Application.DoEvents(); Application.DoEvents();
} }
this.ResultView.Rows[index].Cells[s.ToString() + "r"].Value = pageinfo + (maybe ? "(可能不准确)" : ""); this.ResultView.Rows[index].Cells[s.ToString() + "r"].Value = pageinfo + (maybe ? "(可能不准确)" : "");
this.ResultView.Rows[index].Cells[s.ToString() + "r"].Style.ForeColor = maybe ? Color.Red : this.ResultView.Rows[index].Cells[s.ToString() + "r"].Style.ForeColor;
this.ResultView.Rows[index].Cells["查询时间"].Value = DateTime.Now.ToString(); this.ResultView.Rows[index].Cells["查询时间"].Value = DateTime.Now.ToString();
Application.DoEvents(); Application.DoEvents();
} }

90
HttpHelper.cs
View File

@ -13,15 +13,15 @@ using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
namespace EnAliKeywordSearch namespace CityCraft
{ {
public enum HttpMethod
{
GET,
POST
}
public class HttpArgs public class HttpArgs
{ {
public enum HttpMethod
{
GET,
POST
}
public string Url { get; set; } public string Url { get; set; }
public string Host { get; set; } public string Host { get; set; }
public int Port { get; set; } public int Port { get; set; }
@ -32,10 +32,22 @@ namespace EnAliKeywordSearch
public string UA { get; set; } public string UA { get; set; }
public HttpMethod Method { get; set; } public HttpMethod Method { get; set; }
} }
public enum HttpReadyState
{
未初始化,
载入,
载入完成,
交互,
完成
}
public class HttpHelper public class HttpHelper
{ {
public static int State = 0; public static HttpReadyState readyState = HttpReadyState.未初始化;
public static int Status = 0;
public static string responseBody = "";
public static string responseText = "";
public static byte[] responseByte = null;
public static HttpArgs args = new HttpArgs();
public static string ErrMsg = string.Empty; public static string ErrMsg = string.Empty;
/// <summary> /// <summary>
/// 提交方法 /// 提交方法
@ -48,14 +60,19 @@ namespace EnAliKeywordSearch
/// <param name="geturl">请求地址</param> /// <param name="geturl">请求地址</param>
/// <param name="cookieser">Cookies存储器</param> /// <param name="cookieser">Cookies存储器</param>
/// <returns>请求返回的Stream</returns> /// <returns>请求返回的Stream</returns>
public string Get(string url) public void Send(HttpMethod method, string url)
{ {
HttpArgs args = ParseURL(url); readyState = HttpReadyState.载入;
args.Method = HttpArgs.HttpMethod.GET; ParseURL(url);
string strhtml = InternalSocketHttp(args); args.Method = method;
return strhtml; new Thread(new ThreadStart(ReciveData)).Start();
} }
public void ReciveData()
{
responseBody = InternalSocketHttp();
readyState = HttpReadyState.完成;
}
/// <summary> /// <summary>
/// Post方法 /// Post方法
/// </summary> /// </summary>
@ -76,25 +93,23 @@ namespace EnAliKeywordSearch
/// </summary> /// </summary>
/// <param name="strUrl">url字符串</param> /// <param name="strUrl">url字符串</param>
/// <returns>host字符串</returns> /// <returns>host字符串</returns>
private HttpArgs ParseURL(string strUrl) private void ParseURL(string strUrl)
{ {
HttpArgs args = new HttpArgs(); if (args == null)
args = new HttpArgs();
args.Host = ""; args.Host = "";
args.Port = 80; args.Port = 80;
args.Referer = ""; args.Referer = "";
args.Cookie = ""; args.Cookie = "";
args.Url = ""; args.Url = "";
args.Accept = "text/html";//,application/xhtml+xml,application/xml,application/json;" args.Accept = "text/html";//,application/xhtml+xml,application/xml,application/json;";
args.UA = "Mozilla/5.0+(Compatible;+Baiduspider/2.0;++http://www.baidu.com/search/spider.html)";
//args.UA = "Mozilla/5.0+(Compatible;+Baiduspider/2.0;++http://www.baidu.com/search/spider.html)";
//args.UA = @"Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
args.UA = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 UBrowser/5.2.2603.31 Safari/537.36";
//http://www.alibaba.com/products/Egg_Laying_Block_Machine/1.html //http://www.alibaba.com/products/Egg_Laying_Block_Machine/1.html
int iIndex = strUrl.IndexOf(@"//"); int iIndex = strUrl.IndexOf(@"//");
if (iIndex <= 0) if (iIndex <= 0)
return null; args = null;
//www.alibaba.com:80/products/Egg_Laying_Block_Machine/1.html //www.alibaba.com:80/products/Egg_Laying_Block_Machine/1.html
string nohttpurl = strUrl.Substring(iIndex + 2); string nohttpurl = strUrl.Substring(iIndex + 2);
string address = nohttpurl; string address = nohttpurl;
@ -108,7 +123,7 @@ namespace EnAliKeywordSearch
iIndex = nohttpurl.IndexOf(@":"); iIndex = nohttpurl.IndexOf(@":");
if (iIndex > 0) if (iIndex > 0)
{ {
string[] tempargs = nohttpurl.Trim().Split(char.Parse(":")); string[] tempargs = address.Trim().Split(char.Parse(":"));
args.Host = tempargs[0]; args.Host = tempargs[0];
args.Port = int.Parse(tempargs[1]); args.Port = int.Parse(tempargs[1]);
} }
@ -118,13 +133,12 @@ namespace EnAliKeywordSearch
args.Host = address; args.Host = address;
args.Port = 80; args.Port = 80;
} }
return args;
} }
#endregion #endregion
#region Socket #region Socket
string InternalSocketHttp(HttpArgs args) string InternalSocketHttp()
{ {
using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp))
{ {
@ -135,7 +149,7 @@ namespace EnAliKeywordSearch
socket.Connect(args.Host, args.Port); socket.Connect(args.Host, args.Port);
if (socket.Connected) if (socket.Connected)
{ {
byte[] buff = ParseHttpArgs(args); byte[] buff = ParseHttpArgs();
if (socket.Send(buff) > 0) if (socket.Send(buff) > 0)
{ {
List<byte> responseBytes = new List<byte>(); List<byte> responseBytes = new List<byte>();
@ -146,7 +160,9 @@ namespace EnAliKeywordSearch
responseBytes.AddRange(new List<byte>(buffer));//添加数据到List responseBytes.AddRange(new List<byte>(buffer));//添加数据到List
iNumber = socket.Receive(buffer, buffer.Length, SocketFlags.None);//继续接收数据 iNumber = socket.Receive(buffer, buffer.Length, SocketFlags.None);//继续接收数据
} }
return ParseResponse(responseBytes.ToArray()/*转换List为数组*/, args); responseByte = responseBytes.ToArray();
readyState = HttpReadyState.载入完成;
return ParseResponse(responseByte);
} }
} }
} }
@ -158,7 +174,7 @@ namespace EnAliKeywordSearch
} }
} }
private string ParseResponse(byte[] responseBytes, HttpArgs args) private string ParseResponse(byte[] responseBytes)
{ {
string responseStr = Encoding.UTF8.GetString(responseBytes); string responseStr = Encoding.UTF8.GetString(responseBytes);
int splitindex = responseStr.IndexOf("\r\n\r\n"); int splitindex = responseStr.IndexOf("\r\n\r\n");
@ -166,20 +182,19 @@ namespace EnAliKeywordSearch
{ {
string responseHeader = responseStr.Substring(0, splitindex); string responseHeader = responseStr.Substring(0, splitindex);
string responseBody = responseStr.Substring(splitindex + 4); string responseBody = responseStr.Substring(splitindex + 4);
if (responseHeader.StartsWith("HTTP/1.1 400"))
if (responseHeader.StartsWith("HTTP/1.1 400 Bad Request"))
{ {
State = 400; Status = 400;
return string.Empty; return string.Empty;
} }
else if (responseHeader.StartsWith("HTTP/1.1 404")) else if (responseHeader.StartsWith("HTTP/1.1 404"))
{ {
State = 404; Status = 404;
return string.Empty; return string.Empty;
} }
else if (responseHeader.StartsWith("HTTP/1.1 302") || responseHeader.StartsWith("HTTP/1.1 301")) else if (responseHeader.StartsWith("HTTP/1.1 302") || responseHeader.StartsWith("HTTP/1.1 301"))
{ {
State = 302; Status = 302;
int start = responseHeader.ToUpper().IndexOf("LOCATION"); int start = responseHeader.ToUpper().IndexOf("LOCATION");
if (start > 0) if (start > 0)
{ {
@ -188,12 +203,13 @@ namespace EnAliKeywordSearch
args.Url = sArry[0].Remove(0, 10); args.Url = sArry[0].Remove(0, 10);
if (args.Url == "") if (args.Url == "")
return string.Empty; return string.Empty;
return InternalSocketHttp(args); //注意：302协议需要重定向 return InternalSocketHttp(); //注意：302协议需要重定向
} }
} }
else if (responseHeader.StartsWith("HTTP/1.1 200")) //读取内容 else if (responseHeader.StartsWith("HTTP/1.1 200")) //读取内容
{ {
State = 200; Status = 200;
//解压
DecompressWebPage(ref responseBytes, responseHeader); DecompressWebPage(ref responseBytes, responseHeader);
//转码 //转码
responseBody = DecodeWebStringByHttpHeader(responseBytes, responseHeader); responseBody = DecodeWebStringByHttpHeader(responseBytes, responseHeader);
@ -298,10 +314,10 @@ namespace EnAliKeywordSearch
return strResponse; return strResponse;
} }
private byte[] ParseHttpArgs(HttpArgs args) private byte[] ParseHttpArgs()
{ {
StringBuilder bulider = new StringBuilder(); StringBuilder bulider = new StringBuilder();
if (args.Method == HttpArgs.HttpMethod.POST) if (args.Method == HttpMethod.POST)
{ {
bulider.AppendLine(string.Format("POST {0} HTTP/1.1", args.Url)); bulider.AppendLine(string.Format("POST {0} HTTP/1.1", args.Url));
bulider.AppendLine("Content-Type: application/x-www-form-urlencoded"); bulider.AppendLine("Content-Type: application/x-www-form-urlencoded");
@ -328,7 +344,7 @@ namespace EnAliKeywordSearch
if (!string.IsNullOrEmpty(args.Cookie)) if (!string.IsNullOrEmpty(args.Cookie))
bulider.AppendLine(string.Format("Cookie: {0}", args.Cookie)); bulider.AppendLine(string.Format("Cookie: {0}", args.Cookie));
if (args.Method == HttpArgs.HttpMethod.POST) if (args.Method == HttpMethod.POST)
{ {
bulider.AppendLine(string.Format("Content-Length: {0}\r\n", Encoding.Default.GetBytes(args.Data).Length)); bulider.AppendLine(string.Format("Content-Length: {0}\r\n", Encoding.Default.GetBytes(args.Data).Length));
bulider.Append(args.Data); bulider.Append(args.Data);

40
Program.cs
View File

@ -1,20 +1,20 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Windows.Forms; using System.Windows.Forms;
namespace EnAliKeywordSearch namespace SEOKeywordSearch
{ {
static class Program static class Program
{ {
/// <summary> /// <summary>
/// 应用程序的主入口点。 /// 应用程序的主入口点。
/// </summary> /// </summary>
[STAThread] [STAThread]
static void Main() static void Main()
{ {
Application.EnableVisualStyles(); Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false); Application.SetCompatibleTextRenderingDefault(false);
Application.Run(new Frm_Main()); Application.Run(new Frm_Main());
} }
} }
} }