public void Dispose()
{
if (!_disposed)
{
try
{
if (_xlsBook != null)
{
_xlsBook.Close(false, Missing.Value, Missing.Value);
ReleaseComObj(_xlsBook);
_xlsBook = null;
}
if (_xlsApp != null)
{
_xlsApp.Quit();
ReleaseComObj(_xlsApp);
_xlsApp = null;
}
_disposed = true;
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch (Exception e)
{
GlobalLog.Err(e, "Exception while dispose XLSWorker");
}
}
}
private void EndPostImage(IAsyncResult ar)
{
HttpWebRequest request = (ar.AsyncState as HttpWebRequest);
try
{
HttpWebResponse response = request.EndGetResponse(ar) as HttpWebResponse;
byte[] buffer = new byte[8 * 1024];
byte[] data;
using (Stream stream = response.GetResponseStream())
{
using (MemoryStream ms = new MemoryStream())
{
int readed;
while ((readed = stream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, readed);
}
data = ms.GetBuffer();
}
}
string dataStr = StreamEncoding.GetString(data);
HandlePostState(dataStr);
}
catch (Exception e)
{
#if DEBUG
GlobalLog.Err(e, "error while data reading");
#endif
Err(ErrorState.ConnectProblem);
}
}
void EndCheckAnonimous(DownloaderObj obj)
{
AnonymousRegexes anonRegxs = GlobalResourceCache.AnonymCheck;
if (obj.DataStr != null)
{
if (anonRegxs.Anonymous.IsMatch(obj.DataStr))
{
obj.Proxy.AnonymousLevel = AnonymousLevel.Anonymous;
}
else if (anonRegxs.HightAnonymous.IsMatch(obj.DataStr))
{
obj.Proxy.AnonymousLevel = AnonymousLevel.HightAnonymous;
}
else
{
obj.Proxy.AnonymousLevel = AnonymousLevel.NotAnonymous;
}
}
else
{
if (_logging)
{
GlobalLog.Err("Cant download page from anon checking site, p:" + obj.Proxy.Address.Host);
}
}
}
internal void BeginReceive(DownloaderObj obj)
{
if (obj.Attempts > 0)
{
obj.Attempts--;
try
{
obj.Request = CreateRequest(obj);
if (obj.PostData != null)
{
PostRequestData(obj.Request, obj.PostData);
}
//Console.WriteLine("Send request to -- {0}", obj.Uri.OriginalString);
obj.Request.BeginGetResponse(EndReceive, obj);
}
catch (WebException e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
HandleWebState(e, obj);
RetryOrCallback(obj);
}
catch (Exception e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
RetryOrCallback(obj);
}
}
}
private void SessionCallback()
{
try
{
_callBack.Invoke(this);
}
catch (Exception e)
{
GlobalLog.Err(e, "Error while callback in HttpSession, please add catch below");
}
}
/// <summary>
/// Post data callback
/// </summary>
private void EndPostImage(DownloaderObj obj)
{
if (obj.DataStr != null)
{
HandlePostState(obj.DataStr);
}
else
{
#if DEBUG
GlobalLog.Err("null data in EndPostImage, POST_DATA:\n{0}", obj.PostData);
#endif
Err(ErrorState.ConnectProblem);
}
}
void CallbackAndContinue(DownloaderObj obj)
{
if (obj.CallBack != null)
{
try
{
obj.CallBack.Invoke(obj);
}
catch (Exception e)
{
GlobalLog.Err(e, "CallBack err");
}
}
Downloader.ProcessNext(obj.Uri);
}
internal void EndReceive(IAsyncResult ar)
{
if (!ExecutionContext.IsFlowSuppressed())
{
ExecutionContext.SuppressFlow();
}
DownloaderObj obj = ar.AsyncState as DownloaderObj;
try
{
#if DEBUG
GlobalLog.Write("Get responce from " + obj.Uri.Host);
#endif
obj.Response = obj.Request.EndGetResponse(ar) as HttpWebResponse;
HandleRedirectAndCookies(obj);
if (TryReceiveData(obj))
{
CallbackAndContinue(obj);
return;
}
#if DEBUG
GlobalLog.Write("CANT Downloaded data from " + obj.Uri.Host);
#endif
}
catch (WebException e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
HandleWebState(e, obj);
}
catch (Exception e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
}
finally
{
if (obj.Response != null)
{
obj.Response.Close();
}
}
RetryOrCallback(obj);
}
private void AskRecognize(string id)
{
Uri resolveUri = new Uri(_resolveStr + id);
DownloaderObj obj = new DownloaderObj(resolveUri, null, true, null, CookieOptions.Empty, 5);
Downloader.DownloadSync(obj);
if (obj.DataStr != null)
{
HandleRecognizeState(obj.DataStr, id);
}
else
{
#if DEBUG
GlobalLog.Err("can't ASK data, id:{0}", id);
#endif
Err(ErrorState.ConnectProblem);
}
}
void DownloadGooglePage(RatedProxy proxy)
{
Uri googleUri = new Uri("http://www.google.com/search?q=flowers&sourceid=opera&ie=utf-8&oe=utf-8");
DownloaderObj obj = new DownloaderObj(googleUri, CheckGooglePage, true, proxy, CookieOptions.Empty, 2);
if (Downloader.HaveResponce(obj))
{
obj.Attempts = 2;
Downloader.Queue(obj);
}
else
{
if (_logging)
{
GlobalLog.Err("Cant get responce from http://google.com, perhaps proxy or ya host is down, p:" + obj.Proxy.Address.Host);
}
}
}
private void ReleaseComObj(object obj)
{
if (obj == null)
{
return;
}
try
{
Marshal.FinalReleaseComObject(obj);
obj = null;
}
catch (Exception e)
{
obj = null;
GlobalLog.Err(e, "Error when release xls com Obj");
}
}
void DownloadYaPage(RatedProxy proxy)
{
Uri yandexUri = new Uri("http://kiks.yandex.ru/su/");
DownloaderObj obj = new DownloaderObj(yandexUri, CheckYaPage, true, proxy, CookieOptions.Take, 2);
if (Downloader.HaveResponce(obj))
{
obj.Uri = new Uri("http://wordstat.yandex.ru/");
obj.Attempts = 2;
Downloader.Queue(obj);
}
else
{
if (_logging)
{
GlobalLog.Err("Cant get responce from http://kiks.yandex.ru, perhaps proxy or ya host is down, p:" + obj.Proxy.Address.Host);
}
}
}
private void HandleSuccessDownload(DownloaderObj obj)
{
_proxies.Release(obj.PrxContainer as ProxyContainer, true);
DevourTarget target = obj.Arg as DevourTarget;
//MoveReadQueue();
try
{
MoveReadQueue();
target.Reader.ReadData(obj.DataStr, target);
MoveReadComplete(target);
}
catch (Exception ex)
{
GlobalLog.Err(ex, "error while reading data in devourer");
}
AddSuccessTarget(obj.Uri);
}
public CookieCollection GetCookies(DownloaderObj obj)
{
while (obj.Attempts > 0)
{
obj.Attempts--;
try
{
obj.Request = CreateRequest(obj);
obj.Response = obj.Request.GetResponse() as HttpWebResponse;
if (obj.Request.HaveResponse && obj.Response.Cookies != null)
{
return(obj.Response.Cookies);
}
}
catch (WebException e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
HandleWebState(e, obj);
}
catch (Exception e)
{
throw e;
}
finally
{
if (obj.Request != null)
{
obj.Request.Abort();
}
if (obj.Response != null)
{
obj.Response.Close();
}
}
}
return(null);
}
public bool HaveResponce(DownloaderObj obj)
{
while (obj.Attempts > 0)
{
obj.Attempts--;
try
{
obj.Request = CreateRequest(obj);
bool haveResp = obj.Request.HaveResponse;
obj.Request.Abort();
return(haveResp);
}
catch (WebException e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
HandleWebState(e, obj);
}
catch (Exception e)
{
throw e;
}
}
return(false);
}
static CookieContainer _sharedCookies = new CookieContainer(); //PUBLIC SHARED COOKIES
internal void Receive(DownloaderObj obj)
{
while (obj.Attempts > 0)
{
obj.Attempts--;
try
{
obj.Request = CreateRequest(obj);
if (obj.PostData != null)
{
PostRequestData(obj.Request, obj.PostData);
}
obj.Response = obj.Request.GetResponse() as HttpWebResponse;
HandleRedirectAndCookies(obj);
if (TryReceiveData(obj))
{
break;
}
}
catch (WebException e)
{
GlobalLog.Err(e, "Host: " + obj.Uri.Host);
HandleWebState(e, obj);
}
finally
{
if (obj.Response != null)
{
obj.Response.Close();
}
}
}
}
