public static LossRunCrawlerResult Process(WebBrowser browser, string policy, bool runDuplicate, int documentLevel, string batName)
{
batchName = batName;
Debug.WriteLine("Started Web Crawler");
AutoResetEvent monitor = new AutoResetEvent(false);
CookieContainer cookieJar = new CookieContainer();
browser.ClearEventHandlers("DocumentCompleted");
browser.DocumentCompleted += (s, e) => monitor.Set();
if (!runDuplicate)
{
if (documentLevel == 0)
{
if (FindBatchFile(policy))
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Duplicate
});
}
}
else
{
if (FindBatchFile(policy, documentLevel))
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Duplicate
});
}
}
}
var connected = false;
int attempts = 0;
while (!connected)
{
try
{
browser.Navigate(@"https://my.navg.com/LossRuns/Login.aspx");
connected = true;
}
catch
{
connected = false;
//crawlLogger.AddLine("Failed to connect: " + attempts.ToString() + "Attempts thus far");
if (attempts % 5 == 0)
{
//crawlLogger.AddLine("Sleeping");
if (attempts > 5)
{
Thread.Sleep(5000 * 60);
}
if (attempts > 50)
{
Thread.Sleep(1000 * 60 * 80);
}
}
attempts++;
}
}
browser.UntilLoaded(monitor);
if (!browser.Document.ExpectLoginScreen())
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = "Was not the login screen."
});
}
HtmlDocument loginScreen = browser.Document;
if (
!loginScreen.SetFieldInTemplate("lgnBroker_UserName", "BTIS") ||
!loginScreen.SetFieldInTemplate("lgnBroker_Password", "BTIS2015") ||
!loginScreen.ClickInTemplate("lgnBroker_LoginButton")
)
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = "Could not login on login screen."
});
}
browser.UntilLoaded(monitor);
if (!browser.Document.ExpectSearchScreen())
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = "Could not log in. Was not the search screen."
});
}
HtmlDocument searchScreen = browser.Document;
if (
!searchScreen.SetFieldInTemplate("txtPolicyNumber", policy) ||
!searchScreen.ClickInTemplate("btnSearch")
)
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = "Could not search on search screen."
});
}
browser.UntilLoaded(monitor);
IEnumerable <int> documentLevels = Enumerable.Empty <int>();
if (browser.Document.ExpectMultipleResultsScreen())
{
IEnumerable <HtmlElement> elements = browser.Document.NavgSpecificSelectButtons().Skip(documentLevel).Take(1);
if (!elements.Any())
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = string.Format("Could not retrieve expected multiple results for {0} - {1}", policy, documentLevel)
});
}
elements.Single().InvokeMember("click");
browser.UntilLoaded(monitor);
}
else if (documentLevel > 0)
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = string.Format("Could not retrieve expected multiple results for {0} - {1}", policy, documentLevel)
});
}
if (!browser.Document.ExpectSingleResultScreen())
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = "No results on the result page."
});
}
HtmlDocument resultsScreen = browser.Document;
//set login cookie
int cookieSize = 1024;
StringBuilder cookieDough = new StringBuilder(cookieSize);
if (!InternetGetCookieEx("https://my.navg.com", null, cookieDough, ref cookieSize, 0x00002000, IntPtr.Zero))
{
if (cookieSize > 0)
{
cookieDough = new StringBuilder(cookieSize);
InternetGetCookieEx("https://my.navg.com", null, cookieDough, ref cookieSize, 0x00002000, IntPtr.Zero);
}
}
foreach (HtmlElement iframe in
resultsScreen
.ElementInTemplate("resultsIFrameId")
.Where(iframe => !string.IsNullOrEmpty(iframe.GetAttribute("src")))
)
{
string contentUrl = iframe.GetAttribute("src");
string cannonicalUrl =
contentUrl.StartsWith("http") ? contentUrl :
contentUrl.StartsWith("/") ? string.Format("https://my.navg.com{0}", contentUrl) :
string.Format("https://my.navg.com/LossRuns/{0}", contentUrl);
HttpWebRequest resultRequest = WebRequest.CreateHttp(cannonicalUrl);
resultRequest.CookieContainer = new CookieContainer();
resultRequest.CookieContainer.SetCookies(new Uri("https://my.navg.com"), cookieDough.ToString());
resultRequest.Referer = "https://my.navg.com/LossRuns/LossReportsSearch.aspx";
resultRequest.KeepAlive = false;
resultRequest.ProtocolVersion = HttpVersion.Version10;
HttpWebResponse resultResponse = resultRequest.GetResponse() as HttpWebResponse;
byte[] resultResponseContent;
using (MemoryStream buffer = new MemoryStream())
{
using (Stream responseStream = resultResponse.GetResponseStream())
{
byte[] intermediary = new byte[16384];
int bytesRead;
while (0 < (bytesRead = responseStream.Read(intermediary, 0, intermediary.Length)))
{
buffer.Write(intermediary, 0, bytesRead);
}
buffer.Flush();
}
buffer.Seek(0L, SeekOrigin.Begin);
resultResponseContent = buffer.ToArray();
}
//78e0 - 37 38 65 30
//newline - 0d 0A
if (
resultResponseContent.Length < 4 ||
resultResponseContent[0] != 0x25 ||
resultResponseContent[1] != 0x50 ||
resultResponseContent[2] != 0x44 ||
resultResponseContent[3] != 0x46
)
{
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Failure, Message = "Not a pdf file."
});
}
SavePolicy(policy, documentLevel, resultResponseContent);
}
foreach (int otherLevel in documentLevels)
{
LossRunCrawlerResult otherResult = Process(browser, policy, runDuplicate, otherLevel, batchName);
if (otherResult.Result == LossRunResult.Failure)
{
return(otherResult);
}
}
Debug.WriteLine("Crawler Success");
return(new LossRunCrawlerResult()
{
Result = LossRunResult.Success
});
}