public void crawler_CurrentPageContentEvent(object sender, CurrentPageContentEventArgs e)
{
HttpWebRequest webRequest;
HttpWebResponse webResponse = null;
String postUrl;
try
{
_history.Add(e.Url);
postUrl = String.Format("{0}update/extract?literal.id={1}&prefix=attr_&fmap.content=body&commit=true", HelperLib.Util.GetSolrUrl(true), HttpUtility.UrlEncode(e.Url));
webRequest = (HttpWebRequest)WebRequest.Create(postUrl);
webRequest.Method = "POST";
using (Stream requestStream = webRequest.GetRequestStream())
{
byte[] fileContent = Encoding.UTF8.GetBytes(e.Content);
requestStream.Write(fileContent, 0, fileContent.Length);
}
webResponse = (HttpWebResponse)webRequest.GetResponse();
if (_history.UrlCount % SolrCommitAfterUrls == 0)
{
CommitToSolrIndex();
}
}
finally
{
if (webResponse != null)
{
webResponse.Close();
}
}
//using (Stream responseStream = webResponse.GetResponseStream())
//using (StreamReader responseReader = new StreamReader(responseStream))
//{
// Console.WriteLine(responseReader.ReadToEnd());
//}
}
public void crawler_CurrentPageContentEvent(object sender, CurrentPageContentEventArgs e)
{
HttpWebRequest webRequest;
HttpWebResponse webResponse = null;
String postUrl;
try
{
_history.Add(e.Url);
postUrl = String.Format("{0}update/extract?literal.id={1}&prefix=attr_&fmap.content=body&commit=true", HelperLib.Util.GetSolrUrl(true), HttpUtility.UrlEncode(e.Url));
webRequest = (HttpWebRequest)WebRequest.Create(postUrl);
webRequest.Method = "POST";
using (Stream requestStream = webRequest.GetRequestStream())
{
byte[] fileContent = Encoding.UTF8.GetBytes(e.Content);
requestStream.Write(fileContent, 0, fileContent.Length);
}
webResponse = (HttpWebResponse)webRequest.GetResponse();
if (_history.UrlCount % SolrCommitAfterUrls == 0)
{
CommitToSolrIndex();
}
}
finally
{
if (webResponse != null)
{
webResponse.Close();
}
}
//using (Stream responseStream = webResponse.GetResponseStream())
//using (StreamReader responseReader = new StreamReader(responseStream))
//{
// Console.WriteLine(responseReader.ReadToEnd());
//}
}
