public void InsertPage(Page p)
{
var database = Client.GetDatabase("webdb");
var collection = database.GetCollection<BsonDocument>("pages");
collection.InsertOne(new BsonDocument
{
{"PageId", p.PageId.ToString() },
{"Tag", p.Link.GetFullPath(false) },
{"Title", p.Name },
//{"Content", Utility.GetBytes(p.Content) }
{"AggregatedDateTime", DateTime.Now }
});
}
public void PersistData(Page PageItem)
{
try
{
Console.WriteLine("Persisting: " + PageItem.Link.GetFullPath(false));
// ascertain the number of total operations
var numberOfOperations = 0;
numberOfOperations += PageItem.FileTags.Count;
numberOfOperations += PageItem.LinkTags.Count;
numberOfOperations += PageItem.ImageTags.Count;
numberOfOperations++;
// wait until the page queue contains no more items
int numberOfCompletedOperations = 0;
try
{
// insert the page and related items
InsertPage(PageItem);
foreach (LinkTag Hyperlink in PageItem.LinkTags)
{
InsertLink(Hyperlink);
numberOfCompletedOperations++;
}
foreach (BinaryFile Image in PageItem.ImageTags)
{
InsertBinaryFile(Image, "IMAGE");
numberOfCompletedOperations++;
}
foreach (BinaryFile FileItem in PageItem.FileTags)
{
InsertBinaryFile(FileItem, "FILE");
numberOfCompletedOperations++;
}
}
catch (Exception e)
{
throw;
}
finally
{
numberOfCompletedOperations++;
}
}
catch (Exception e)
{
throw;
}
}
public void PersistData(Page PageItem)
{
try
{
// ascertain the number of total operations
int NumberOfOperations = 0;
NumberOfOperations += PageItem.FileTags.Count();
NumberOfOperations += PageItem.LinkTags.Count();
NumberOfOperations += PageItem.ImageTags.Count();
NumberOfOperations++;
// wait until the page queue contains no more items
int NumberOfCompletedOperations = 0;
try
{
// insert the page and related items
InsertPage(PageItem);
foreach (LinkTag Hyperlink in PageItem.LinkTags)
{
InsertLink(Hyperlink);
NumberOfCompletedOperations++;
}
foreach (BinaryFile Image in PageItem.ImageTags)
{
InsertBinaryFile(Image, "IMAGE");
NumberOfCompletedOperations++;
}
foreach (BinaryFile FileItem in PageItem.FileTags)
{
InsertBinaryFile(FileItem, "FILE");
NumberOfCompletedOperations++;
}
}
catch (Exception e)
{
log.Error(e);
throw;
}
finally
{
NumberOfCompletedOperations++;
}
}
catch (Exception e)
{
log.Error(e);
// throw;
}
}
public void InsertPage(Page p)
{
string sql = "INSERT INTO PageTable (PageId, Tag, Title, Content) VALUES (@PageId, @Tag, @Title, @Content);";
try
{
using (MySqlCommand c = new MySqlCommand(sql, client))
{
c.Parameters.AddWithValue("@PageId", p.PageId.ToString());
c.Parameters.AddWithValue("@Tag", p.Link.GetFullPath(false));
c.Parameters.AddWithValue("@Title", p.Name);
c.Parameters.AddWithValue("@Content", Utility.GetBytes(p.Content));
c.ExecuteNonQuery();
}
Console.WriteLine("Page persisted: " + p.Link.GetFullPath(false));
}
catch (Exception e)
{
log.Error("Inserting page failed", e);
throw;
}
}
public void ProcessNewPaths(Page p, UrlObject domainObject)
{
if (p != null && domainObject != null)
{
Console.WriteLine("Visited: " + p.Link.GetFullPath(false));
Unvisited.Remove(p.Link.GetFullPath(false));
if (!Visited.ContainsKey(p.Link.GetFullPath(false)))
{
Visited.Add(p.Link.GetFullPath(false), p);
}
foreach (LinkTag l in p.LinkTags)
{
var toBeVisited = false;
var visited = false;
try
{
var key = Unvisited[l.Url.GetFullPath(false)];
toBeVisited = true;
}
catch (KeyNotFoundException /* knfe */) { }
try
{
var key = Visited[l.Url.GetFullPath(false)];
visited = true;
}
catch (KeyNotFoundException /* knfe */) { }
if (toBeVisited != true
& visited != true)
{
if (l.Url.GetDomain() == domainObject.GetDomain())
Unvisited.Add(l.Url.GetFullPath(false), l.Url);
}
}
}
}
