public void InsertPage(Page p) { var database = Client.GetDatabase("webdb"); var collection = database.GetCollection<BsonDocument>("pages"); collection.InsertOne(new BsonDocument { {"PageId", p.PageId.ToString() }, {"Tag", p.Link.GetFullPath(false) }, {"Title", p.Name }, //{"Content", Utility.GetBytes(p.Content) } {"AggregatedDateTime", DateTime.Now } }); }
public void PersistData(Page PageItem) { try { Console.WriteLine("Persisting: " + PageItem.Link.GetFullPath(false)); // ascertain the number of total operations var numberOfOperations = 0; numberOfOperations += PageItem.FileTags.Count; numberOfOperations += PageItem.LinkTags.Count; numberOfOperations += PageItem.ImageTags.Count; numberOfOperations++; // wait until the page queue contains no more items int numberOfCompletedOperations = 0; try { // insert the page and related items InsertPage(PageItem); foreach (LinkTag Hyperlink in PageItem.LinkTags) { InsertLink(Hyperlink); numberOfCompletedOperations++; } foreach (BinaryFile Image in PageItem.ImageTags) { InsertBinaryFile(Image, "IMAGE"); numberOfCompletedOperations++; } foreach (BinaryFile FileItem in PageItem.FileTags) { InsertBinaryFile(FileItem, "FILE"); numberOfCompletedOperations++; } } catch (Exception e) { throw; } finally { numberOfCompletedOperations++; } } catch (Exception e) { throw; } }
public void PersistData(Page PageItem) { try { // ascertain the number of total operations int NumberOfOperations = 0; NumberOfOperations += PageItem.FileTags.Count(); NumberOfOperations += PageItem.LinkTags.Count(); NumberOfOperations += PageItem.ImageTags.Count(); NumberOfOperations++; // wait until the page queue contains no more items int NumberOfCompletedOperations = 0; try { // insert the page and related items InsertPage(PageItem); foreach (LinkTag Hyperlink in PageItem.LinkTags) { InsertLink(Hyperlink); NumberOfCompletedOperations++; } foreach (BinaryFile Image in PageItem.ImageTags) { InsertBinaryFile(Image, "IMAGE"); NumberOfCompletedOperations++; } foreach (BinaryFile FileItem in PageItem.FileTags) { InsertBinaryFile(FileItem, "FILE"); NumberOfCompletedOperations++; } } catch (Exception e) { log.Error(e); throw; } finally { NumberOfCompletedOperations++; } } catch (Exception e) { log.Error(e); // throw; } }
public void InsertPage(Page p) { string sql = "INSERT INTO PageTable (PageId, Tag, Title, Content) VALUES (@PageId, @Tag, @Title, @Content);"; try { using (MySqlCommand c = new MySqlCommand(sql, client)) { c.Parameters.AddWithValue("@PageId", p.PageId.ToString()); c.Parameters.AddWithValue("@Tag", p.Link.GetFullPath(false)); c.Parameters.AddWithValue("@Title", p.Name); c.Parameters.AddWithValue("@Content", Utility.GetBytes(p.Content)); c.ExecuteNonQuery(); } Console.WriteLine("Page persisted: " + p.Link.GetFullPath(false)); } catch (Exception e) { log.Error("Inserting page failed", e); throw; } }
public void ProcessNewPaths(Page p, UrlObject domainObject) { if (p != null && domainObject != null) { Console.WriteLine("Visited: " + p.Link.GetFullPath(false)); Unvisited.Remove(p.Link.GetFullPath(false)); if (!Visited.ContainsKey(p.Link.GetFullPath(false))) { Visited.Add(p.Link.GetFullPath(false), p); } foreach (LinkTag l in p.LinkTags) { var toBeVisited = false; var visited = false; try { var key = Unvisited[l.Url.GetFullPath(false)]; toBeVisited = true; } catch (KeyNotFoundException /* knfe */) { } try { var key = Visited[l.Url.GetFullPath(false)]; visited = true; } catch (KeyNotFoundException /* knfe */) { } if (toBeVisited != true & visited != true) { if (l.Url.GetDomain() == domainObject.GetDomain()) Unvisited.Add(l.Url.GetFullPath(false), l.Url); } } } }