public void ReadDBIndex()
{
SQLiteDataReader reader = WebData.GetReaderByExecution(connection, "select url, idx from webs");
string url;
int idx;
url2idx.Clear();
idx2url.Clear();
while (reader.Read())
{
url = reader.GetString(0);
idx = reader.GetInt32(1);
url2idx.Add(url, idx);
idx2url.Add(idx, url);
}
}
public int ReadDBContent()
{
SQLiteDataReader reader = WebData.GetReaderByExecution(connection, "select url, md5 from webs");
string url, md5;
url2page.Clear();
while (reader.Read())
{
url = reader.GetString(0);
md5 = reader.GetString(1);
int idx = url2idx[url];
if (WebData.LoadFromLocal(idx, out string content))
{
if (content.GetMD5() == md5)
{
url2page.Add(url, new WEBPAGE(url, content));
}
else
{
Console.WriteLine("File at index {0} has different md5, md5 of content is {1}, but in db is {2}", idx, content.GetMD5(), md5);
}
}
else
{
Console.WriteLine("Failed to load file at index {0}", idx);
}
}
foreach (string src in url2idx.Keys)
{
foreach (string key in url2page[src].LINKS)
{
if (url2idx.ContainsKey(key))
{
url2page[src].point_id.Add(url2idx[key]);
}
}
}
return(url2page.Count);
}