public void ReadDBIndex() { SQLiteDataReader reader = WebData.GetReaderByExecution(connection, "select url, idx from webs"); string url; int idx; url2idx.Clear(); idx2url.Clear(); while (reader.Read()) { url = reader.GetString(0); idx = reader.GetInt32(1); url2idx.Add(url, idx); idx2url.Add(idx, url); } }
public int ReadDBContent() { SQLiteDataReader reader = WebData.GetReaderByExecution(connection, "select url, md5 from webs"); string url, md5; url2page.Clear(); while (reader.Read()) { url = reader.GetString(0); md5 = reader.GetString(1); int idx = url2idx[url]; if (WebData.LoadFromLocal(idx, out string content)) { if (content.GetMD5() == md5) { url2page.Add(url, new WEBPAGE(url, content)); } else { Console.WriteLine("File at index {0} has different md5, md5 of content is {1}, but in db is {2}", idx, content.GetMD5(), md5); } } else { Console.WriteLine("Failed to load file at index {0}", idx); } } foreach (string src in url2idx.Keys) { foreach (string key in url2page[src].LINKS) { if (url2idx.ContainsKey(key)) { url2page[src].point_id.Add(url2idx[key]); } } } return(url2page.Count); }