private static void CheckFiles(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); int count = fileDatabase.GetFileCount(); var baseDirectory = Path.GetDirectoryName(databaseFileName); var webCrawler = new WebCrawler(baseDirectory); for (var i = 0; i < count; i++) { var fileDatabaseRecord = fileDatabase.GetFile(i); if (fileDatabaseRecord.Size < 0) { var webHeaders = webCrawler.DownloadHeaders(fileDatabaseRecord.Url); fileDatabaseRecord.Modify(webHeaders.LastModified, webHeaders.ContentLength, webHeaders.ContentType, "", false, false); } var fileName = Path.Combine(baseDirectory, fileDatabaseRecord.FileName); var fileInfo = new FileInfo(fileName); if (fileInfo.Exists) { fileDatabaseRecord.Available = true; fileDatabaseRecord.OutOfDate = (fileInfo.Length != fileDatabaseRecord.Size) || (fileInfo.LastWriteTime != fileDatabaseRecord.Modified); } fileDatabase.AddOrReplaceFile(fileDatabaseRecord); } }
static private void CheckFiles(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); int count = fileDatabase.GetFileCount(); var baseDirectory = Path.GetDirectoryName(databaseFileName); var webCrawler = new WebCrawler(baseDirectory); for (var i = 0; i < count; i++) { var fileDatabaseRecord = fileDatabase.GetFile(i); if (fileDatabaseRecord.Size < 0) { var webHeaders = webCrawler.DownloadHeaders(fileDatabaseRecord.Url); fileDatabaseRecord.Modify(webHeaders.LastModified, webHeaders.ContentLength, webHeaders.ContentType, "", false, false); } var fileName = Path.Combine(baseDirectory, fileDatabaseRecord.FileName); var fileInfo = new FileInfo(fileName); if (fileInfo.Exists) { fileDatabaseRecord.Available = true; fileDatabaseRecord.OutOfDate = (fileInfo.Length != fileDatabaseRecord.Size) || (fileInfo.LastWriteTime != fileDatabaseRecord.Modified); } fileDatabase.AddOrReplaceFile(fileDatabaseRecord); } }
public WebCrawler(String baseDirectory = null, Int32 minDelay = 1000, Int32 maxDelay = 3000) : base(baseDirectory, minDelay, maxDelay) { _baseDirectory = baseDirectory; EnsureDirectoryExists(_baseDirectory); _tempDirectory = Path.Combine(_baseDirectory, "_temp"); EnsureDirectoryExists(_tempDirectory); var fileName = Path.Combine(_baseDirectory, "_files.sqlite"); _fileDatabase = new FileDatabase(fileName); }
static private void PrintInfo(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); int count = fileDatabase.GetFileCount(); int available = fileDatabase.GetAvailableFileCount(); int outOfDate = fileDatabase.GetOutOfDateFileCount(); Console.WriteLine("Total files: {0}", count); Console.WriteLine("Available files: {0}", available); Console.WriteLine("Out-of-date files: {0}", outOfDate); Console.WriteLine("Up-to-date files: {0} ({1:N1}%)", available - outOfDate, (available - outOfDate) * 100.0 / count); }
static private void UpdateFiles(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); var webCrawler = new WebCrawler(Path.GetDirectoryName(databaseFileName)); while (true) { var fileDatabaseRecord = fileDatabase.GetNextNotAvailableOrOutOfDateFile(); if (null == fileDatabaseRecord) { return; } webCrawler.UpdateFile(fileDatabaseRecord); } }
private static void ListFiles(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); int count = fileDatabase.GetFileCount(); Console.WriteLine("{0} files in database:", count); Console.WriteLine(); for (var i = 0; i < count; i++) { var fileDatabaseRecord = fileDatabase.GetFile(i); Console.WriteLine("Url: {0}", fileDatabaseRecord.Url); Console.WriteLine("FileName: {0}", fileDatabaseRecord.FileName); Console.WriteLine("Modified: {0}", fileDatabaseRecord.Modified); Console.WriteLine("Size: {0}", fileDatabaseRecord.Size); Console.WriteLine("Type: {0}", fileDatabaseRecord.Type); Console.WriteLine("Checksum: {0}", fileDatabaseRecord.Checksum); Console.WriteLine("Available: {0}", fileDatabaseRecord.Available); Console.WriteLine("OutOfDate: {0}", fileDatabaseRecord.OutOfDate); Console.WriteLine(); } }
static private void ListFiles(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); int count = fileDatabase.GetFileCount(); Console.WriteLine("{0} files in database:", count); Console.WriteLine(); for (var i = 0; i < count; i++) { var fileDatabaseRecord = fileDatabase.GetFile(i); Console.WriteLine("Url: {0}", fileDatabaseRecord.Url); Console.WriteLine("FileName: {0}", fileDatabaseRecord.FileName); Console.WriteLine("Modified: {0}", fileDatabaseRecord.Modified); Console.WriteLine("Size: {0}", fileDatabaseRecord.Size); Console.WriteLine("Type: {0}", fileDatabaseRecord.Type); Console.WriteLine("Checksum: {0}", fileDatabaseRecord.Checksum); Console.WriteLine("Available: {0}", fileDatabaseRecord.Available); Console.WriteLine("OutOfDate: {0}", fileDatabaseRecord.OutOfDate); Console.WriteLine(); } }
private static void PrintInfo(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); int count = fileDatabase.GetFileCount(); int available = fileDatabase.GetAvailableFileCount(); int outOfDate = fileDatabase.GetOutOfDateFileCount(); Console.WriteLine("Total files: {0}", count); Console.WriteLine("Available files: {0}", available); Console.WriteLine("Out-of-date files: {0}", outOfDate); Console.WriteLine("Up-to-date files: {0} ({1:N1}%)", available - outOfDate, (available - outOfDate) * 100.0 / count); }
private static void UpdateFiles(String databaseFileName) { var fileDatabase = new FileDatabase(databaseFileName); var webCrawler = new WebCrawler(Path.GetDirectoryName(databaseFileName)); while (true) { var fileDatabaseRecord = fileDatabase.GetNextNotAvailableOrOutOfDateFile(); if (null == fileDatabaseRecord) { return; } webCrawler.UpdateFile(fileDatabaseRecord); } }