private void AddProcessedWebDirectory(WebDirectory webDirectory, WebDirectory parsedWebDirectory, bool processSubdirectories = true)
{
webDirectory.Description = parsedWebDirectory.Description;
webDirectory.StartTime = parsedWebDirectory.StartTime;
webDirectory.Files = parsedWebDirectory.Files;
webDirectory.Finished = parsedWebDirectory.Finished;
webDirectory.FinishTime = parsedWebDirectory.FinishTime;
webDirectory.Name = parsedWebDirectory.Name;
webDirectory.Subdirectories = parsedWebDirectory.Subdirectories;
webDirectory.Url = parsedWebDirectory.Url;
if (processSubdirectories)
{
foreach (WebDirectory subdirectory in webDirectory.Subdirectories)
{
if (!Session.ProcessedUrls.Contains(subdirectory.Url))
{
if (subdirectory.Uri.Host != Constants.GoogleDriveDomain && subdirectory.Uri.Host != Constants.BlitzfilesTechDomain && !SameHostAndDirectory(Session.Root.Uri, subdirectory.Uri))
{
Logger.Debug($"Removed subdirectory {subdirectory.Uri} from parsed webdirectory because it is not the same host");
}
else
{
WebDirectoriesQueue.Enqueue(subdirectory);
}
}
else
{
//Logger.Warn($"Url '{subdirectory.Url}' already processed, skipping! Source: {webDirectory.Url}");
}
}
}
if (parsedWebDirectory.Error && !Session.UrlsWithErrors.Contains(webDirectory.Url))
{
Session.UrlsWithErrors.Add(webDirectory.Url);
}
webDirectory.Files.Where(f =>
{
Uri uri = new Uri(f.Url);
if (uri.Host == Constants.GoogleDriveDomain || uri.Host == Constants.BlitzfilesTechDomain)
{
return(false);
}
return((uri.Scheme != Constants.UriScheme.Https && uri.Scheme != Constants.UriScheme.Http && uri.Scheme != Constants.UriScheme.Ftp && uri.Scheme != Constants.UriScheme.Ftps) || uri.Host != Session.Root.Uri.Host || !SameHostAndDirectory(uri, Session.Root.Uri));
}).ToList().ForEach(wd => webDirectory.Files.Remove(wd));
foreach (WebFile webFile in webDirectory.Files.Where(f => f.FileSize == Constants.NoFileSize || OpenDirectoryIndexerSettings.CommandLineOptions.ExactFileSizes))
{
WebFilesFileSizeQueue.Enqueue(webFile);
}
}
private void AddProcessedWebDirectory(WebDirectory webDirectory, WebDirectory parsedWebDirectory, bool processSubdirectories = true)
{
webDirectory.Description = parsedWebDirectory.Description;
webDirectory.StartTime = parsedWebDirectory.StartTime;
webDirectory.Files = parsedWebDirectory.Files;
webDirectory.Finished = parsedWebDirectory.Finished;
webDirectory.Name = parsedWebDirectory.Name;
webDirectory.Subdirectories = parsedWebDirectory.Subdirectories;
webDirectory.Url = parsedWebDirectory.Url;
if (processSubdirectories)
{
foreach (WebDirectory subdirectory in webDirectory.Subdirectories)
{
if (!Session.ProcessedUrls.Contains(subdirectory.Url))
{
if (subdirectory.Uri.Host != Constants.GoogleDriveDomain && (subdirectory.Uri.Host != Session.Root.Uri.Host || !subdirectory.Uri.LocalPath.StartsWith(Session.Root.Uri.LocalPath)))
{
Logger.Debug($"Removed subdirectory {subdirectory.Uri} from parsed webdirectory because it is not the same host");
}
else
{
WebDirectoriesQueue.Enqueue(subdirectory);
}
}
else
{
//Logger.Warn($"Url '{subdirectory.Url}' already processed, skipping! Source: {webDirectory.Url}");
}
}
}
if (parsedWebDirectory.Error && !Session.UrlsWithErrors.Contains(webDirectory.Url))
{
Session.UrlsWithErrors.Add(webDirectory.Url);
}
webDirectory.Files.RemoveAll(f =>
{
Uri uri = new Uri(f.Url);
if (uri.Host == Constants.GoogleDriveDomain)
{
return(false);
}
return((uri.Scheme != "https" && uri.Scheme != "http" && uri.Scheme != "ftp") || uri.Host != Session.Root.Uri.Host || !uri.LocalPath.StartsWith(Session.Root.Uri.LocalPath));
});
foreach (WebFile webFile in webDirectory.Files.Where(f => f.FileSize == -1 || OpenDirectoryIndexerSettings.CommandLineOptions.ExactFileSizes))
{
WebFilesFileSizeQueue.Enqueue(webFile);
}
}
private void TimerStatistics_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
if (WebDirectoriesQueue.Any() || RunningWebDirectoryThreads > 0)
{
Logger.Warn(Statistics.GetSessionStats(Session));
Logger.Warn($"Queue: {Library.FormatWithThousands(WebDirectoriesQueue.Count)}, Queue (filesizes): {Library.FormatWithThousands(WebFilesFileSizeQueue.Count)}");
}
if (WebFilesFileSizeQueue.Any() || RunningWebFileFileSizeThreads > 0)
{
Logger.Warn($"Remaing urls to retrieve filesize: {Library.FormatWithThousands(WebFilesFileSizeQueue.Count)}");
}
}
