internal static void ProcessConsoleInput(OpenDirectoryIndexer openDirectoryIndexer)
{
if (Console.IsInputRedirected)
{
string message = "Console input is redirect, maybe it is run inside another host. This could mean that no input will be send/processed.";
Console.WriteLine(message);
Logger.Warn(message);
}
while (true)
{
try
{
if (Console.IsInputRedirected)
{
int keyPressed = Console.Read();
//if (char.IsControl((char)keyPressed))
//{
// Console.WriteLine($"Pressed Console.Read(): {keyPressed}");
//}
//else
//{
// Console.WriteLine($"Pressed Console.Read(): {(char)keyPressed}");
//}
switch (keyPressed)
{
case 'x':
case 'X':
KillApplication();
break;
case 'i':
ShowInfoAndCommands();
break;
case 'c':
if (OpenDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue)
{
new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true));
KillApplication();
}
break;
case 's':
case 'S':
ShowStatistics(openDirectoryIndexer);
break;
case 't':
case 'T':
ShowThreads(openDirectoryIndexer);
break;
case 'j':
case 'J':
SaveSession(openDirectoryIndexer);
break;
default:
break;
}
}
else
{
ConsoleKey keyPressed = Console.ReadKey(intercept: true).Key;
//Console.WriteLine($"Pressed (Console.ReadKey(): {keyPressed}");
switch (keyPressed)
{
case ConsoleKey.X:
case ConsoleKey.Escape:
KillApplication();
break;
case ConsoleKey.I:
ShowInfoAndCommands();
break;
case ConsoleKey.C:
if (OpenDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue)
{
new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true));
KillApplication();
}
break;
case ConsoleKey.S:
ShowStatistics(openDirectoryIndexer);
break;
case ConsoleKey.T:
ShowThreads(openDirectoryIndexer);
break;
case ConsoleKey.J:
SaveSession(openDirectoryIndexer);
break;
default:
break;
}
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error processing action");
throw;
}
}
}
public async void StartIndexingAsync()
{
bool fromFile = !string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.FileName);
if (fromFile)
{
Session = Library.LoadSessionJson(OpenDirectoryIndexerSettings.FileName);
Console.WriteLine(Statistics.GetSessionStats(Session, includeExtensions: true));
Console.ReadKey(intercept: true);
return;
}
else
{
Session = new Session
{
Started = DateTimeOffset.UtcNow,
Root = new WebDirectory(parentWebDirectory: null)
{
Name = "ROOT",
Url = OpenDirectoryIndexerSettings.Url
}
};
}
if (Session.Root.Uri.Host == Constants.GoogleDriveDomain)
{
Logger.Warn("Google Drive scanning is limited to 9 directories per second!");
}
if (Session.Root.Uri.Scheme == "ftp")
{
Logger.Warn("Retrieving FTP software!");
// TODO: Replace with library?
Logger.Warn(await FtpParser.GetFtpServerInfo(Session.Root));
//AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
}
TimerStatistics = new System.Timers.Timer
{
Enabled = true,
Interval = TimeSpan.FromSeconds(30).TotalMilliseconds
};
TimerStatistics.Elapsed += TimerStatistics_Elapsed;
IndexingTask = Task.Run(async() =>
{
try
{
WebDirectoriesQueue = new ConcurrentQueue <WebDirectory>();
if (fromFile)
{
SetParentDirectories(Session.Root);
// TODO: Add unfinished items to queue, very complicated, we need to ALSO fill the ParentDirectory...
//// With filter predicate, with selection function
//var flatList = nodes.Flatten(n => n.IsDeleted == false, n => n.Children);
//var directoriesToDo = Session.Root.Subdirectories.Flatten(null, wd => wd.Subdirectories).Where(wd => !wd.Finished);
}
else
{
// Add root
WebDirectoriesQueue.Enqueue(Session.Root);
}
IndexingTaskCTS = new CancellationTokenSource();
for (int i = 1; i <= WebDirectoryProcessors.Length; i++)
{
string processorId = i.ToString();
WebDirectoryProcessors[i - 1] = WebDirectoryProcessor(WebDirectoriesQueue, $"Processor {processorId}", IndexingTaskCTS.Token);
}
for (int i = 1; i <= WebFileFileSizeProcessors.Length; i++)
{
string processorId = i.ToString();
WebFileFileSizeProcessors[i - 1] = WebFileFileSizeProcessor(WebFilesFileSizeQueue, $"Processor {processorId}", IndexingTaskCTS.Token, WebDirectoryProcessors);
}
await Task.WhenAll(WebDirectoryProcessors);
Console.WriteLine("Finshed indexing");
Logger.Info("Finshed indexing");
if (Session.Root.Uri.Scheme == "ftp")
{
FtpParser.CloseAll();
}
if (WebFilesFileSizeQueue.Any())
{
TimerStatistics.Interval = TimeSpan.FromSeconds(5).TotalMilliseconds;
Console.WriteLine($"Retrieving filesize of {WebFilesFileSizeQueue.Count} urls");
}
await Task.WhenAll(WebFileFileSizeProcessors);
TimerStatistics.Stop();
Session.Finished = DateTimeOffset.UtcNow;
Session.TotalFiles = Session.Root.TotalFiles;
Session.TotalFileSizeEstimated = Session.Root.TotalFileSize;
if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoUrls && Session.Root.Uri.Host != Constants.GoogleDriveDomain)
{
if (Session.TotalFiles > 0)
{
Logger.Info("Saving URL list to file...");
Console.WriteLine("Saving URL list to file...");
string scansPath = Library.GetScansPath();
try
{
string fileUrls = string.Join(Environment.NewLine, Session.Root.AllFileUrls.Distinct());
string urlsFileName = $"{Library.CleanUriToFilename(Session.Root.Uri)}.txt";
string urlsPath = Path.Combine(scansPath, urlsFileName);
Logger.Info("String joined");
File.WriteAllText(urlsPath, fileUrls);
Logger.Info($"Saved URL list to file: {urlsFileName}");
Console.WriteLine($"Saved URL list to file: {urlsFileName}");
if (OpenDirectoryIndexerSettings.CommandLineOptions.UploadUrls && Session.TotalFiles > 0)
{
Console.WriteLine("Uploading URLs...");
//UploadFilesFile uploadFilesFile = await UploadFileIo.UploadFile(HttpClient, urlsPath);
//HistoryLogger.Info($"uploadfiles.io: {JsonConvert.SerializeObject(uploadFilesFile)}");
//Session.UploadedUrlsUrl = uploadFilesFile.Url.ToString();
GoFilesFile uploadedFile = await GoFileIo.UploadFile(HttpClient, urlsPath);
HistoryLogger.Info($"goFile.io: {JsonConvert.SerializeObject(uploadedFile)}");
Session.UploadedUrlsUrl = uploadedFile.Url.ToString();
Console.WriteLine($"Uploaded URLs: {Session.UploadedUrlsUrl}");
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
else
{
Logger.Info("No URLs to save");
Console.WriteLine("No URLs to save");
}
}
if (OpenDirectoryIndexerSettings.CommandLineOptions.Speedtest && Session.Root.Uri.Host != Constants.GoogleDriveDomain)
{
if (Session.TotalFiles > 0)
{
if (Session.Root.Uri.Scheme == "https" || Session.Root.Uri.Scheme == "http")
{
try
{
WebFile biggestFile = Session.Root.AllFiles.OrderByDescending(f => f.FileSize).First();
Console.WriteLine($"Starting speedtest (10-25 seconds)...");
Console.WriteLine($"Test file: {FileSizeHelper.ToHumanReadable(biggestFile.FileSize)} {biggestFile.Url}");
Session.SpeedtestResult = await Library.DoSpeedTestAsync(HttpClient, biggestFile.Url);
if (Session.SpeedtestResult != null)
{
Console.WriteLine($"Finished speedtest. Downloaded: {FileSizeHelper.ToHumanReadable(Session.SpeedtestResult.DownloadedBytes)}, Time: {Session.SpeedtestResult.ElapsedMiliseconds / 1000:F1} s, Speed: {Session.SpeedtestResult.MaxMBsPerSecond:F1} MB/s ({Session.SpeedtestResult.MaxMBsPerSecond * 8:F0} mbit)");
}
}
catch (Exception ex)
{
Logger.Error(ex, "Speedtest failed");
}
}
else
{
Logger.Warn($"Only a speedtest for HTTP(S), not '{Session.Root.Uri.Scheme}'");
}
}
}
Logger.Info("Logging sessions stats...");
try
{
string sessionStats = Statistics.GetSessionStats(Session, includeExtensions: true);
Logger.Info(sessionStats);
HistoryLogger.Info(sessionStats);
Logger.Info("Logged sessions stats");
if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoReddit)
{
// Also log to screen, when saving links or JSON fails and the logs keep filling by other sessions, this will be saved
Console.WriteLine(sessionStats);
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
if (Session.UrlsWithErrors.Any())
{
Logger.Info("URLs with errors:");
Console.WriteLine("URLs with errors:");
foreach (string urlWithError in Session.UrlsWithErrors.OrderBy(u => u))
{
Logger.Info(urlWithError);
Console.WriteLine(urlWithError);
}
}
if (OpenDirectoryIndexerSettings.CommandLineOptions.Json)
{
Logger.Info("Save session to JSON");
Console.WriteLine("Save session to JSON");
try
{
Library.SaveSessionJson(Session);
Logger.Info($"Saved session: {PathHelper.GetValidPath(Session.Root.Url)}.json");
Console.WriteLine($"Saved session: {PathHelper.GetValidPath(Session.Root.Url)}.json");
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
Logger.Info("Finished indexing!");
Console.WriteLine("Finished indexing!");
Program.SetConsoleTitle($"✔ {Program.ConsoleTitle}");
if (OpenDirectoryIndexerSettings.CommandLineOptions.Quit)
{
Command.KillApplication();
}
else
{
Console.WriteLine("Press ESC to exit! Or C to copy to clipboard and quit!");
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
});
}
private static void ShowStatistics(OpenDirectoryIndexer openDirectoryIndexer)
{
Console.WriteLine(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true));
Console.WriteLine($"Queue: {Library.FormatWithThousands(openDirectoryIndexer.WebDirectoriesQueue.Count)}, Threads: {openDirectoryIndexer.RunningWebDirectoryThreads}");
Console.WriteLine($"Queue (filesize): {Library.FormatWithThousands(openDirectoryIndexer.WebFilesFileSizeQueue.Count)}, Threads (filesize): {openDirectoryIndexer.RunningWebFileFileSizeThreads}");
}