internal static void ProcessConsoleInput(OpenDirectoryIndexer openDirectoryIndexer) { if (Console.IsInputRedirected) { string message = "Console input is redirect, maybe it is run inside another host. This could mean that no input will be send/processed."; Console.WriteLine(message); Logger.Warn(message); } while (true) { try { if (Console.IsInputRedirected) { int keyPressed = Console.Read(); //if (char.IsControl((char)keyPressed)) //{ // Console.WriteLine($"Pressed Console.Read(): {keyPressed}"); //} //else //{ // Console.WriteLine($"Pressed Console.Read(): {(char)keyPressed}"); //} switch (keyPressed) { case 'x': case 'X': KillApplication(); break; case 'i': ShowInfoAndCommands(); break; case 'c': if (OpenDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue) { new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true)); KillApplication(); } break; case 's': case 'S': ShowStatistics(openDirectoryIndexer); break; case 't': case 'T': ShowThreads(openDirectoryIndexer); break; case 'j': case 'J': SaveSession(openDirectoryIndexer); break; default: break; } } else { ConsoleKey keyPressed = Console.ReadKey(intercept: true).Key; //Console.WriteLine($"Pressed (Console.ReadKey(): {keyPressed}"); switch (keyPressed) { case ConsoleKey.X: case ConsoleKey.Escape: KillApplication(); break; case ConsoleKey.I: ShowInfoAndCommands(); break; case ConsoleKey.C: if (OpenDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue) { new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true)); KillApplication(); } break; case ConsoleKey.S: ShowStatistics(openDirectoryIndexer); break; case ConsoleKey.T: ShowThreads(openDirectoryIndexer); break; case ConsoleKey.J: SaveSession(openDirectoryIndexer); break; default: break; } } } catch (Exception ex) { Logger.Error(ex, "Error processing action"); throw; } } }
public async void StartIndexingAsync() { bool fromFile = !string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.FileName); if (fromFile) { Session = Library.LoadSessionJson(OpenDirectoryIndexerSettings.FileName); Console.WriteLine(Statistics.GetSessionStats(Session, includeExtensions: true)); Console.ReadKey(intercept: true); return; } else { Session = new Session { Started = DateTimeOffset.UtcNow, Root = new WebDirectory(parentWebDirectory: null) { Name = "ROOT", Url = OpenDirectoryIndexerSettings.Url } }; } if (Session.Root.Uri.Host == Constants.GoogleDriveDomain) { Logger.Warn("Google Drive scanning is limited to 9 directories per second!"); } if (Session.Root.Uri.Scheme == "ftp") { Logger.Warn("Retrieving FTP software!"); // TODO: Replace with library? Logger.Warn(await FtpParser.GetFtpServerInfo(Session.Root)); //AddProcessedWebDirectory(webDirectory, parsedWebDirectory); } TimerStatistics = new System.Timers.Timer { Enabled = true, Interval = TimeSpan.FromSeconds(30).TotalMilliseconds }; TimerStatistics.Elapsed += TimerStatistics_Elapsed; IndexingTask = Task.Run(async() => { try { WebDirectoriesQueue = new ConcurrentQueue <WebDirectory>(); if (fromFile) { SetParentDirectories(Session.Root); // TODO: Add unfinished items to queue, very complicated, we need to ALSO fill the ParentDirectory... //// With filter predicate, with selection function //var flatList = nodes.Flatten(n => n.IsDeleted == false, n => n.Children); //var directoriesToDo = Session.Root.Subdirectories.Flatten(null, wd => wd.Subdirectories).Where(wd => !wd.Finished); } else { // Add root WebDirectoriesQueue.Enqueue(Session.Root); } IndexingTaskCTS = new CancellationTokenSource(); for (int i = 1; i <= WebDirectoryProcessors.Length; i++) { string processorId = i.ToString(); WebDirectoryProcessors[i - 1] = WebDirectoryProcessor(WebDirectoriesQueue, $"Processor {processorId}", IndexingTaskCTS.Token); } for (int i = 1; i <= WebFileFileSizeProcessors.Length; i++) { string processorId = i.ToString(); WebFileFileSizeProcessors[i - 1] = WebFileFileSizeProcessor(WebFilesFileSizeQueue, $"Processor {processorId}", IndexingTaskCTS.Token, WebDirectoryProcessors); } await Task.WhenAll(WebDirectoryProcessors); Console.WriteLine("Finshed indexing"); Logger.Info("Finshed indexing"); if (Session.Root.Uri.Scheme == "ftp") { FtpParser.CloseAll(); } if (WebFilesFileSizeQueue.Any()) { TimerStatistics.Interval = TimeSpan.FromSeconds(5).TotalMilliseconds; Console.WriteLine($"Retrieving filesize of {WebFilesFileSizeQueue.Count} urls"); } await Task.WhenAll(WebFileFileSizeProcessors); TimerStatistics.Stop(); Session.Finished = DateTimeOffset.UtcNow; Session.TotalFiles = Session.Root.TotalFiles; Session.TotalFileSizeEstimated = Session.Root.TotalFileSize; if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoUrls && Session.Root.Uri.Host != Constants.GoogleDriveDomain) { if (Session.TotalFiles > 0) { Logger.Info("Saving URL list to file..."); Console.WriteLine("Saving URL list to file..."); string scansPath = Library.GetScansPath(); try { string fileUrls = string.Join(Environment.NewLine, Session.Root.AllFileUrls.Distinct()); string urlsFileName = $"{Library.CleanUriToFilename(Session.Root.Uri)}.txt"; string urlsPath = Path.Combine(scansPath, urlsFileName); Logger.Info("String joined"); File.WriteAllText(urlsPath, fileUrls); Logger.Info($"Saved URL list to file: {urlsFileName}"); Console.WriteLine($"Saved URL list to file: {urlsFileName}"); if (OpenDirectoryIndexerSettings.CommandLineOptions.UploadUrls && Session.TotalFiles > 0) { Console.WriteLine("Uploading URLs..."); //UploadFilesFile uploadFilesFile = await UploadFileIo.UploadFile(HttpClient, urlsPath); //HistoryLogger.Info($"uploadfiles.io: {JsonConvert.SerializeObject(uploadFilesFile)}"); //Session.UploadedUrlsUrl = uploadFilesFile.Url.ToString(); GoFilesFile uploadedFile = await GoFileIo.UploadFile(HttpClient, urlsPath); HistoryLogger.Info($"goFile.io: {JsonConvert.SerializeObject(uploadedFile)}"); Session.UploadedUrlsUrl = uploadedFile.Url.ToString(); Console.WriteLine($"Uploaded URLs: {Session.UploadedUrlsUrl}"); } } catch (Exception ex) { Logger.Error(ex); } } else { Logger.Info("No URLs to save"); Console.WriteLine("No URLs to save"); } } if (OpenDirectoryIndexerSettings.CommandLineOptions.Speedtest && Session.Root.Uri.Host != Constants.GoogleDriveDomain) { if (Session.TotalFiles > 0) { if (Session.Root.Uri.Scheme == "https" || Session.Root.Uri.Scheme == "http") { try { WebFile biggestFile = Session.Root.AllFiles.OrderByDescending(f => f.FileSize).First(); Console.WriteLine($"Starting speedtest (10-25 seconds)..."); Console.WriteLine($"Test file: {FileSizeHelper.ToHumanReadable(biggestFile.FileSize)} {biggestFile.Url}"); Session.SpeedtestResult = await Library.DoSpeedTestAsync(HttpClient, biggestFile.Url); if (Session.SpeedtestResult != null) { Console.WriteLine($"Finished speedtest. Downloaded: {FileSizeHelper.ToHumanReadable(Session.SpeedtestResult.DownloadedBytes)}, Time: {Session.SpeedtestResult.ElapsedMiliseconds / 1000:F1} s, Speed: {Session.SpeedtestResult.MaxMBsPerSecond:F1} MB/s ({Session.SpeedtestResult.MaxMBsPerSecond * 8:F0} mbit)"); } } catch (Exception ex) { Logger.Error(ex, "Speedtest failed"); } } else { Logger.Warn($"Only a speedtest for HTTP(S), not '{Session.Root.Uri.Scheme}'"); } } } Logger.Info("Logging sessions stats..."); try { string sessionStats = Statistics.GetSessionStats(Session, includeExtensions: true); Logger.Info(sessionStats); HistoryLogger.Info(sessionStats); Logger.Info("Logged sessions stats"); if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoReddit) { // Also log to screen, when saving links or JSON fails and the logs keep filling by other sessions, this will be saved Console.WriteLine(sessionStats); } } catch (Exception ex) { Logger.Error(ex); } if (Session.UrlsWithErrors.Any()) { Logger.Info("URLs with errors:"); Console.WriteLine("URLs with errors:"); foreach (string urlWithError in Session.UrlsWithErrors.OrderBy(u => u)) { Logger.Info(urlWithError); Console.WriteLine(urlWithError); } } if (OpenDirectoryIndexerSettings.CommandLineOptions.Json) { Logger.Info("Save session to JSON"); Console.WriteLine("Save session to JSON"); try { Library.SaveSessionJson(Session); Logger.Info($"Saved session: {PathHelper.GetValidPath(Session.Root.Url)}.json"); Console.WriteLine($"Saved session: {PathHelper.GetValidPath(Session.Root.Url)}.json"); } catch (Exception ex) { Logger.Error(ex); } } Logger.Info("Finished indexing!"); Console.WriteLine("Finished indexing!"); Program.SetConsoleTitle($"✔ {Program.ConsoleTitle}"); if (OpenDirectoryIndexerSettings.CommandLineOptions.Quit) { Command.KillApplication(); } else { Console.WriteLine("Press ESC to exit! Or C to copy to clipboard and quit!"); } } catch (Exception ex) { Logger.Error(ex); } }); }
private static void ShowStatistics(OpenDirectoryIndexer openDirectoryIndexer) { Console.WriteLine(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true)); Console.WriteLine($"Queue: {Library.FormatWithThousands(openDirectoryIndexer.WebDirectoriesQueue.Count)}, Threads: {openDirectoryIndexer.RunningWebDirectoryThreads}"); Console.WriteLine($"Queue (filesize): {Library.FormatWithThousands(openDirectoryIndexer.WebFilesFileSizeQueue.Count)}, Threads (filesize): {openDirectoryIndexer.RunningWebFileFileSizeThreads}"); }