private static void ShowThreads(OpenDirectoryIndexer openDirectoryIndexer)
{
Console.WriteLine($"Running threads:");
lock (openDirectoryIndexer.WebDirectoryProcessorInfoLock)
{
foreach (KeyValuePair <string, WebDirectory> webDirectory in openDirectoryIndexer.WebDirectoryProcessorInfo.OrderBy(i => i.Key))
{
Console.WriteLine($"[{webDirectory.Key}] {Library.FormatWithThousands((DateTimeOffset.UtcNow - webDirectory.Value.StartTime).TotalMilliseconds)}ms | {webDirectory.Value.Url}");
}
}
}
private static void SaveSession(OpenDirectoryIndexer openDirectoryIndexer)
{
try
{
Console.WriteLine("Saving session to JSON");
Library.SaveSessionJson(openDirectoryIndexer.Session);
Console.WriteLine("Saved session to JSON");
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
internal static void ProcessConsoleInput(OpenDirectoryIndexer openDirectoryIndexer)
{
while (true)
{
try
{
switch (Console.ReadKey(true).Key)
{
case ConsoleKey.Escape:
KillApplication();
break;
case ConsoleKey.I:
ShowInfoAndCommands();
break;
case ConsoleKey.C:
if (openDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue)
{
Clipboard.SetText(Statistics.GetSessionStats(openDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true));
KillApplication();
}
break;
case ConsoleKey.S:
ShowStatistics(openDirectoryIndexer);
break;
case ConsoleKey.T:
ShowThreads(openDirectoryIndexer);
break;
case ConsoleKey.J:
SaveSession(openDirectoryIndexer);
break;
default:
break;
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error processing action");
}
}
}
private static void SaveSession(OpenDirectoryIndexer openDirectoryIndexer)
{
try
{
string jsonPath = Library.GetOutputFullPath(OpenDirectoryIndexer.Session, openDirectoryIndexer.OpenDirectoryIndexerSettings, "json");
Logger.Info("Saving session to JSON..");
Console.WriteLine("Saving session to JSON..");
Library.SaveSessionJson(OpenDirectoryIndexer.Session, jsonPath);
Logger.Info($"Saved session to JSON: {jsonPath}");
Console.WriteLine($"Saved session to JSON: {jsonPath}");
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
private static void SaveSessionText(OpenDirectoryIndexer openDirectoryIndexer)
{
try
{
Logger.Info("Saving URL list to file..");
Console.WriteLine("Saving URL list to file..");
IEnumerable <string> distinctUrls = OpenDirectoryIndexer.Session.Root.AllFileUrls.Distinct();
string urlsPath = Library.GetOutputFullPath(OpenDirectoryIndexer.Session, openDirectoryIndexer.OpenDirectoryIndexerSettings, "txt");
File.WriteAllLines(urlsPath, distinctUrls);
Logger.Info($"Saved URL list to file: {urlsPath}");
Console.WriteLine($"Saved URL list to file: {urlsPath}");
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
private static void SaveSessionText(OpenDirectoryIndexer openDirectoryIndexer)
{
try
{
Logger.Info("Saving URL list to file...");
Console.WriteLine("Saving URL list to file...");
IEnumerable <string> distinctUrls = OpenDirectoryIndexer.Session.Root.AllFileUrls.Distinct();
string scansPath = Library.GetScansPath();
string urlsFileName = $"{Library.CleanUriToFilename(OpenDirectoryIndexer.Session.Root.Uri)}.txt";
string urlsPath = Path.Combine(scansPath, urlsFileName);
File.WriteAllLines(urlsPath, distinctUrls);
Logger.Info($"Saved URL list to file: {urlsFileName}");
Console.WriteLine($"Saved URL list to file: {urlsFileName}");
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
internal static void ProcessConsoleInput(OpenDirectoryIndexer openDirectoryIndexer)
{
OpenDirectoryIndexer = openDirectoryIndexer;
while (true)
{
try
{
switch (Console.ReadKey(true).Key)
{
case ConsoleKey.Escape:
KillApplication();
break;
case ConsoleKey.I:
ShowInfoAndCommands();
break;
case ConsoleKey.S:
ShowStatistics(openDirectoryIndexer);
break;
case ConsoleKey.T:
ShowThreads(openDirectoryIndexer);
break;
case ConsoleKey.J:
SaveSession(openDirectoryIndexer);
break;
default:
break;
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error processing action");
}
}
}
private static void ShowStatistics(OpenDirectoryIndexer openDirectoryIndexer)
{
Console.WriteLine(Statistics.GetSessionStats(openDirectoryIndexer.Session, includeExtensions: true));
Console.WriteLine($"Queue: {Library.FormatWithThousands(openDirectoryIndexer.WebDirectoriesQueue.Count)}, Threads: {openDirectoryIndexer.RunningWebDirectoryThreads}");
Console.WriteLine($"Queue (filesize): {Library.FormatWithThousands(openDirectoryIndexer.WebFilesFileSizeQueue.Count)}, Threads (filesize): {openDirectoryIndexer.RunningWebFileFileSizeThreads}");
}
static async Task <int> Main(string[] args)
{
SetConsoleTitle("OpenDirectoryDownloader");
Stream nlogConfigFile = Library.GetEmbeddedResourceStream(Assembly.GetEntryAssembly(), "NLog.config");
if (nlogConfigFile != null)
{
XmlReader xmlReader = XmlReader.Create(nlogConfigFile);
LogManager.Configuration = new XmlLoggingConfiguration(xmlReader, null);
}
Process currentProcess = Process.GetCurrentProcess();
Console.WriteLine($"Started with PID {currentProcess.Id}");
Logger.Info($"Started with PID {currentProcess.Id}");
Thread.CurrentThread.Name = "Main thread";
Parser.Default.ParseArguments <CommandLineOptions>(args)
.WithNotParsed(o =>
{
List <Error> errors = o.ToList();
if (errors.Any())
{
foreach (Error error in errors)
{
Console.WriteLine($"Error command line parameter '{error.Tag}'");
}
}
})
.WithParsed(o => CommandLineOptions = o);
if (CommandLineOptions.Threads < 1 || CommandLineOptions.Threads > 100)
{
Console.WriteLine("Threads must be between 1 and 100");
return(1);
}
string url = CommandLineOptions.Url;
if (string.IsNullOrWhiteSpace(url))
{
Console.WriteLine("Which URL do you want to index?");
url = Console.ReadLine();
}
// Wait until this ticket is closed: https://github.com/dotnet/corefx/pull/37050
//AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2Support", true);
OpenDirectoryIndexerSettings openDirectoryIndexerSettings = new OpenDirectoryIndexerSettings
{
CommandLineOptions = CommandLineOptions
};
if (File.Exists(url))
{
openDirectoryIndexerSettings.FileName = url;
}
else
{
Console.WriteLine($"URL specified: {url}");
string newUrl = Library.FixUrl(url);
if (newUrl != url)
{
Console.WriteLine($"URL fixed : {newUrl}");
}
openDirectoryIndexerSettings.Url = newUrl;
}
openDirectoryIndexerSettings.Threads = openDirectoryIndexerSettings.CommandLineOptions.Threads;
openDirectoryIndexerSettings.Timeout = openDirectoryIndexerSettings.CommandLineOptions.Timeout;
openDirectoryIndexerSettings.Username = openDirectoryIndexerSettings.CommandLineOptions.Username;
openDirectoryIndexerSettings.Password = openDirectoryIndexerSettings.CommandLineOptions.Password;
// FTP
if (openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftp) == true || openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftps) == true)
{
openDirectoryIndexerSettings.Threads = 6;
}
OpenDirectoryIndexer openDirectoryIndexer = new OpenDirectoryIndexer(openDirectoryIndexerSettings);
SetConsoleTitle($"{new Uri(url).Host.Replace("www.", string.Empty)} - {ConsoleTitle}");
openDirectoryIndexer.StartIndexingAsync();
Console.WriteLine("Started indexing!");
Command.ShowInfoAndCommands();
Command.ProcessConsoleInput(openDirectoryIndexer);
await openDirectoryIndexer.IndexingTask;
if (!CommandLineOptions.Quit)
{
Console.WriteLine("Press ESC to exit");
Console.ReadKey();
}
return(0);
}
static async Task <int> Main(string[] args)
{
SetConsoleTitle("OpenDirectoryDownloader");
Console.WriteLine("Started");
Logger.Info("Started");
Parser.Default.ParseArguments <CommandLineOptions>(args)
.WithNotParsed(o =>
{
List <Error> errors = o.ToList();
if (errors.Any())
{
foreach (Error error in errors)
{
Console.WriteLine($"Error command line parameter '{error.Tag}'");
}
}
})
.WithParsed(o => CommandLineOptions = o);
if (CommandLineOptions.Threads < 1 || CommandLineOptions.Threads > 100)
{
Console.WriteLine("Threads must be between 1 and 100");
return(1);
}
string url = CommandLineOptions.Url;
if (string.IsNullOrWhiteSpace(url))
{
Console.WriteLine("Which URL do you want to index?");
url = Console.ReadLine();
}
// Wait until this ticket is closed: https://github.com/dotnet/corefx/pull/37050
//AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2Support", true);
OpenDirectoryIndexerSettings openDirectoryIndexerSettings = new OpenDirectoryIndexerSettings
{
CommandLineOptions = CommandLineOptions
};
if (File.Exists(url))
{
openDirectoryIndexerSettings.FileName = url;
}
else
{
Console.WriteLine($"URL specified: {url}");
url = Library.FixUrl(url);
Console.WriteLine($"URL fixed: {url}");
openDirectoryIndexerSettings.Url = url;
}
openDirectoryIndexerSettings.Threads = openDirectoryIndexerSettings.CommandLineOptions.Threads;
// FTP
// TODO: Make dynamic
if (openDirectoryIndexerSettings.Url?.StartsWith("ftp") == true)
{
openDirectoryIndexerSettings.Threads = 6;
}
OpenDirectoryIndexer openDirectoryIndexer = new OpenDirectoryIndexer(openDirectoryIndexerSettings);
SetConsoleTitle($"{new Uri(url).Host.Replace("www.", string.Empty)} - {ConsoleTitle}");
openDirectoryIndexer.StartIndexingAsync();
Console.WriteLine("Started indexing!");
Command.ShowInfoAndCommands();
Command.ProcessConsoleInput(openDirectoryIndexer);
await openDirectoryIndexer.IndexingTask;
if (!CommandLineOptions.Quit)
{
Console.WriteLine("Press ESC to exit");
Console.ReadKey();
}
return(0);
}
internal static void ProcessConsoleInput(OpenDirectoryIndexer openDirectoryIndexer)
{
if (Console.IsInputRedirected)
{
string message = "Console input is redirect, maybe it is run inside another host. This could mean that no input will be send/processed.";
Console.WriteLine(message);
Logger.Warn(message);
}
while (true)
{
try
{
if (Console.IsInputRedirected)
{
int keyPressed = Console.Read();
if (keyPressed == -1)
{
// Needed, when input is redirected it will immediately return with -1
Task.Delay(10).Wait();
}
//if (char.IsControl((char)keyPressed))
//{
// Console.WriteLine($"Pressed Console.Read(): {keyPressed}");
//}
//else
//{
// Console.WriteLine($"Pressed Console.Read(): {(char)keyPressed}");
//}
switch (keyPressed)
{
case 'x':
case 'X':
KillApplication();
break;
case 'i':
ShowInfoAndCommands();
break;
case 'c':
if (OpenDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue)
{
new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true));
KillApplication();
}
break;
case 's':
case 'S':
ShowStatistics(openDirectoryIndexer);
break;
case 't':
case 'T':
ShowThreads(openDirectoryIndexer);
break;
case 'j':
case 'J':
SaveSession(openDirectoryIndexer);
break;
default:
break;
}
}
else
{
if (!Console.KeyAvailable)
{
continue;
}
ConsoleKey keyPressed = Console.ReadKey(intercept: true).Key;
//Console.WriteLine($"Pressed (Console.ReadKey(): {keyPressed}");
switch (keyPressed)
{
case ConsoleKey.X:
case ConsoleKey.Escape:
KillApplication();
break;
case ConsoleKey.I:
ShowInfoAndCommands();
break;
case ConsoleKey.C:
if (OpenDirectoryIndexer.Session.Finished != DateTimeOffset.MinValue)
{
try
{
new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true));
}
catch (Exception ex)
{
Logger.Error($"Error copying stats to clipboard: {ex.Message}");
}
KillApplication();
}
break;
case ConsoleKey.S:
ShowStatistics(openDirectoryIndexer);
break;
case ConsoleKey.T:
ShowThreads(openDirectoryIndexer);
break;
case ConsoleKey.J:
SaveSession(openDirectoryIndexer);
break;
default:
break;
}
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error processing action");
throw;
}
}
}