public OpenDirectoryIndexer(OpenDirectoryIndexerSettings openDirectoryIndexerSettings)
{
OpenDirectoryIndexerSettings = openDirectoryIndexerSettings;
HttpClientHandler = new HttpClientHandler
{
ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli
};
HttpClient = new HttpClient(HttpClientHandler)
{
Timeout = TimeSpan.FromSeconds(OpenDirectoryIndexerSettings.Timeout)
};
HttpClient.DefaultRequestHeaders.AcceptEncoding.ParseAdd("gzip, deflate, br");
if (!string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.Username) && !string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.Password))
{
HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{OpenDirectoryIndexerSettings.Username}:{OpenDirectoryIndexerSettings.Password}")));
}
// Fix encoding issue with "windows-1251"
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
WebDirectoryProcessors = new Task[OpenDirectoryIndexerSettings.Threads];
WebFileFileSizeProcessors = new Task[OpenDirectoryIndexerSettings.Threads];
//HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Curl);
//HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Chrome);
}
public static string GetOutputFullPath(Session session, OpenDirectoryIndexerSettings openDirectoryIndexerSettings, string extension)
{
string fileName = openDirectoryIndexerSettings.CommandLineOptions.OutputFile is not null ? $"{openDirectoryIndexerSettings.CommandLineOptions.OutputFile}.{extension}" : $"{CleanUriToFilename(session.Root.Uri)}.{extension}";
string path;
if (Path.IsPathFullyQualified(fileName))
{
path = fileName;
}
else
{
string scansPath = GetScansPath();
path = Path.Combine(scansPath, fileName);
}
return(path);
}
public OpenDirectoryIndexer(OpenDirectoryIndexerSettings openDirectoryIndexerSettings)
{
OpenDirectoryIndexerSettings = openDirectoryIndexerSettings;
HttpClientHandler = new HttpClientHandler
{
ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true
};
HttpClient = new HttpClient(HttpClientHandler)
{
Timeout = TimeSpan.FromMinutes(5)
};
// Fix encoding issue with "windows-1251"
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
WebDirectoryProcessors = new Task[OpenDirectoryIndexerSettings.Threads];
WebFileFileSizeProcessors = new Task[OpenDirectoryIndexerSettings.Threads];
//HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Curl);
//HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Chrome);
}
static async Task <int> Main(string[] args)
{
SetConsoleTitle("OpenDirectoryDownloader");
Stream nlogConfigFile = Library.GetEmbeddedResourceStream(Assembly.GetEntryAssembly(), "NLog.config");
if (nlogConfigFile != null)
{
XmlReader xmlReader = XmlReader.Create(nlogConfigFile);
LogManager.Configuration = new XmlLoggingConfiguration(xmlReader, null);
}
Process currentProcess = Process.GetCurrentProcess();
Console.WriteLine($"Started with PID {currentProcess.Id}");
Logger.Info($"Started with PID {currentProcess.Id}");
Thread.CurrentThread.Name = "Main thread";
Parser.Default.ParseArguments <CommandLineOptions>(args)
.WithNotParsed(o =>
{
List <Error> errors = o.ToList();
if (errors.Any())
{
foreach (Error error in errors)
{
Console.WriteLine($"Error command line parameter '{error.Tag}'");
}
}
})
.WithParsed(o => CommandLineOptions = o);
if (CommandLineOptions.Threads < 1 || CommandLineOptions.Threads > 100)
{
Console.WriteLine("Threads must be between 1 and 100");
return(1);
}
string url = CommandLineOptions.Url;
if (string.IsNullOrWhiteSpace(url))
{
Console.WriteLine("Which URL do you want to index?");
url = Console.ReadLine();
}
// Wait until this ticket is closed: https://github.com/dotnet/corefx/pull/37050
//AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2Support", true);
OpenDirectoryIndexerSettings openDirectoryIndexerSettings = new OpenDirectoryIndexerSettings
{
CommandLineOptions = CommandLineOptions
};
if (File.Exists(url))
{
openDirectoryIndexerSettings.FileName = url;
}
else
{
Console.WriteLine($"URL specified: {url}");
string newUrl = Library.FixUrl(url);
if (newUrl != url)
{
Console.WriteLine($"URL fixed : {newUrl}");
}
openDirectoryIndexerSettings.Url = newUrl;
}
openDirectoryIndexerSettings.Threads = openDirectoryIndexerSettings.CommandLineOptions.Threads;
openDirectoryIndexerSettings.Timeout = openDirectoryIndexerSettings.CommandLineOptions.Timeout;
openDirectoryIndexerSettings.Username = openDirectoryIndexerSettings.CommandLineOptions.Username;
openDirectoryIndexerSettings.Password = openDirectoryIndexerSettings.CommandLineOptions.Password;
// FTP
if (openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftp) == true || openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftps) == true)
{
openDirectoryIndexerSettings.Threads = 6;
}
OpenDirectoryIndexer openDirectoryIndexer = new OpenDirectoryIndexer(openDirectoryIndexerSettings);
SetConsoleTitle($"{new Uri(url).Host.Replace("www.", string.Empty)} - {ConsoleTitle}");
openDirectoryIndexer.StartIndexingAsync();
Console.WriteLine("Started indexing!");
Command.ShowInfoAndCommands();
Command.ProcessConsoleInput(openDirectoryIndexer);
await openDirectoryIndexer.IndexingTask;
if (!CommandLineOptions.Quit)
{
Console.WriteLine("Press ESC to exit");
Console.ReadKey();
}
return(0);
}
static async Task <int> Main(string[] args)
{
SetConsoleTitle("OpenDirectoryDownloader");
Console.WriteLine("Started");
Logger.Info("Started");
Parser.Default.ParseArguments <CommandLineOptions>(args)
.WithNotParsed(o =>
{
List <Error> errors = o.ToList();
if (errors.Any())
{
foreach (Error error in errors)
{
Console.WriteLine($"Error command line parameter '{error.Tag}'");
}
}
})
.WithParsed(o => CommandLineOptions = o);
if (CommandLineOptions.Threads < 1 || CommandLineOptions.Threads > 100)
{
Console.WriteLine("Threads must be between 1 and 100");
return(1);
}
string url = CommandLineOptions.Url;
if (string.IsNullOrWhiteSpace(url))
{
Console.WriteLine("Which URL do you want to index?");
url = Console.ReadLine();
}
// Wait until this ticket is closed: https://github.com/dotnet/corefx/pull/37050
//AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2Support", true);
OpenDirectoryIndexerSettings openDirectoryIndexerSettings = new OpenDirectoryIndexerSettings
{
CommandLineOptions = CommandLineOptions
};
if (File.Exists(url))
{
openDirectoryIndexerSettings.FileName = url;
}
else
{
Console.WriteLine($"URL specified: {url}");
url = Library.FixUrl(url);
Console.WriteLine($"URL fixed: {url}");
openDirectoryIndexerSettings.Url = url;
}
openDirectoryIndexerSettings.Threads = openDirectoryIndexerSettings.CommandLineOptions.Threads;
// FTP
// TODO: Make dynamic
if (openDirectoryIndexerSettings.Url?.StartsWith("ftp") == true)
{
openDirectoryIndexerSettings.Threads = 6;
}
OpenDirectoryIndexer openDirectoryIndexer = new OpenDirectoryIndexer(openDirectoryIndexerSettings);
SetConsoleTitle($"{new Uri(url).Host.Replace("www.", string.Empty)} - {ConsoleTitle}");
openDirectoryIndexer.StartIndexingAsync();
Console.WriteLine("Started indexing!");
Command.ShowInfoAndCommands();
Command.ProcessConsoleInput(openDirectoryIndexer);
await openDirectoryIndexer.IndexingTask;
if (!CommandLineOptions.Quit)
{
Console.WriteLine("Press ESC to exit");
Console.ReadKey();
}
return(0);
}