Exemplo n.º 1
0
        public OpenDirectoryIndexer(OpenDirectoryIndexerSettings openDirectoryIndexerSettings)
        {
            OpenDirectoryIndexerSettings = openDirectoryIndexerSettings;

            HttpClientHandler = new HttpClientHandler
            {
                ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true,
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli
            };

            HttpClient = new HttpClient(HttpClientHandler)
            {
                Timeout = TimeSpan.FromSeconds(OpenDirectoryIndexerSettings.Timeout)
            };

            HttpClient.DefaultRequestHeaders.AcceptEncoding.ParseAdd("gzip, deflate, br");

            if (!string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.Username) && !string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.Password))
            {
                HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{OpenDirectoryIndexerSettings.Username}:{OpenDirectoryIndexerSettings.Password}")));
            }

            // Fix encoding issue with "windows-1251"
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);

            WebDirectoryProcessors    = new Task[OpenDirectoryIndexerSettings.Threads];
            WebFileFileSizeProcessors = new Task[OpenDirectoryIndexerSettings.Threads];

            //HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Curl);
            //HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Chrome);
        }
        public static string GetOutputFullPath(Session session, OpenDirectoryIndexerSettings openDirectoryIndexerSettings, string extension)
        {
            string fileName = openDirectoryIndexerSettings.CommandLineOptions.OutputFile is not null ? $"{openDirectoryIndexerSettings.CommandLineOptions.OutputFile}.{extension}" : $"{CleanUriToFilename(session.Root.Uri)}.{extension}";

            string path;

            if (Path.IsPathFullyQualified(fileName))
            {
                path = fileName;
            }
            else
            {
                string scansPath = GetScansPath();
                path = Path.Combine(scansPath, fileName);
            }

            return(path);
        }
        public OpenDirectoryIndexer(OpenDirectoryIndexerSettings openDirectoryIndexerSettings)
        {
            OpenDirectoryIndexerSettings = openDirectoryIndexerSettings;

            HttpClientHandler = new HttpClientHandler
            {
                ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true
            };

            HttpClient = new HttpClient(HttpClientHandler)
            {
                Timeout = TimeSpan.FromMinutes(5)
            };

            // Fix encoding issue with "windows-1251"
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);

            WebDirectoryProcessors    = new Task[OpenDirectoryIndexerSettings.Threads];
            WebFileFileSizeProcessors = new Task[OpenDirectoryIndexerSettings.Threads];

            //HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Curl);
            //HttpClient.DefaultRequestHeaders.Add("User-Agent", Constants.UserAgent.Chrome);
        }
Exemplo n.º 4
0
        static async Task <int> Main(string[] args)
        {
            SetConsoleTitle("OpenDirectoryDownloader");

            Stream nlogConfigFile = Library.GetEmbeddedResourceStream(Assembly.GetEntryAssembly(), "NLog.config");

            if (nlogConfigFile != null)
            {
                XmlReader xmlReader = XmlReader.Create(nlogConfigFile);
                LogManager.Configuration = new XmlLoggingConfiguration(xmlReader, null);
            }

            Process currentProcess = Process.GetCurrentProcess();

            Console.WriteLine($"Started with PID {currentProcess.Id}");
            Logger.Info($"Started with PID {currentProcess.Id}");

            Thread.CurrentThread.Name = "Main thread";

            Parser.Default.ParseArguments <CommandLineOptions>(args)
            .WithNotParsed(o =>
            {
                List <Error> errors = o.ToList();

                if (errors.Any())
                {
                    foreach (Error error in errors)
                    {
                        Console.WriteLine($"Error command line parameter '{error.Tag}'");
                    }
                }
            })
            .WithParsed(o => CommandLineOptions = o);

            if (CommandLineOptions.Threads < 1 || CommandLineOptions.Threads > 100)
            {
                Console.WriteLine("Threads must be between 1 and 100");
                return(1);
            }

            string url = CommandLineOptions.Url;

            if (string.IsNullOrWhiteSpace(url))
            {
                Console.WriteLine("Which URL do you want to index?");
                url = Console.ReadLine();
            }

            // Wait until this ticket is closed: https://github.com/dotnet/corefx/pull/37050
            //AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2Support", true);

            OpenDirectoryIndexerSettings openDirectoryIndexerSettings = new OpenDirectoryIndexerSettings
            {
                CommandLineOptions = CommandLineOptions
            };

            if (File.Exists(url))
            {
                openDirectoryIndexerSettings.FileName = url;
            }
            else
            {
                Console.WriteLine($"URL specified: {url}");

                string newUrl = Library.FixUrl(url);

                if (newUrl != url)
                {
                    Console.WriteLine($"URL fixed    : {newUrl}");
                }

                openDirectoryIndexerSettings.Url = newUrl;
            }

            openDirectoryIndexerSettings.Threads  = openDirectoryIndexerSettings.CommandLineOptions.Threads;
            openDirectoryIndexerSettings.Timeout  = openDirectoryIndexerSettings.CommandLineOptions.Timeout;
            openDirectoryIndexerSettings.Username = openDirectoryIndexerSettings.CommandLineOptions.Username;
            openDirectoryIndexerSettings.Password = openDirectoryIndexerSettings.CommandLineOptions.Password;

            // FTP
            if (openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftp) == true || openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftps) == true)
            {
                openDirectoryIndexerSettings.Threads = 6;
            }

            OpenDirectoryIndexer openDirectoryIndexer = new OpenDirectoryIndexer(openDirectoryIndexerSettings);

            SetConsoleTitle($"{new Uri(url).Host.Replace("www.", string.Empty)} - {ConsoleTitle}");

            openDirectoryIndexer.StartIndexingAsync();
            Console.WriteLine("Started indexing!");

            Command.ShowInfoAndCommands();
            Command.ProcessConsoleInput(openDirectoryIndexer);

            await openDirectoryIndexer.IndexingTask;

            if (!CommandLineOptions.Quit)
            {
                Console.WriteLine("Press ESC to exit");
                Console.ReadKey();
            }

            return(0);
        }
Exemplo n.º 5
0
        static async Task <int> Main(string[] args)
        {
            SetConsoleTitle("OpenDirectoryDownloader");

            Console.WriteLine("Started");
            Logger.Info("Started");

            Parser.Default.ParseArguments <CommandLineOptions>(args)
            .WithNotParsed(o =>
            {
                List <Error> errors = o.ToList();

                if (errors.Any())
                {
                    foreach (Error error in errors)
                    {
                        Console.WriteLine($"Error command line parameter '{error.Tag}'");
                    }
                }
            })
            .WithParsed(o => CommandLineOptions = o);

            if (CommandLineOptions.Threads < 1 || CommandLineOptions.Threads > 100)
            {
                Console.WriteLine("Threads must be between 1 and 100");
                return(1);
            }

            string url = CommandLineOptions.Url;

            if (string.IsNullOrWhiteSpace(url))
            {
                Console.WriteLine("Which URL do you want to index?");
                url = Console.ReadLine();
            }

            // Wait until this ticket is closed: https://github.com/dotnet/corefx/pull/37050
            //AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2Support", true);

            OpenDirectoryIndexerSettings openDirectoryIndexerSettings = new OpenDirectoryIndexerSettings
            {
                CommandLineOptions = CommandLineOptions
            };

            if (File.Exists(url))
            {
                openDirectoryIndexerSettings.FileName = url;
            }
            else
            {
                Console.WriteLine($"URL specified: {url}");

                url = Library.FixUrl(url);
                Console.WriteLine($"URL fixed: {url}");

                openDirectoryIndexerSettings.Url = url;
            }

            openDirectoryIndexerSettings.Threads = openDirectoryIndexerSettings.CommandLineOptions.Threads;

            // FTP
            // TODO: Make dynamic
            if (openDirectoryIndexerSettings.Url?.StartsWith("ftp") == true)
            {
                openDirectoryIndexerSettings.Threads = 6;
            }

            OpenDirectoryIndexer openDirectoryIndexer = new OpenDirectoryIndexer(openDirectoryIndexerSettings);

            SetConsoleTitle($"{new Uri(url).Host.Replace("www.", string.Empty)} - {ConsoleTitle}");

            openDirectoryIndexer.StartIndexingAsync();
            Console.WriteLine("Started indexing!");

            Command.ShowInfoAndCommands();
            Command.ProcessConsoleInput(openDirectoryIndexer);

            await openDirectoryIndexer.IndexingTask;

            if (!CommandLineOptions.Quit)
            {
                Console.WriteLine("Press ESC to exit");
                Console.ReadKey();
            }

            return(0);
        }