Example #1
0
        public async void StartIndexingAsync()
        {
            bool fromFile = !string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.FileName);

            if (fromFile)
            {
                Session = Library.LoadSessionJson(OpenDirectoryIndexerSettings.FileName);
                Console.WriteLine(Statistics.GetSessionStats(Session, includeExtensions: true));
                Console.ReadKey(intercept: true);
                return;
            }
            else
            {
                Session = new Session
                {
                    Started = DateTimeOffset.UtcNow,
                    Root    = new WebDirectory(parentWebDirectory: null)
                    {
                        Name = Constants.Root,
                        Url  = OpenDirectoryIndexerSettings.Url
                    },
                    MaxThreads = OpenDirectoryIndexerSettings.Threads
                };
            }

            Session.MaxThreads = OpenDirectoryIndexerSettings.Threads;

            if (Session.Root.Uri.Host == Constants.GoogleDriveDomain)
            {
                Logger.Warn("Google Drive scanning is limited to 9 directories per second!");
            }

            if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
            {
                Logger.Warn("Retrieving FTP(S) software!");

                if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
                {
                    if (Session.Root.Uri.Port == -1)
                    {
                        Logger.Warn("Using default port (990) for FTPS");

                        UriBuilder uriBuilder = new UriBuilder(Session.Root.Uri)
                        {
                            Port = 990
                        };

                        Session.Root.Url = uriBuilder.Uri.ToString();
                    }
                }

                string serverInfo = await FtpParser.GetFtpServerInfo(Session.Root, OpenDirectoryIndexerSettings.Username, OpenDirectoryIndexerSettings.Password);

                if (string.IsNullOrWhiteSpace(serverInfo))
                {
                    serverInfo = "Failed or no server info available.";
                }
                else
                {
                    // Remove IP from server info
                    Regex.Replace(serverInfo, @"(Connected to )(\d*\.\d*.\d*.\d*)", "$1IP Address");

                    Session.Description = $"FTP INFO{Environment.NewLine}{serverInfo}";
                }

                Logger.Warn(serverInfo);
            }

            TimerStatistics = new System.Timers.Timer
            {
                Enabled  = true,
                Interval = TimeSpan.FromSeconds(30).TotalMilliseconds
            };

            TimerStatistics.Elapsed += TimerStatistics_Elapsed;

            IndexingTask = Task.Run(async() =>
            {
                try
                {
                    WebDirectoriesQueue = new ConcurrentQueue <WebDirectory>();

                    if (fromFile)
                    {
                        SetParentDirectories(Session.Root);

                        // TODO: Add unfinished items to queue, very complicated, we need to ALSO fill the ParentDirectory...
                        //// With filter predicate, with selection function
                        //var flatList = nodes.Flatten(n => n.IsDeleted == false, n => n.Children);
                        //var directoriesToDo = Session.Root.Subdirectories.Flatten(null, wd => wd.Subdirectories).Where(wd => !wd.Finished);
                    }
                    else
                    {
                        // Add root
                        WebDirectoriesQueue.Enqueue(Session.Root);
                    }

                    IndexingTaskCTS = new CancellationTokenSource();

                    for (int i = 1; i <= WebDirectoryProcessors.Length; i++)
                    {
                        string processorId = i.ToString();

                        WebDirectoryProcessors[i - 1] = WebDirectoryProcessor(WebDirectoriesQueue, $"Processor {processorId}", IndexingTaskCTS.Token);
                    }

                    for (int i = 1; i <= WebFileFileSizeProcessors.Length; i++)
                    {
                        string processorId = i.ToString();

                        WebFileFileSizeProcessors[i - 1] = WebFileFileSizeProcessor(WebFilesFileSizeQueue, $"Processor {processorId}", WebDirectoryProcessors, IndexingTaskCTS.Token);
                    }

                    await Task.WhenAll(WebDirectoryProcessors);
                    Console.WriteLine("Finshed indexing");
                    Logger.Info("Finshed indexing");

                    if (WebFilesFileSizeQueue.Any())
                    {
                        TimerStatistics.Interval = TimeSpan.FromSeconds(5).TotalMilliseconds;
                        Console.WriteLine($"Retrieving filesize of {WebFilesFileSizeQueue.Count} urls");
                    }

                    await Task.WhenAll(WebFileFileSizeProcessors);

                    TimerStatistics.Stop();

                    Session.Finished               = DateTimeOffset.UtcNow;
                    Session.TotalFiles             = Session.Root.TotalFiles;
                    Session.TotalFileSizeEstimated = Session.Root.TotalFileSize;

                    IEnumerable <string> distinctUrls = Session.Root.AllFileUrls.Distinct();

                    if (Session.TotalFiles != distinctUrls.Count())
                    {
                        Logger.Error($"Indexed files and unique files is not the same, please check results. Found a total of {Session.TotalFiles} files resulting in {distinctUrls.Count()} urls");
                    }

                    if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoUrls && Session.Root.Uri.Host != Constants.GoogleDriveDomain && Session.Root.Uri.Host != Constants.BlitzfilesTechDomain)
                    {
                        if (Session.TotalFiles > 0)
                        {
                            Logger.Info("Saving URL list to file...");
                            Console.WriteLine("Saving URL list to file...");

                            string scansPath = Library.GetScansPath();

                            try
                            {
                                string urlsFileName = $"{Library.CleanUriToFilename(Session.Root.Uri)}.txt";
                                string urlsPath     = Path.Combine(scansPath, urlsFileName);
                                File.WriteAllLines(urlsPath, distinctUrls);
                                Logger.Info($"Saved URL list to file: {urlsFileName}");
                                Console.WriteLine($"Saved URL list to file: {urlsFileName}");

                                if (OpenDirectoryIndexerSettings.CommandLineOptions.UploadUrls && Session.TotalFiles > 0)
                                {
                                    Console.WriteLine($"Uploading URLs ({FileSizeHelper.ToHumanReadable(new FileInfo(urlsPath).Length)})...");

                                    bool uploadSucceeded = false;

                                    try
                                    {
                                        GoFileIoFile uploadedFile = await GoFileIo.UploadFile(HttpClient, urlsPath);
                                        HistoryLogger.Info($"goFile.io: {JsonConvert.SerializeObject(uploadedFile)}");
                                        Session.UploadedUrlsUrl = uploadedFile.Url.ToString();
                                        uploadSucceeded         = true;

                                        Console.WriteLine($"Uploaded URLs link: {Session.UploadedUrlsUrl}");
                                    }
                                    catch (Exception ex)
                                    {
                                        Logger.Warn($"Error uploading URLs: {ex.Message}");
                                    }

                                    if (!uploadSucceeded)
                                    {
                                        Logger.Warn($"Using fallback for uploading URLs file.");

                                        try
                                        {
                                            UploadFilesIoFile uploadedFile = await UploadFilesIo.UploadFile(HttpClient, urlsPath);
                                            HistoryLogger.Info($"UploadFiles.io: {JsonConvert.SerializeObject(uploadedFile)}");
                                            Session.UploadedUrlsUrl = uploadedFile.Url.ToString();
                                            uploadSucceeded         = true;

                                            Console.WriteLine($"Uploaded URLs link: {Session.UploadedUrlsUrl}");
                                        }
                                        catch (Exception ex)
                                        {
                                            Logger.Warn($"Error uploading URLs: {ex.Message}");
                                        }
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                Logger.Error(ex);
                            }
                        }
                        else
                        {
                            Logger.Info("No URLs to save");
                            Console.WriteLine("No URLs to save");
                        }
                    }

                    distinctUrls = null;

                    if (OpenDirectoryIndexerSettings.CommandLineOptions.Speedtest && Session.Root.Uri.Host != Constants.GoogleDriveDomain && Session.Root.Uri.Host != Constants.BlitzfilesTechDomain)
                    {
                        if (Session.TotalFiles > 0)
                        {
                            if (Session.Root.Uri.Scheme == Constants.UriScheme.Http || Session.Root.Uri.Scheme == Constants.UriScheme.Https)
                            {
                                try
                                {
                                    WebFile biggestFile = Session.Root.AllFiles.OrderByDescending(f => f.FileSize).First();

                                    Console.WriteLine($"Starting speedtest (10-25 seconds)...");
                                    Console.WriteLine($"Test file: {FileSizeHelper.ToHumanReadable(biggestFile.FileSize)} {biggestFile.Url}");
                                    Session.SpeedtestResult = await Library.DoSpeedTestHttpAsync(HttpClient, biggestFile.Url);

                                    if (Session.SpeedtestResult != null)
                                    {
                                        Console.WriteLine($"Finished speedtest. Downloaded: {FileSizeHelper.ToHumanReadable(Session.SpeedtestResult.DownloadedBytes)}, Time: {Session.SpeedtestResult.ElapsedMilliseconds / 1000:F1} s, Speed: {Session.SpeedtestResult.MaxMBsPerSecond:F1} MB/s ({Session.SpeedtestResult.MaxMBsPerSecond * 8:F0} mbit)");
                                    }
                                }
                                catch (Exception ex)
                                {
                                    // Give empty speedtest, so it will be reported as Failed
                                    Session.SpeedtestResult = new Shared.SpeedtestResult();
                                    Logger.Error(ex, "Speedtest failed");
                                }
                            }
                            else if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
                            {
                                try
                                {
                                    FluentFTP.FtpClient ftpClient = FtpParser.FtpClients.FirstOrDefault(c => c.Value.IsConnected).Value;

                                    FtpParser.CloseAll(exceptFtpClient: ftpClient);

                                    if (ftpClient != null)
                                    {
                                        WebFile biggestFile = Session.Root.AllFiles.OrderByDescending(f => f.FileSize).First();

                                        Console.WriteLine($"Starting speedtest (10-25 seconds)...");
                                        Console.WriteLine($"Test file: {FileSizeHelper.ToHumanReadable(biggestFile.FileSize)} {biggestFile.Url}");

                                        Session.SpeedtestResult = await Library.DoSpeedTestFtpAsync(ftpClient, biggestFile.Url);

                                        if (Session.SpeedtestResult != null)
                                        {
                                            Console.WriteLine($"Finished speedtest. Downloaded: {FileSizeHelper.ToHumanReadable(Session.SpeedtestResult.DownloadedBytes)}, Time: {Session.SpeedtestResult.ElapsedMilliseconds / 1000:F1} s, Speed: {Session.SpeedtestResult.MaxMBsPerSecond:F1} MB/s ({Session.SpeedtestResult.MaxMBsPerSecond * 8:F0} mbit)");
                                        }
                                    }
                                    else
                                    {
                                        Console.WriteLine($"Cannot do speedtest because there is no connected FTP client anymore");
                                    }
                                }
                                catch (Exception ex)
                                {
                                    // Give empty speedtest, so it will be reported as Failed
                                    Session.SpeedtestResult = new Shared.SpeedtestResult();
                                    Logger.Error(ex, "Speedtest failed");
                                }
                            }
                        }
                    }

                    if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
                    {
                        FtpParser.CloseAll();
                    }

                    Logger.Info("Logging sessions stats...");
                    try
                    {
                        string sessionStats = Statistics.GetSessionStats(Session, includeExtensions: true, includeBanner: true);
                        Logger.Info(sessionStats);
                        HistoryLogger.Info(sessionStats);
                        Logger.Info("Logged sessions stats");

                        if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoReddit)
                        {
                            // Also log to screen, when saving links or JSON fails and the logs keep filling by other sessions, this will be saved
                            Console.WriteLine(sessionStats);
                        }
                    }
                    catch (Exception ex)
                    {
                        Logger.Error(ex);
                    }

                    if (Session.UrlsWithErrors.Any())
                    {
                        Logger.Info("URLs with errors:");
                        Console.WriteLine("URLs with errors:");

                        foreach (string urlWithError in Session.UrlsWithErrors.OrderBy(u => u))
                        {
                            Logger.Info(urlWithError);
                            Console.WriteLine(urlWithError);
                        }
                    }

                    if (OpenDirectoryIndexerSettings.CommandLineOptions.Json)
                    {
                        Logger.Info("Save session to JSON");
                        Console.WriteLine("Save session to JSON");

                        try
                        {
                            Library.SaveSessionJson(Session);
                            Logger.Info($"Saved session: {Library.CleanUriToFilename(Session.Root.Uri)}.json");
                            Console.WriteLine($"Saved session: {Library.CleanUriToFilename(Session.Root.Uri)}.json");
                        }
                        catch (Exception ex)
                        {
                            Logger.Error(ex);
                        }
                    }

                    Logger.Info("Finished indexing!");
                    Console.WriteLine("Finished indexing!");

                    Program.SetConsoleTitle($"✔ {Program.ConsoleTitle}");

                    if (OpenDirectoryIndexerSettings.CommandLineOptions.Quit)
                    {
                        Command.KillApplication();
                    }
                    else
                    {
                        Console.WriteLine("Press ESC to exit! Or C to copy to clipboard and quit!");
                    }
                }
                catch (Exception ex)
                {
                    Logger.Error(ex);
                }
            });
        }
        public async void StartIndexingAsync()
        {
            bool fromFile = !string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.FileName);

            if (fromFile)
            {
                Session = Library.LoadSessionJson(OpenDirectoryIndexerSettings.FileName);
                Console.WriteLine(Statistics.GetSessionStats(Session, includeExtensions: true));
                Console.ReadKey(intercept: true);
                return;
            }
            else
            {
                Session = new Session
                {
                    Started = DateTimeOffset.UtcNow,
                    Root    = new WebDirectory(parentWebDirectory: null)
                    {
                        Name = "ROOT",
                        Url  = OpenDirectoryIndexerSettings.Url
                    }
                };
            }

            if (Session.Root.Uri.Host == Constants.GoogleDriveDomain)
            {
                Logger.Warn("Google Drive scanning is limited to 9 directories per second!");
            }

            if (Session.Root.Uri.Scheme == "ftp")
            {
                Logger.Warn("Retrieving FTP software!");
                // TODO: Replace with library?
                Logger.Warn(await FtpParser.GetFtpServerInfo(Session.Root));
                //AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
            }

            TimerStatistics = new System.Timers.Timer
            {
                Enabled  = true,
                Interval = TimeSpan.FromSeconds(30).TotalMilliseconds
            };

            TimerStatistics.Elapsed += TimerStatistics_Elapsed;

            IndexingTask = Task.Run(async() =>
            {
                try
                {
                    WebDirectoriesQueue = new ConcurrentQueue <WebDirectory>();

                    if (fromFile)
                    {
                        SetParentDirectories(Session.Root);

                        // TODO: Add unfinished items to queue, very complicated, we need to ALSO fill the ParentDirectory...
                        //// With filter predicate, with selection function
                        //var flatList = nodes.Flatten(n => n.IsDeleted == false, n => n.Children);
                        //var directoriesToDo = Session.Root.Subdirectories.Flatten(null, wd => wd.Subdirectories).Where(wd => !wd.Finished);
                    }
                    else
                    {
                        // Add root
                        WebDirectoriesQueue.Enqueue(Session.Root);
                    }

                    IndexingTaskCTS = new CancellationTokenSource();

                    for (int i = 1; i <= WebDirectoryProcessors.Length; i++)
                    {
                        string processorId = i.ToString();

                        WebDirectoryProcessors[i - 1] = WebDirectoryProcessor(WebDirectoriesQueue, $"Processor {processorId}", IndexingTaskCTS.Token);
                    }

                    for (int i = 1; i <= WebFileFileSizeProcessors.Length; i++)
                    {
                        string processorId = i.ToString();

                        WebFileFileSizeProcessors[i - 1] = WebFileFileSizeProcessor(WebFilesFileSizeQueue, $"Processor {processorId}", IndexingTaskCTS.Token, WebDirectoryProcessors);
                    }

                    await Task.WhenAll(WebDirectoryProcessors);
                    Console.WriteLine("Finshed indexing");
                    Logger.Info("Finshed indexing");

                    if (Session.Root.Uri.Scheme == "ftp")
                    {
                        FtpParser.CloseAll();
                    }

                    if (WebFilesFileSizeQueue.Any())
                    {
                        TimerStatistics.Interval = TimeSpan.FromSeconds(5).TotalMilliseconds;
                        Console.WriteLine($"Retrieving filesize of {WebFilesFileSizeQueue.Count} urls");
                    }

                    await Task.WhenAll(WebFileFileSizeProcessors);

                    TimerStatistics.Stop();

                    Session.Finished               = DateTimeOffset.UtcNow;
                    Session.TotalFiles             = Session.Root.TotalFiles;
                    Session.TotalFileSizeEstimated = Session.Root.TotalFileSize;

                    if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoUrls && Session.Root.Uri.Host != Constants.GoogleDriveDomain)
                    {
                        if (Session.TotalFiles > 0)
                        {
                            Logger.Info("Saving URL list to file...");
                            Console.WriteLine("Saving URL list to file...");

                            string scansPath = Library.GetScansPath();

                            try
                            {
                                string fileUrls = string.Join(Environment.NewLine, Session.Root.AllFileUrls.Distinct());

                                string urlsFileName = $"{Library.CleanUriToFilename(Session.Root.Uri)}.txt";
                                string urlsPath     = Path.Combine(scansPath, urlsFileName);
                                Logger.Info("String joined");
                                File.WriteAllText(urlsPath, fileUrls);
                                Logger.Info($"Saved URL list to file: {urlsFileName}");
                                Console.WriteLine($"Saved URL list to file: {urlsFileName}");

                                if (OpenDirectoryIndexerSettings.CommandLineOptions.UploadUrls && Session.TotalFiles > 0)
                                {
                                    Console.WriteLine("Uploading URLs...");

                                    //UploadFilesFile uploadFilesFile = await UploadFileIo.UploadFile(HttpClient, urlsPath);
                                    //HistoryLogger.Info($"uploadfiles.io: {JsonConvert.SerializeObject(uploadFilesFile)}");
                                    //Session.UploadedUrlsUrl = uploadFilesFile.Url.ToString();

                                    GoFilesFile uploadedFile = await GoFileIo.UploadFile(HttpClient, urlsPath);
                                    HistoryLogger.Info($"goFile.io: {JsonConvert.SerializeObject(uploadedFile)}");
                                    Session.UploadedUrlsUrl = uploadedFile.Url.ToString();

                                    Console.WriteLine($"Uploaded URLs: {Session.UploadedUrlsUrl}");
                                }
                            }
                            catch (Exception ex)
                            {
                                Logger.Error(ex);
                            }
                        }
                        else
                        {
                            Logger.Info("No URLs to save");
                            Console.WriteLine("No URLs to save");
                        }
                    }

                    if (OpenDirectoryIndexerSettings.CommandLineOptions.Speedtest && Session.Root.Uri.Host != Constants.GoogleDriveDomain)
                    {
                        if (Session.TotalFiles > 0)
                        {
                            if (Session.Root.Uri.Scheme == "https" || Session.Root.Uri.Scheme == "http")
                            {
                                try
                                {
                                    WebFile biggestFile = Session.Root.AllFiles.OrderByDescending(f => f.FileSize).First();

                                    Console.WriteLine($"Starting speedtest (10-25 seconds)...");
                                    Console.WriteLine($"Test file: {FileSizeHelper.ToHumanReadable(biggestFile.FileSize)} {biggestFile.Url}");
                                    Session.SpeedtestResult = await Library.DoSpeedTestAsync(HttpClient, biggestFile.Url);
                                    Console.WriteLine($"Finished speedtest. Downloaded: {FileSizeHelper.ToHumanReadable(Session.SpeedtestResult.DownloadedBytes)}, Time: {Session.SpeedtestResult.ElapsedMiliseconds / 1000:F1} s, Speed: {Session.SpeedtestResult.MaxMBsPerSecond:F1} MB/s ({Session.SpeedtestResult.MaxMBsPerSecond * 8:F0} mbit)");
                                }
                                catch (Exception ex)
                                {
                                    Logger.Error(ex, "Speedtest failed");
                                }
                            }
                            else
                            {
                                Logger.Warn($"Only a speedtest for HTTP(S), not '{Session.Root.Uri.Scheme}'");
                            }
                        }
                    }

                    Logger.Info("Logging sessions stats...");
                    try
                    {
                        string sessionStats = Statistics.GetSessionStats(Session, includeExtensions: true);
                        Logger.Info(sessionStats);
                        HistoryLogger.Info(sessionStats);
                        Logger.Info("Logged sessions stats");

                        if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoReddit)
                        {
                            // Also log to screen, when saving links or JSON fails and the logs keep filling by other sessions, this will be saved
                            Console.WriteLine(sessionStats);
                        }
                    }
                    catch (Exception ex)
                    {
                        Logger.Error(ex);
                    }

                    if (Session.UrlsWithErrors.Any())
                    {
                        Logger.Info("URLs with errors:");
                        Console.WriteLine("URLs with errors:");

                        foreach (string urlWithError in Session.UrlsWithErrors.OrderBy(u => u))
                        {
                            Logger.Info(urlWithError);
                            Console.WriteLine(urlWithError);
                        }
                    }

                    if (OpenDirectoryIndexerSettings.CommandLineOptions.Json)
                    {
                        Logger.Info("Save session to JSON");
                        Console.WriteLine("Save session to JSON");

                        try
                        {
                            Library.SaveSessionJson(Session);
                            Logger.Info($"Saved session: {PathHelper.GetValidPath(Session.Root.Url)}.json");
                            Console.WriteLine($"Saved session: {PathHelper.GetValidPath(Session.Root.Url)}.json");
                        }
                        catch (Exception ex)
                        {
                            Logger.Error(ex);
                        }
                    }

                    Logger.Info("Finished indexing!");
                    Console.WriteLine("Finished indexing!");

                    Program.SetConsoleTitle($"✔ {Program.ConsoleTitle}");

                    if (OpenDirectoryIndexerSettings.CommandLineOptions.Quit)
                    {
                        Command.KillApplication();
                    }
                    else
                    {
                        Console.WriteLine("Press ESC to exit! Or C to copy to clipboard and quit!");
                    }
                }
                catch (Exception ex)
                {
                    Logger.Error(ex);
                }
            });
        }