Esempio n. 1
0
        public static async Task <WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory)
        {
            if (OpenDirectoryIndexer.Session.MaxThreads > 1)
            {
                throw new FriendlyException($"{Parser} can only scan at maximum of 1 thread, please call with -t 1 or --threads 1");
            }

            try
            {
                string driveHash = GetDriveHash(webDirectory);

                if (!OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.Parameters_Password))
                {
                    Console.WriteLine($"{Parser} will always be indexed with only 1 thread, else you will run into problems and errors.");
                    Logger.Info($"{Parser} will always be indexed with only 1 thread, else you will run into problems and errors.");
                    //OpenDirectoryIndexer.Session.MaxThreads = 1;

                    Console.WriteLine("Check if password is needed (unsupported currently)...");
                    Logger.Info("Check if password is needed (unsupported currently)...");
                    OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] = string.Empty;

                    HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetFolderUrl(driveHash, string.Empty, 0));

                    if (httpResponseMessage.IsSuccessStatusCode)
                    {
                        string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                        BlitzfilesTechResponse response = BlitzfilesTechResponse.FromJson(responseJson);

                        webDirectory = await ScanAsync(httpClient, webDirectory);
                    }
                }
                else
                {
                    webDirectory = await ScanAsync(httpClient, webDirectory);
                }
            }
            catch (Exception ex)
            {
                Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}");
                webDirectory.Error = true;

                OpenDirectoryIndexer.Session.Errors++;

                if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
                {
                    OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
                }

                throw;
            }

            return(webDirectory);
        }
        private static async Task <WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory)
        {
            Logger.Debug($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}");

            webDirectory.Parser = Parser;

            try
            {
                await RateLimiter.RateLimit();

                string driveHash  = GetDriveHash(webDirectory);
                string entryHash  = string.Empty;
                long   pageIndex  = 0;
                long   totalPages = 0;

                do
                {
                    Logger.Warn($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}, page {pageIndex + 1}");

                    HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetFolderUrl(driveHash, entryHash, pageIndex));

                    webDirectory.ParsedSuccesfully = httpResponseMessage.IsSuccessStatusCode;
                    httpResponseMessage.EnsureSuccessStatusCode();

                    string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                    BlitzfilesTechResponse indexResponse = BlitzfilesTechResponse.FromJson(responseJson);
                    entryHash = indexResponse.Link.Entry.Hash;

                    pageIndex  = indexResponse.FolderChildren.CurrentPage;
                    totalPages = indexResponse.FolderChildren.LastPage;

                    foreach (Entry entry in indexResponse.FolderChildren.Data)
                    {
                        if (entry.Type == "folder")
                        {
                            webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
                            {
                                Parser = Parser,
                                Url    = $"https://blitzfiles.tech/files/drive/s/{driveHash}:{entry.Hash}",
                                Name   = entry.Name
                            });
                        }
                        else
                        {
                            webDirectory.Files.Add(new WebFile
                            {
                                Url      = $"https://blitzfiles.tech/files/secure/uploads/download?hashes={entry.Hash}&shareable_link={indexResponse.Link.Id}",
                                FileName = entry.Name,
                                FileSize = entry.FileSize
                            });
                        }
                    }
                } while (pageIndex < totalPages);
            }
            catch (Exception ex)
            {
                Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}");
                webDirectory.Error = true;

                OpenDirectoryIndexer.Session.Errors++;

                if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
                {
                    OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
                }

                //throw;
            }

            return(webDirectory);
        }
Esempio n. 3
0
 public static string ToJson(this BlitzfilesTechResponse self) => JsonConvert.SerializeObject(self, Converter.Settings);