protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url, CancellationToken ct) { try { var fileDownloader = new FileDownloader(); return(await fileDownloader.DownloadFileWithResumeAsync(url, fileLocation, shellService.Settings, ct)); } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70) { Logger.Error("ManagerController:Download: {0}", ex); shellService.ShowError(ex, Resources.DiskFull); crawlerService.StopCommand.Execute(null); return(false); } catch (WebException webException) when((webException.Response != null)) { var webRespStatusCode = (int)((HttpWebResponse)webException?.Response).StatusCode; if (webRespStatusCode >= 400 && webRespStatusCode < 600) // removes inaccessible files: status code 400 -- 599 { File.Delete(fileLocation); } return(false); } catch { return(false); } }
protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url) { try { return(await fileDownloader.DownloadFileWithResumeAsync(url, fileLocation).TimeoutAfter(shellService.Settings.TimeOut)); } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70) { // Disk Full, HRESULT: â€-2147024784‬ == 0xFFFFFFFF80070070 Logger.Error("Downloader:DownloadBinaryFile: {0}", ex); shellService.ShowError(ex, Resources.DiskFull); crawlerService.StopCommand.Execute(null); return(false); } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x20) { // The process cannot access the file because it is being used by another process.", HRESULT: -2147024864 == 0xFFFFFFFF80070020 return(true); } catch (WebException webException) when((webException.Response != null)) { var webRespStatusCode = (int)((HttpWebResponse)webException?.Response).StatusCode; if (webRespStatusCode >= 400 && webRespStatusCode < 600) // removes inaccessible files: http status codes 400 to 599 { try { File.Delete(fileLocation); } // could be open again in a different thread catch { } } return(false); } catch { return(false); } }
private async Task LoadLibrary() { Logger.Verbose("ManagerController.LoadLibrary:Start"); managerService.BlogFiles.Clear(); string path = Path.Combine(shellService.Settings.DownloadLocation, "Index"); try { if (Directory.Exists(path)) { { IReadOnlyList <IBlog> files = await GetIBlogsAsync(path); foreach (IBlog file in files) { managerService.BlogFiles.Add(file); } BlogManagerFinishedLoadingLibrary?.Invoke(this, EventArgs.Empty); } } } catch (Exception ex) { Logger.Verbose("ManagerController:LoadLibrary: {0}", ex); shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]); } Logger.Verbose("ManagerController.LoadLibrary:End"); }
public override async Task UpdateMetaInformationAsync() { try { if (blog.Online) { string document = await GetSvcPageAsync("1", "0"); var response = ConvertJsonToClass <TumblrJson>(document); if (response.meta.status == 200) { blog.Title = response.response.posts.FirstOrDefault().blog.title; blog.Description = response.response.posts.FirstOrDefault().blog.description; } } } catch (WebException webException) { if (webException.Message.Contains("503")) { Logger.Error("TumblrDownloader:GetUrlsAsync: {0}", "User not logged in"); shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name); return; } } }
private IReadOnlyList <IBlog> GetIBlogsCore(string directory) { Logger.Verbose("ManagerController:GetIBlogsCore Start"); var blogs = new List <IBlog>(); var failedToLoadBlogs = new List <string>(); string[] supportedFileTypes = Enum.GetNames(typeof(BlogTypes)).ToArray(); foreach (string filename in Directory.GetFiles(directory, "*").Where( fileName => supportedFileTypes.Any(fileName.Contains) && !fileName.Contains("_files"))) { //TODO: Refactor try { if (filename.EndsWith(BlogTypes.tumblr.ToString())) { blogs.Add(new TumblrBlog().Load(filename)); } if (filename.EndsWith(BlogTypes.tmblrpriv.ToString())) { blogs.Add(new TumblrHiddenBlog().Load(filename)); } if (filename.EndsWith(BlogTypes.tlb.ToString())) { blogs.Add(new TumblrLikedByBlog().Load(filename)); } if (filename.EndsWith(BlogTypes.tumblrsearch.ToString())) { blogs.Add(new TumblrSearchBlog().Load(filename)); } if (filename.EndsWith(BlogTypes.tumblrtagsearch.ToString())) { blogs.Add(new TumblrTagSearchBlog().Load(filename)); } } catch (SerializationException ex) { failedToLoadBlogs.Add(ex.Data["Filename"].ToString()); } } if (failedToLoadBlogs.Any()) { string failedBlogNames = failedToLoadBlogs.Aggregate((a, b) => a + ", " + b); Logger.Verbose("ManagerController:GetIBlogsCore: {0}", failedBlogNames); _shellService.ShowError(new SerializationException(), Resources.CouldNotLoadLibrary, failedBlogNames); } Logger.Verbose("ManagerController.GetIBlogsCore End"); return(blogs); }
private async Task GetUrlsAsync(IProgress <DownloadProgress> progress, CancellationToken ct, PauseToken pt) { var semaphoreSlim = new SemaphoreSlim(shellService.Settings.ParallelScans); var trackedTasks = new List <Task>(); foreach (int crawlerNumber in Enumerable.Range(1, shellService.Settings.ParallelScans)) { await semaphoreSlim.WaitAsync(); trackedTasks.Add(new Func <Task>(async() => { var tags = new List <string>(); if (!string.IsNullOrWhiteSpace(blog.Tags)) { tags = blog.Tags.Split(',').Select(x => x.Trim()).ToList(); } try { string document = await RequestDataAsync(crawlerNumber); await AddUrlsToDownloadList(document, tags, progress, crawlerNumber, ct, pt); } catch (WebException webException) { if (webException.Message.Contains("503")) { Logger.Error("TumblrDownloader:GetUrlsAsync: {0}", "User not logged in"); shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name); return; } if (webException.Message.Contains("429")) { // TODO: add retry logic? Logger.Error("TumblrDownloader:GetUrls:WebException {0}", webException); shellService.ShowError(webException, Resources.LimitExceeded, blog.Name); } } catch { } finally { semaphoreSlim.Release(); } })()); } await Task.WhenAll(trackedTasks); producerConsumerCollection.CompleteAdding(); if (!ct.IsCancellationRequested) { UpdateBlogStats(); } }
private async Task SaveChangesAsync(MusicFile musicFile) { if (musicFile == null) { return; } IReadOnlyCollection <MusicFile> allFilesToSave; if (musicFile.SharedMusicFiles.Any()) { allFilesToSave = musicFile.SharedMusicFiles; } else { allFilesToSave = new[] { musicFile }; } // Filter out the music file that is currently playing var playingMusicFile = PlaylistManager.CurrentItem?.MusicFile; var filesToSave = allFilesToSave.Except(new[] { playingMusicFile }).ToArray(); foreach (var x in allFilesToSave.Intersect(new[] { playingMusicFile })) { musicFilesToSaveAfterPlaying.Add(x); } if (!filesToSave.Any()) { return; } var tasks = filesToSave.Select(x => SaveChangesCoreAsync(x)).ToArray(); try { await Task.WhenAll(tasks); } catch (Exception ex) { Log.Default.Error(ex, "SaveChangesAsync"); if (filesToSave.Count() == 1) { shellService.ShowError(ex, Resources.CouldNotSaveFile, filesToSave.First().FileName); } else { shellService.ShowError(ex, Resources.CouldNotSaveFiles); } } finally { RemoveMusicFilesToSaveAfterPlaying(filesToSave); } }
public virtual async Task IsBlogOnlineAsync() { try { await RequestDataAsync(blog.Url); blog.Online = true; } catch (WebException webException) { Logger.Error("TumblrBlogCrawler:IsBlogOnlineAsync:WebException {0}", webException); shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name); blog.Online = false; } }
private async Task AppendToTextFileAsync(string fileLocation, T data) { try { using (var stream = new FileStream(fileLocation, FileMode.Create, FileAccess.Write)) { using (XmlDictionaryWriter writer = JsonReaderWriterFactory.CreateJsonWriter( stream, Encoding.UTF8, true, true, " ")) { var serializer = new DataContractJsonSerializer(data.GetType()); serializer.WriteObject(writer, data); writer.Flush(); await Task.CompletedTask; } } } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70) { Logger.Error("TumblrJsonDownloader:AppendToTextFile: {0}", ex); shellService.ShowError(ex, Resources.DiskFull); crawlerService.StopCommand.Execute(null); } catch { } }
public async Task ConfirmPrivacyConsentAsync() { try { await PerformPrivacyConsentRequestAsync(); } catch (TimeoutException timeoutException) { Logger.Error("{0}, {1}", string.Format(CultureInfo.CurrentCulture, Resources.TimeoutReachedShort, Resources.ConfirmingTumblrPrivacyConsent), timeoutException); shellService.ShowError(timeoutException, Resources.ConfirmingTumblrPrivacyConsentFailed); } catch (Exception exception) { Logger.Error("{0}, {1}", string.Format(CultureInfo.CurrentCulture, Resources.ConfirmingTumblrPrivacyConsentFailed), exception); shellService.ShowError(exception, Resources.ConfirmingTumblrPrivacyConsentFailed); } }
public virtual async Task IsBlogOnlineAsync() { try { await RequestDataAsync(blog.Url); blog.Online = true; } catch (WebException webException) { if (webException.Status == WebExceptionStatus.RequestCanceled) { return; } Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", webException); shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name); blog.Online = false; } catch (TimeoutException timeoutException) { HandleTimeoutException(timeoutException, Resources.OnlineChecking); blog.Online = false; } }
private void OpenListCore(string playlistFileName) { Playlist playlist; try { var playlistFile = StorageFile.GetFileFromPathAsync(playlistFileName).GetResult(); // MS Issue: LoadAsync cannot load a playlist when one of the files do not exists anymore. playlist = Playlist.LoadAsync(playlistFile).GetResult(); } catch (Exception ex) { Log.Default.Error(ex, "OpenListCore"); shellService.ShowError(ex, Resources.CouldNotLoadPlaylist); return; } InsertFilesCore(PlaylistManager.Items.Count, playlist.Files.Select(x => x.Path).ToArray()); }
protected virtual async Task <bool> DownloadBinaryFile(string fileLocation, string url, CancellationToken ct) { try { return(await ThrottledStream.DownloadFileWithResumeAsync(url, fileLocation, shellService.Settings, ct)); } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70) { Logger.Error("ManagerController:Download: {0}", ex); shellService.ShowError(ex, Resources.DiskFull); crawlerService.StopCommand.Execute(null); return(false); } catch { return(false); } }
private void OpenListCore(string queuelistFileName) { QueueSettings queueList; try { using (var stream = new FileStream(queuelistFileName, FileMode.Open, FileAccess.Read, FileShare.Read)) { var serializer = new DataContractJsonSerializer(typeof(QueueSettings)); queueList = (QueueSettings)serializer.ReadObject(stream); } } catch (Exception ex) { Logger.Error("QueueController:OpenListCore: {0}", ex); shellService.ShowError(ex, Resources.CouldNotLoadQueuelist); return; } InsertFilesCore(QueueManager.Items.Count(), queueList.Names.ToArray(), queueList.Types.ToArray()); }
public void SetText(string text) { try { Clipboard.SetText(text); } catch (Exception ex) { Logger.Error($"ClipboardService:SetText: {ex}"); _shellService.ShowError(new ClipboardContentException(ex), "error setting clipboard content"); } }
public virtual async Task IsBlogOnlineAsync() { try { await RequestDataAsync(blog.Url); blog.Online = true; } catch (WebException webException) { Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", webException); shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name); blog.Online = false; } catch (TimeoutException timeoutException) { Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", timeoutException); shellService.ShowError(timeoutException, Resources.TimeoutReached, Resources.OnlineChecking, blog.Name); blog.Online = false; } }
private async Task LoadLibrary() { Logger.Verbose("ManagerController.LoadLibrary:Start"); managerService.BlogFiles.Clear(); string path = Path.Combine(shellService.Settings.DownloadLocation, "Index"); try { if (Directory.Exists(path)) { { IReadOnlyList <IBlog> files = await GetFilesAsync(path); foreach (IBlog file in files) { managerService.BlogFiles.Add(file); } BlogManagerFinishedLoading?.Invoke(this, EventArgs.Empty); if (shellService.Settings.CheckOnlineStatusAtStartup) { foreach (IBlog blog in files) { IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); await downloader.IsBlogOnlineAsync(); } } } } } catch (Exception ex) { Logger.Error("ManagerController:LoadLibrary: {0}", ex); shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]); } Logger.Verbose("ManagerController.LoadLibrary:End"); }
private async void DeleteFileAsync(string fileName) { try { var file = await StorageFile.GetFileFromPathAsync(fileName); await file.DeleteAsync(); } catch (Exception ex) { shellService.ShowError(ex, Resources.CouldNotDeleteFile, fileName); } }
private void OpenListCore(string queuelistFileName) { List <string> queueList; try { using (FileStream stream = new FileStream(queuelistFileName, FileMode.Open, FileAccess.Read, FileShare.Read)) { IFormatter formatter = new BinaryFormatter(); queueList = (List <string>)formatter.Deserialize(stream); } } catch (Exception ex) { Logger.Error("OpenListCore: {0}", ex); shellService.ShowError(ex, Resources.CouldNotLoadQueuelist); return; } InsertFilesCore(QueueManager.Items.Count(), queueList.ToArray()); }
private async void LoadLibrary() { Logger.Verbose("ManagerController.UpdateBlogFiles:Start"); selectionService.BlogFiles.Clear(); var path = Path.Combine(shellService.Settings.DownloadLocation, "Index"); try { if (Directory.Exists(path)) { { var files = await GetFilesAsync(path); foreach (var file in files) { selectionService.BlogFiles.Add(file); } if (shellService.Settings.CheckOnlineStatusAtStartup == true) { foreach (var file in files) { file.Online = await IsBlogOnline(file.Name); } } } } } catch (Exception ex) { Logger.Error("ManagerController:LoadLibrary: {0}", ex); shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]); return; } Logger.Verbose("ManagerController.LoadLibrary:End"); }
private void InsertFiles(int index, IEnumerable <string> fileNames) { MusicFile[] musicFiles; try { musicFiles = fileNames.Select(musicFileContext.Create).ToArray(); } catch (Exception ex) { Log.Default.Error(ex, "TranscodingController.InsertFile"); shellService.ShowError(ex, Resources.CouldNotOpenFiles); return; } Transcode(GetMusicFilesSupportedToConvert(musicFiles).ToArray()); }
protected AbstractDownloader(IShellService shellService, IManagerService managerService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, IPostQueue <AbstractPost> postQueue, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null) { this.shellService = shellService; this.crawlerService = crawlerService; this.managerService = managerService; this.blog = blog; this.files = files; this.ct = ct; this.pt = pt; this.progress = progress; this.postQueue = postQueue; this.fileDownloader = fileDownloader; Progress <Exception> prog = new Progress <Exception>((e) => shellService.ShowError(e, Resources.CouldNotSaveBlog, blog.Name)); _saveTimer = new Timer(_ => OnSaveTimedEvent(prog), null, SAVE_TIMESPAN_SECS * 1000, SAVE_TIMESPAN_SECS * 1000); }
private IFiles LoadFiles(IBlog blog) { if (settings.LoadAllDatabases) { var files = managerService.Databases.FirstOrDefault(file => file.Name.Equals(blog.Name) && file.BlogType.Equals(blog.OriginalBlogType)); if (files == null) { var s = string.Format("{0} ({1})", blog.Name, blog.BlogType); Logger.Error(Resources.CouldNotLoadLibrary, s); shellService.ShowError(new KeyNotFoundException(), Resources.CouldNotLoadLibrary, s); throw new KeyNotFoundException(s); } return(files); } return(Files.Load(blog.ChildId)); }
private async Task AppendToTextFile(string fileLocation, XContainer data) { try { using (var sw = new StreamWriter(fileLocation, true)) { await sw.WriteAsync(PrettyXml(data)); } } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70) { Logger.Error("TumblrXmlDownloader:AppendToTextFile: {0}", ex); shellService.ShowError(ex, Resources.DiskFull); crawlerService.StopCommand.Execute(null); } catch { } }
private async Task AppendToTextFileAsync(string fileLocation, T data) { try { if (typeof(T) == typeof(DataModels.TumblrSearchJson.Datum) || typeof(T) == typeof(DataModels.Twitter.TimelineTweets.Tweet)) { var serializer = new JsonSerializer(); using (StreamWriter sw = new StreamWriter(fileLocation, false)) using (JsonWriter writer = new JsonTextWriter(sw) { Formatting = Newtonsoft.Json.Formatting.Indented }) { serializer.Serialize(writer, data); } } else { using (var stream = new FileStream(fileLocation, FileMode.Create, FileAccess.Write)) { using (XmlDictionaryWriter writer = JsonReaderWriterFactory.CreateJsonWriter( stream, Encoding.UTF8, true, true, " ")) { var serializer = new DataContractJsonSerializer(data.GetType()); serializer.WriteObject(writer, data); writer.Flush(); } } } await Task.CompletedTask; } catch (IOException ex) when((ex.HResult & 0xFFFF) == 0x27 || (ex.HResult & 0xFFFF) == 0x70) { Logger.Error("TumblrJsonDownloader:AppendToTextFile: {0}", ex); shellService.ShowError(ex, Resources.DiskFull); crawlerService.StopCommand.Execute(null); } catch { } }
private async Task GetUrlsAsync(IProgress <DownloadProgress> progress, CancellationToken ct, PauseToken pt) { var semaphoreSlim = new SemaphoreSlim(shellService.Settings.ParallelScans); var trackedTasks = new List <Task>(); foreach (int crawlerNumber in Enumerable.Range(0, shellService.Settings.ParallelScans)) { await semaphoreSlim.WaitAsync(); trackedTasks.Add(new Func <Task>(async() => { try { string document = await RequestDataAsync(blog.Url + "/page/" + crawlerNumber); if (!CheckIfLoggedIn(document)) { Logger.Error("TumblrLikedByDownloader:GetUrlsAsync: {0}", "User not logged in"); shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name); return; } await AddUrlsToDownloadList(document, progress, crawlerNumber, ct, pt); } catch (WebException) { } finally { semaphoreSlim.Release(); } })()); } await Task.WhenAll(trackedTasks); producerConsumerCollection.CompleteAdding(); if (!ct.IsCancellationRequested) { UpdateBlogStats(); } }
public ManagerViewModel(IManagerView view, IShellService shellService, Lazy <ISelectionService> selectionService, ICrawlerService crawlerService) : base(view) { ShellService = shellService; this.selectionService = selectionService; this.crawlerService = crawlerService; try { if (shellService.Settings.ColumnWidths.Count != 0) { view.DataGridColumnRestore = ShellService.Settings.ColumnWidths; } } catch (Exception ex) { Logger.Error("ManagerController: {0}", ex); shellService.ShowError(ex, Resources.CouldNotRestoreUISettings); return; } ShellService.Closing += ViewClosed; }
private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) { while (true) { if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } bool lockTaken = false; Monitor.Enter(_lockObject, ref lockTaken); try { if (_crawlerService.ActiveItems.Count < QueueManager.Items.Count) { QueueListItem nextQueueItem = QueueManager.Items.Except(_crawlerService.ActiveItems).First(); IBlog blog = nextQueueItem.Blog; ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress <DownloadProgress>(), pt, ct); crawler.IsBlogOnlineAsync().Wait(4000); crawler.Dispose(); if (_crawlerService.ActiveItems.Any(item => item.Blog.Name.Equals(nextQueueItem.Blog.Name) && item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(_lockObject); continue; } _crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(_lockObject); lockTaken = false; await StartSiteSpecificDownloaderAsync(nextQueueItem, pt, ct); } else { Monitor.Exit(_lockObject); lockTaken = false; await Task.Delay(4000, ct); } } catch (Exception e) { Logger.Error("CrawlerController.RunCrawlerTasksAsync: {0}", e); _shellService.ShowError(e, "Error starting the next item in the queue."); if (lockTaken) { Monitor.Exit(_lockObject); } } } }
public static void ShowError(this IShellService shellService, Exception exception, string format, params object[] args) => shellService.ShowError(exception, string.Format(CultureInfo.CurrentCulture, format, args));
private async Task <Tuple <ulong, bool> > GetUrlsAsync(IProgress <DownloadProgress> progress, CancellationToken ct, PauseToken pt) { var semaphoreSlim = new SemaphoreSlim(shellService.Settings.ParallelScans); var trackedTasks = new List <Task>(); var numberOfPostsCrawled = 0; var apiLimitHit = false; var completeGrab = true; ulong lastId = GetLastPostId(); await UpdateTotalPostCount(); int totalPosts = blog.Posts; ulong highestId = await GetHighestPostId(); foreach (int pageNumber in GetPageNumbers()) { await semaphoreSlim.WaitAsync(); if (!completeGrab) { break; } if (ct.IsCancellationRequested) { break; } if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(); } trackedTasks.Add(new Func <Task>(async() => { try { XDocument document = await GetApiPageAsync(pageNumber); completeGrab = CheckPostAge(document, lastId); var tags = new List <string>(); if (!string.IsNullOrWhiteSpace(blog.Tags)) { tags = blog.Tags.Split(',').Select(x => x.Trim()).ToList(); } AddUrlsToDownloadList(document, tags); } catch (WebException webException) { if (webException.Message.Contains("429")) { // TODO: add retry logic? apiLimitHit = true; Logger.Error("TumblrDownloader:GetUrls:WebException {0}", webException); shellService.ShowError(webException, Resources.LimitExceeded, blog.Name); } } finally { semaphoreSlim.Release(); } numberOfPostsCrawled += blog.PageSize; UpdateProgressQueueInformation(progress, Resources.ProgressGetUrlLong, numberOfPostsCrawled, totalPosts); })()); } await Task.WhenAll(trackedTasks); producerConsumerCollection.CompleteAdding(); if (!ct.IsCancellationRequested && completeGrab) { UpdateBlogStats(); } return(new Tuple <ulong, bool>(highestId, apiLimitHit)); }