public TextureSender(IDownloader downloader, GriderProxy _proxy, int discardLevel, uint packetNumber) { proxy = _proxy; RequestedDiscardLevel = discardLevel; StartPacketNumber = packetNumber; m_downloader = downloader; }
public void DeleteDownloader(Job job, IDownloader downloader) { if (!(downloader is FileDispatch.Downloader)) { this.wwwFallback.DeleteDownloader(job, downloader); } }
public DownloadViewModel(DownloadedChapterInfo downloadInfo, ISemaphore downloadSemaphore) { if (downloadInfo == null) throw new ArgumentNullException("downloadInfo"); if (downloadSemaphore == null) throw new ArgumentNullException("downloadSemaphore"); if (downloadInfo.ChapterRecord == null) throw new ArgumentException("Chapter record is invalid.", "downloadInfo"); if (String.IsNullOrEmpty(downloadInfo.ChapterRecord.ChapterId)) throw new ArgumentException("Chapter record id is invalid.", "downloadInfo"); if (downloadInfo.ChapterRecord.MangaRecord == null) throw new ArgumentException("Manga record is invalid.", "downloadInfo"); if (String.IsNullOrEmpty(downloadInfo.ChapterRecord.MangaRecord.MangaId)) throw new ArgumentException("Manga record id is invalid.", "downloadInfo"); _downloadInfo = downloadInfo; _downloadSemaphore = downloadSemaphore; _scraper = ScraperLoader.Instance.AllScrapers.FirstOrDefault(s => s.ScraperGuid == downloadInfo.ChapterRecord.Scraper); if (_scraper != null) { _downloader = _scraper.GetDownloader(); // register downloader events _downloader.DownloadProgress += _downloader_DownloadProgress; _downloader.DownloadCompleted += _downloader_DownloadCompleted; } if (!String.IsNullOrEmpty(_downloadInfo.Path)) { // file was already downloaded State = DownloadState.Unknown; Completed = true; } else { // we will be downloading the file now State = DownloadState.Ok; Completed = false; } CurrentActionText = String.Empty; _cancelDownloadCommand = new RelayCommand(Cancel, x => !Completed); _removeDownloadCommand = new RelayCommand(Remove); _openDownloadCommand = new RelayCommand(Open, x => DownloadExists); _retryDownloadCommand = new RelayCommand(RetryDownload, x => _downloader != null && Completed && !DownloadExists && !String.IsNullOrEmpty(_downloadInfo.DownloadFolder)); CancelText = ButtonCancelText; }
private JetImageLoaderConfig(Builder builder) { IsLogEnabled = builder.IsLogEnabled; CacheMode = builder.CacheMode; DownloaderImpl = builder.DownloaderImpl; MemoryCacheImpl = builder.MemoryCacheImpl; StorageCacheImpl = builder.StorageCacheImpl; }
public MainForm() { InitializeComponent(); foreach (var textBox in Controls.OfType<TextBox>()) { textBox.Text = Uri; } _downloader = new Downloader(this); }
public void DeleteDownloader(Job job, IDownloader idownloader) { if (idownloader is WWWDispatch.Downloader) { WWWDispatch.Downloader downloader = (WWWDispatch.Downloader)idownloader; downloader.Job = null; downloader.Download = null; } }
public Updater() { ServiceLocator.Configure(); settingsservice = ServiceLocator.GetInstance<ISettingsService>(); localizationService = ServiceLocator.GetInstance<ILocalizationService>(); downloader = ServiceLocator.GetInstance<IDownloader>(); Configuration = ServiceLocator.GetInstance<IConfiguration>(); ApplySettings(); }
public Updater() { downloader = SimpleIoc.Default.GetInstance<IDownloader>(); isoStorage = SimpleIoc.Default.GetInstance<IIsolatedStorageManager>(); worker = new BackgroundWorker(); worker.DoWork += worker_DoWork; worker.WorkerReportsProgress = true; worker.WorkerSupportsCancellation = false; worker.RunWorkerCompleted += worker_RunWorkerCompleted; worker.ProgressChanged += worker_ProgressChanged; }
public XentoolsUpdate(ISleeper sleeper, IDownloader downloader, IChecksumValidator checksumValidator, IUnzipper unzipper, IInstaller installer, IFinalizer finalizer, IServiceRestarter _serviceRestarter,IConnectionChecker connectionChecker, IAgentUpdateMessageHandler agentUpdateMessageHandler, ILogger logger) { _sleeper = sleeper; _downloader = downloader; _checksumValidator = checksumValidator; _unzipper = unzipper; _installer = installer; _finalizer = finalizer; this._serviceRestarter = _serviceRestarter; _connectionChecker = connectionChecker; _agentUpdateMessageHandler = agentUpdateMessageHandler; _logger = logger; }
public CopyImageLinkAction( ILinkParser linkParser, IImageFileInterpreter imageFileInterpreter, IDownloader downloader, IClipboardInjectionService clipboardInjectionService, IAsyncFilter asyncFilter) { this.asyncFilter = asyncFilter; this.linkParser = linkParser; this.downloader = downloader; this.imageFileInterpreter = imageFileInterpreter; this.clipboardInjectionService = clipboardInjectionService; }
private bool disposedValue = false; // To detect redundant calls #endregion Fields #region Constructors public SonarWebService(IDownloader downloader, string server) { if (downloader == null) { throw new ArgumentNullException("downloader"); } if (string.IsNullOrWhiteSpace(server)) { throw new ArgumentNullException("server"); } this.downloader = downloader; this.server = server.EndsWith("/", StringComparison.OrdinalIgnoreCase) ? server.Substring(0, server.Length - 1) : server; }
public UpdaterUpdate(ISleeper sleeper, IDownloader downloader, IChecksumValidator checksumValidator, IUnzipper unzipper, IFileCopier fileCopier, IFinalizer finalizer, IServiceStopper serviceStopper, IServiceStarter serviceStarter, IConnectionChecker connectionChecker, IAgentUpdateMessageHandler agentUpdateMessageHandler, ILogger logger) { _sleeper = sleeper; _downloader = downloader; _checksumValidator = checksumValidator; _unzipper = unzipper; _fileCopier = fileCopier; _finalizer = finalizer; _serviceStopper = serviceStopper; _serviceStarter = serviceStarter; _connectionChecker = connectionChecker; _agentUpdateMessageHandler = agentUpdateMessageHandler; _logger = logger; }
public UpdateService( IDownloader fileDownloader, IFileManager fileManager, IProcessManager processManager, ILogger logger, IGitHubClientFactory clientFactory) { client = clientFactory.CreateClient(); this.fileDownloader = fileDownloader; this.fileManager = fileManager; this.processManager = processManager; this.logger = logger; }
public UpdateService( IDownloader fileDownloader, IFileManager fileManager, IProcessManager processManager, IEnvironmentInformation environmentInformation, ILogger logger) { client = CreateClient(); this.fileDownloader = fileDownloader; this.fileManager = fileManager; this.processManager = processManager; this.environmentInformation = environmentInformation; this.logger = logger; }
public override void Inject(IDownloader downloader, ISpider spider, bool pauseBeforeInject = true) { if (Driver == null) { return; } if (!CheckFrequency()) { return; } spider.Pause(() => { Login(); spider.Contiune(); }); }
public TumblrBlogCrawler(IShellService shellService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader, ITumblrToTextParser <Post> tumblrJsonParser, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, IPostQueue <TumblrPost> postQueue, IPostQueue <TumblrCrawlerData <Post> > jsonQueue, IBlog blog) : base(shellService, crawlerService, ct, pt, progress, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser, webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog) { this.downloader = downloader; this.tumblrJsonParser = tumblrJsonParser; this.jsonQueue = jsonQueue; this.crawlerDataDownloader = crawlerDataDownloader; }
/// <summary> /// Execute <see cref="ITargetUrlsExtractor"/>. /// </summary> /// <summary xml:lang="zh-CN"> /// 执行目标链接解析器 /// </summary> /// <param name="page">页面数据 <see cref="Page"/></param> /// <param name="downloader">下载器 <see cref="IDownloader"/></param> /// <param name="spider">爬虫 <see cref="ISpider"/></param> public override void Handle(ref Page page, IDownloader downloader, ISpider spider) { if (_targetUrlsExtractor == null || page == null) { return; } var requests = _targetUrlsExtractor.ExtractRequests(page, spider.Site); foreach (var request in requests) { page.AddTargetRequest(request); } page.SkipExtractTargetUrls = !_extractByProcessor; }
public StockUpdateService(IStockQuoteRepository stockQuoteRepository, ILogger logger, IDownloader downloader, IProjectSettings projectSettings, IStockQuotesDownloadService stockQuotesDownloadService, IStocksBulkDeserializer stocksBulkDeserializer, ICompanyRepository companyRepository) { _stockQuoteRepository = stockQuoteRepository; _logger = logger; _downloader = downloader; _projectSettings = projectSettings; _stockQuotesDownloadService = stockQuotesDownloadService; _stocksBulkDeserializer = stocksBulkDeserializer; _companyRepository = companyRepository; }
public DownloadForm(IUpdater updater, IAppCast appcast) { InitializeComponent(); this.updater = updater; this.appcast = appcast; downloader = ServiceLocator.GetInstance<IDownloader>(); installer = ServiceLocator.GetInstance<IInstaller>(); localizationService = ServiceLocator.GetInstance<ILocalizationService>(); ToogleControls(); Localize(); SetTempFileName(); StartDownload(); }
public void Clear() { DownloaderFactory.Dispose(); downloader = null; try { for (int i = 0; i < packetPath.Length; i++) { System.IO.File.Delete(packetPath [i]); dictPacket [i].Clear(); } packetPath = null; dictPacket = null; } catch (Exception e) { Debug.Log(e.Message); } }
protected AstmIOController( IAstmChannel stream, AstmLowLevelSettings lowLevelSettings, IUploader uploader, IDownloader downloader, ITimersManager timersManager, ILogger <AstmIOController> log) { _stream = stream; _uploader = uploader; _downloader = downloader; _lowLevelSettings = lowLevelSettings; _timersManager = timersManager; _log = log; _timersManager.CreateTimer(DISABLE_UPLOAD_TIMER_NAME); }
public static IDisposable SubscribeReport(this IDownloader @this) { const long oneM = 1024 * 1024; const long blockSize = 1024 * 1024 * 20; var previousDateTimeOffset = DateTimeOffset.Now; var previousCompletedSize = 0L; @this .Distinct(item => item.Status) .Timestamp() .Subscribe(item => WriteLine($"{item.Timestamp:HH:mm:ss}: Status is {item.Value.Status}"), error => { WriteLine($"ERROR: {error}"); }); return(@this .Where(item => item.Status == TransferStatus.Transferring) .Sample(TimeSpan.FromMilliseconds(1000)) .Timestamp() .Subscribe(item => { var timestamp = item.Timestamp; var notification = item.Value; var completedSize = @this.GetCompletedSize(); var message = $"{notification.CurrentBlockId:B}: " + $"{notification.Status} " + (notification.CurrentBlockId == Guid.Empty ? string.Empty : $"{@this.BlockContexts[notification.CurrentBlockId].Offset / blockSize:D3} --> {@this.BlockContexts[notification.CurrentBlockId].TotalSize / blockSize} " ) + $"{1.0 * (completedSize - previousCompletedSize) / (timestamp - previousDateTimeOffset).TotalSeconds / oneM:00.00} MB/s " + $"{100.0 * completedSize / @this.Context.TotalSize: 00.00}% " + $"{completedSize:D9}/{@this.Context.TotalSize}"; WriteLine(message); previousCompletedSize = completedSize; previousDateTimeOffset = timestamp; }, error => { WriteLine($"ERROR: {error}"); }, () => { WriteLine($"======= Completed! Time: {DateTimeOffset.Now:yyyy-MM-dd HH:mm:ss} ======="); }));
/// <summary> /// Downloads the specified link. /// </summary> /// <param name="link">The link.</param> /// <param name="token">The token.</param> public void Download(Link link, string token) { if (link.FileURL.StartsWith("magnet:")) { DownloadFileCompleted(null, new EventArgs<string, string, string>(link.FileURL, null, token)); return; } _td = new TaskDialog { Title = "Downloading...", Instruction = link.Release, Content = "Sending request to " + new Uri(link.FileURL).DnsSafeHost.Replace("www.", string.Empty) + "...", CommonButtons = TaskDialogButton.Cancel, ShowProgressBar = true }; _td.SetMarqueeProgressBar(true); _td.Destroyed += TaskDialogDestroyed; _td.ButtonClick += TaskDialogDestroyed; new Thread(() => _res = _td.Show().CommonButton).Start(); var prm = true; _dl = link.Source.Downloader; _dl.DownloadFileCompleted += DownloadFileCompleted; _dl.DownloadProgressChanged += (s, a) => { if (_td != null && _td.IsShowing) { if (prm) { _td.SetMarqueeProgressBar(false); _td.Navigate(_td); prm = false; } _td.Content = "Downloading file... ({0}%)".FormatWith(a.Data); _td.ProgressBarPosition = a.Data; } }; _dl.Download(link, Utils.GetRandomFileName(link.Source.Type == Types.Torrent ? "torrent" : link.Source.Type == Types.Usenet ? "nzb" : null), !string.IsNullOrWhiteSpace(token) ? token : "DownloadFile"); Utils.Win7Taskbar(state: TaskbarProgressBarState.Indeterminate); }
public async Task <IEnumerable <IWorkTask> > WorkAsync(IDownloader downloader, IParserFactory parserFactory) { var html = await downloader.Down(url); var parser = parserFactory.CreateParser("xpath"); parser.Init(html); var keyValuePairs = parser.Parser(new ParaseRule[] { new ParaseRule() { Name = "博客列表", Path = "//div[@class=\"post_item\"]", Rules = new ParaseRule[] { new ParaseRule() { Path = "*//a[@class=\"titlelnk\"]", Name = "标题" }, new ParaseRule() { Path = "*//a[@class=\"titlelnk\"]", Name = "链接", AttrName = "href" } } } }); foreach (var item in keyValuePairs) { Console.WriteLine($"{item}"); } var nextUrlKv = parser.Parser(new ParaseRule[] { new ParaseRule() { Name = "下一页", Path = "//div[@class=\"pager\"]/a[contains(@class,'current')]/following-sibling::a[1]", AttrName = "href" } }); var results = nextUrlKv.Select(x => (IWorkTask) new SpiderTask(x.Value)); return(await Task.FromResult(results)); }
void Start() { #if UNITY_WEBGL && !UNITY_EDITOR Uri baseUri = new Uri(BundleUtil.GetReadOnlyDirectory()); #else DirectoryInfo dir = new DirectoryInfo("./AssetBundles/StandaloneWindows/1.0.0/"); if (!dir.Exists) { Debug.LogFormat("Directory '{0}' does not exist.", dir.FullName); return; } Uri baseUri = new Uri(dir.FullName); #endif this.downloader = new WWWDownloader(baseUri, false); }
public NuGetInstaller(IInstallerFactory factory, IEngineEnvironmentSettings settings, string installPath, IDownloader packageDownloader, IUpdateChecker updateChecker) { Factory = factory ?? throw new ArgumentNullException(nameof(factory)); _environmentSettings = settings ?? throw new ArgumentNullException(nameof(settings)); _packageDownloader = packageDownloader ?? throw new ArgumentNullException(nameof(packageDownloader)); _updateChecker = updateChecker ?? throw new ArgumentNullException(nameof(updateChecker)); if (string.IsNullOrWhiteSpace(installPath)) { throw new ArgumentException($"{nameof(installPath)} should not be null or empty", nameof(installPath)); } if (!_environmentSettings.Host.FileSystem.DirectoryExists(installPath)) { _environmentSettings.Host.FileSystem.CreateDirectory(installPath); } _installPath = installPath; }
public OnlineVersionHelper(UpdateInformationProvider updateInformationProvider, ITempFolderProvider tempFolderProvider, IVersionHelper versionHelper, IUpdateChangeParser changeParser, IFileCacheFactory fileCacheFactory, IDownloader downloader, ICurrentSettings <ApplicationSettings> applicationSettingsProvider) { _updateInformationProvider = updateInformationProvider; _tempFolderProvider = tempFolderProvider; _versionHelper = versionHelper; _changeParser = changeParser; _fileCacheFactory = fileCacheFactory; _downloader = downloader; _applicationSettingsProvider = applicationSettingsProvider; if (_applicationSettingsProvider != null) { _fileCache = GetFileCache(); _applicationSettingsProvider.SettingsChanged += ApplicationSettingsProviderOnSettingsChanged; } _onlineVersion = new ApplicationVersion(new System.Version(0, 0, 0, 0), "", "", new List <Release>()); }
public RssHolder(IDownloader loader, IParser parser, IEventAggregator aggregator) { this.aggregator = aggregator; this.aggregator.Subscribe(this); this.parser = parser; this.loader = loader; TimerCallback callback = this.RefreshOnTime; this.timer = new Timer(callback, null, 60000, 3000); var conn = new SQLiteConnection("feedDB.db"); conn.CreateTable <RssItem>(); conn.CreateTable <RssFeed>(); var savedHeaders = conn.Query <RssFeed>("select * from RssFeed"); newsHeaders = new ObservableCollection <RssFeed>(savedHeaders); }
private async Task downloadAll(ImageDownloadQueue queue, IDownloader downloader, CancellationToken token) { while (true) { if (_abort) { return; } var task = queue.PopTaskFor(downloader); if (task == null) { return; } if (isAlreadyDownloaded(task)) { Console.WriteLine("[Skip] {0} {1}", task.QualityGroup.Name ?? string.Empty, task.Dir.Subdir); Interlocked.Add(ref _countInDownloadedDirs, task.FilesOnline.Count); ProgressChanged?.Invoke(); } else { Interlocked.Add(ref _countInDownloadedDirs, task.FilesDownloaded.Count); ProgressChanged?.Invoke(); bool success = await downloader.Download(task, token); if (success) { ImageDownloadProgressReader.WriteExistingSignatures(task); } else { if (queue.PushFailedTaskBack(downloader, task)) { Console.WriteLine("Other download source available for {0}", task.Dir.Subdir); } else { Console.WriteLine("No other download source available for {0}", task.Dir.Subdir); } } } } }
public override void Handle(ref Page page, IDownloader downloader, ISpider spider) { if (string.IsNullOrEmpty(page?.Content)) { return; } string textValue = string.Empty; MatchCollection collection = Regex.Matches(page.Content, _pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in collection) { textValue += item.Value; } page.Content = textValue; }
public void Init() { int packets = RandomData.Random.Next(3, 6); packetPath = new string[packets]; dictPacket = new Dictionary <string, byte[]> [packets]; versions = new int[packets]; int fileOffset = 0; int version = 0; for (int i = 0; i < packets; i++) { dictPacket [i] = new Dictionary <string, byte[]> (); string where = (i == 0) ? Application.streamingAssetsPath : Application.persistentDataPath; string postFix = (i == 0) ? "_bass.pak" : string.Format("_update_{0}.pak", i); versions [i] = version; packetPath [i] = System.IO.Path.Combine(where, packetName + postFix).Replace("\\", "/"); PreparePacket(dictPacket [i], packetPath [i], versions [i], fileOffset); int adder = RandomData.Random.Next(1, dictPacket [i].Count - 1); fileOffset += adder; version++; } // 初始化时,数据包必须已经存在。 DownloaderFactory.Instance.Initialize(DownloadMode.Packet, packetName); downloader = DownloaderFactory.Instance.Downloader; // 收集最新版本的文件字典。版本号是按照从小到大排列好的。 for (int i = dictPacket.Length - 1; i >= 0; i--) { Dictionary <string, byte[]> .Enumerator e = dictPacket [i].GetEnumerator(); while (e.MoveNext()) { string f = e.Current.Key; byte[] d = e.Current.Value; if (!dictNewestPacket.ContainsKey(f)) { dictNewestPacket.Add(f, d); } } } }
public IDownloader Download(string torrentUri, string savePath) { // Get the associated downloader ObjectPath path = engine.RegisterTorrent(torrentUri, savePath); IDownloader downloader = bus.GetObject <IDownloader> (BusName, path); if (downloader.GetState() == TorrentState.Stopped) { downloader.Start(); Console.WriteLine("Started: {0}", downloader.GetPath()); } else { Console.WriteLine("{0} already running", downloader.GetPath()); } return(downloader); }
/// <summary> /// 创建下载器 /// </summary> /// <param name="type">下载器类型</param> /// <param name="start">开始时间</param> /// <param name="end">结束时间</param> /// <returns></returns> public static IDownloader CreateDownloader(NewsPaperTypeEnum type, DateTime start, DateTime end) { IDownloader downloader = null; switch (type) { case NewsPaperTypeEnum.ChinaTeacher: downloader = new TeacherDownLoader(start, end); break; case NewsPaperTypeEnum.ChinaEducation: downloader = new EducationDownLoader(start, end); break; } return(downloader); }
public TumblrHiddenCrawler(IShellService shellService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader, ITumblrToTextParser <Post> tumblrJsonParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IPostQueue <TumblrPost> postQueue, IPostQueue <TumblrCrawlerData <Post> > jsonQueue, IBlog blog) : base(shellService, ct, progress, webRequestFactory, cookieService, postQueue, blog) { this.crawlerService = crawlerService; this.downloader = downloader; this.pt = pt; this.tumblrJsonParser = tumblrJsonParser; this.imgurParser = imgurParser; this.gfycatParser = gfycatParser; this.webmshareParser = webmshareParser; this.jsonQueue = jsonQueue; this.crawlerDataDownloader = crawlerDataDownloader; }
private async Task RunCrawlerTasks(CancellationToken ct, PauseToken pt) { while (true) { ct.ThrowIfCancellationRequested(); if (pt.IsPaused) { pt.WaitWhilePausedWithResponseAsyc().Wait(ct); } Monitor.Enter(lockObject); if (crawlerService.ActiveItems.Count() < QueueManager.Items.Count()) { IEnumerable <QueueListItem> queueList = QueueManager.Items.Except(crawlerService.ActiveItems); QueueListItem nextQueueItem = queueList.First(); IBlog blog = nextQueueItem.Blog; IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog); downloader.IsBlogOnlineAsync().Wait(4000); if (crawlerService.ActiveItems.Any(item => item.Blog.Name.Contains(nextQueueItem.Blog.Name))) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(lockObject); continue; } if (!nextQueueItem.Blog.Online) { QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem)); Monitor.Exit(lockObject); continue; } crawlerService.AddActiveItems(nextQueueItem); Monitor.Exit(lockObject); await StartSiteSpecificDownloader(nextQueueItem, ct, pt); } else { Monitor.Exit(lockObject); await Task.Delay(4000, ct); } } }
public AgentService( IMessageQueue messageQueue, IOptions <AgentOptions> options, IHostApplicationLifetime applicationLifetime, IDownloader downloader, HostBuilderContext hostBuilderContext, ILogger <AgentService> logger, IStatisticsClient statisticsClient) { _options = options.Value; _logger = logger; _statisticsClient = statisticsClient; _messageQueue = messageQueue; _applicationLifetime = applicationLifetime; _downloader = downloader; _consumers = new List <AsyncMessageConsumer <byte[]> >(); hostBuilderContext.Properties[Const.DefaultDownloader] = downloader.Name; }
/// <summary> /// Searches the current content for all occurrences of a specified regular expression, using the specified matching options. /// </summary> /// <param name="page">页面数据</param> /// <param name="downloader">下载器</param> /// <param name="spider">爬虫</param> public override void Handle(ref Page page, IDownloader downloader, ISpider spider) { if (page == null || string.IsNullOrWhiteSpace(page.Content)) { return; } string textValue = string.Empty; MatchCollection collection = Regex.Matches(page.Content, _pattern, _regexOptions); foreach (Match item in collection) { textValue += item.Value; } page.Content = textValue; }
public UpdateService( IDownloader fileDownloader, IFileManager fileManager, IProcessManager processManager, ILogger logger, IGitHubClientFactory clientFactory, IEnvironmentInformation environmentInformation, ISettingsManager settingsManager) { client = clientFactory.CreateClient(); this.fileDownloader = fileDownloader; this.fileManager = fileManager; this.processManager = processManager; this.logger = logger; this.environmentInformation = environmentInformation; this.settingsManager = settingsManager; }
/// <summary> /// 当页面数据中的异常信息包含指定内容时触发ADSL拨号 /// </summary> /// <param name="page">页面数据</param> /// <param name="downloader">下载器</param> /// <param name="spider">爬虫</param> public override void Handle(ref Page page, IDownloader downloader, ISpider spider) { if (page == null || string.IsNullOrWhiteSpace(page.Content) || page.Exception == null) { return; } if (page.Exception.Message.Contains(_exceptionMessage)) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { Logger.Log(spider.Identity, "Exit program because redial failed.", Level.Error); spider.Exit(); } Spider.AddToCycleRetry(page.Request, spider.Site); page.Exception = new DownloadException("Download failed and redial finished already."); } }
/// <summary> /// 開始任務 /// </summary> public bool Start() { if (BasePlugin == null) { Status = DownloadStatus.Error; throw new Exception("Plugin Not Found"); } if (_downloader == null) { _downloader = BasePlugin.CreateDownloader(); _downloader.TaskInfo = this; } //resourceDownloader = BasePlugin.CreateDownloader(); //resourceDownloader.taskInfo = this; return(_downloader.Download()); }
public static void ClassInit(TestContext context) { httpListener.Prefixes.Add("http://localhost:5000/"); httpListener.Prefixes.Add("http://127.0.0.1:5000/"); httpListener.Start(); EndpointUrl = "http://localhost:5000/"; ServerFilesDirectory = Directory.GetCurrentDirectory() + "/../../TestServerFiles/"; DownloadedFilesDirectory = Directory.GetCurrentDirectory() + "/../../TestDownloadedFiles/"; BackgroundWorker BW = new BackgroundWorker { WorkerReportsProgress = true }; Hasher = new Md5Hasher(); Downloader = new HttpClientDownloader(BW, Hasher); Server = new Thread(ServerThread); Server.Start(); new FileInfo(DownloadedFilesDirectory).Directory.Create(); }
public DefaultFeedManager(IMessenger messenger, IUrlFetcher urlFetcher, IOptions options, IProxy proxy, IDownloader downloader, ILeague league) { _messenger = messenger; _urlFetcher = urlFetcher; _options = options; _proxy = proxy; _league = league; _downloader = downloader; _startDate = _endDate = (options.Date != null) ? DateTime.Parse(options.Date) : DateTime.Now; _startDate = _startDate.Subtract(new TimeSpan(24 * options.Days, 0, 0)); ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => { return(true); }; }
public override void Handle(ref Page page, IDownloader downloader, ISpider spider) { if (!string.IsNullOrEmpty(page?.Content)) { var content = page.Content; var containContent = _contents.FirstOrDefault(c => content.Contains(c)); if (containContent != null) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { Logger.AllLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error); spider.Exit(); } page = Spider.AddToCycleRetry(page.Request, spider.Site); page.Exception = new DownloadException($"Content downloaded contains string: {containContent}."); } } }
protected AbstractCrawler(IShellService shellService, ICrawlerService crawlerService, IProgress <DownloadProgress> progress, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, IPostQueue <AbstractPost> postQueue, IBlog blog, IDownloader downloader, PauseToken pt, CancellationToken ct) { ShellService = shellService; CrawlerService = crawlerService; WebRequestFactory = webRequestFactory; CookieService = cookieService; PostQueue = postQueue; Blog = blog; Downloader = downloader; Progress = progress; Pt = pt; // TODO: Find a better way for this construct InterruptionTokenSource = new CancellationTokenSource(); LinkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, InterruptionTokenSource.Token); Ct = LinkedTokenSource.Token; }
private void btnCheckUrl_Click(object sender, EventArgs e) { // Initialize an instance of HttpDownloadClient. downloader = new HttpDownloadClient(tbURL.Text); // Register the events of HttpDownloadClient. downloader.DownloadCompleted += DownloadCompleted; downloader.DownloadProgressChanged += DownloadProgressChanged; downloader.StatusChanged += StatusChanged; try { string filename = string.Empty; downloader.CheckUrl(out filename); if (string.IsNullOrEmpty(filename)) { this.DownloadPath = string.Format("{0}\\{1}", Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), downloader.Url.Segments.Last()); } else { this.DownloadPath = string.Format("{0}\\{1}", Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), filename); } this.tbURL.Enabled = false; this.btnCheckUrl.Enabled = false; this.tbPath.Enabled = true; this.btnDownload.Enabled = true; } catch { // If there is any exception, like System.Net.WebException or // System.Net.ProtocolViolationException, it means that there may be an // error while reading the information of the file and it cannot be // downloaded. MessageBox.Show("There is an error while get the information of the file." + " Please make sure the url is accessible."); } }
/// <summary> /// Asynchronously submits a task to the downloader. /// </summary> /// <param name="task">The DownloadTask instance.</param> private void SubmitTaskAsync(DownloadTask task) { IDownloader downloader = GetDownloader(task); if (FeedSource.EnclosureCacheSize != FeedSource.DefaultEnclosureCacheSize) { //BUGBUG: If we don't know the file size then we don't error because we treat it is -1 to //err on the side of allowing instead of disallowing. long limitInBytes = FeedSource.EnclosureCacheSize * 1024 * 1024; long filesize = GetFileSize(task); var targetDir = new DirectoryInfo(FeedSource.EnclosureFolder); long spaceUsed = FileHelper.GetSize(targetDir); if (!String.Equals(FeedSource.EnclosureFolder, FeedSource.PodcastFolder, StringComparison.OrdinalIgnoreCase)) { DirectoryInfo podcastDir = new DirectoryInfo(FeedSource.PodcastFolder); spaceUsed += FileHelper.GetSize(podcastDir); } if ((filesize + spaceUsed) > limitInBytes) { DownloadRegistryManager.Current.UnRegisterTask(task); string fileName = task.DownloadItem.File.LocalName; int limit = FeedSource.EnclosureCacheSize; throw new DownloaderException(String.Format(ComponentsText.ExceptionEnclosureCacheLimitReached, fileName, limit)); } } // If there's no files to download // Consider the download as done if ((downloader == null) || (task.DownloadItem.File == null)) { OnDownloadCompleted(null, new TaskEventArgs(task)); } else { downloader.BeginDownload(task); } }
public void Setup() { _agentUpdateInfo = "http://something.com/file.zip,544564abc453de787ad"; _downloader = MockRepository.GenerateMock<IDownloader>(); _checksumValidator = MockRepository.GenerateMock<IChecksumValidator>(); _unzipper = MockRepository.GenerateMock<IUnzipper>(); _installer = MockRepository.GenerateMock<IInstaller>(); _finalizer = MockRepository.GenerateMock<IFinalizer>(); _connectionChecker = MockRepository.GenerateMock<IConnectionChecker>(); _sleeper = MockRepository.GenerateMock<ISleeper>(); _logger = MockRepository.GenerateMock<ILogger>(); _serviceRestarter = MockRepository.GenerateMock<IServiceRestarter>(); _agentUpdateMessageHandler = new AgentUpdateMessageHandler(); _logger.Stub(x => x.Log(Arg<string>.Is.Anything)); _xentoolsUpdate = new XentoolsUpdate(_sleeper, _downloader, _checksumValidator, _unzipper, _installer, _finalizer, _serviceRestarter, _connectionChecker, _agentUpdateMessageHandler, _logger); }
/// <summary> /// Check whether the destination file exists. If not, create a file with the same /// size as the file to be downloaded. /// </summary> public static void CheckFileOrCreateFile(IDownloader downloader, object fileLocker) { // Lock other threads or processes to prevent from creating the file. lock (fileLocker) { FileInfo fileToDownload = new FileInfo(downloader.DownloadPath); if (fileToDownload.Exists) { // The destination file should have the same size as the file to be downloaded. if (fileToDownload.Length != downloader.TotalSize) { throw new ApplicationException( "The download path already has a file which does not match" + " the file to download. "); } } // Create a file. else { if (downloader.TotalSize == 0) { throw new ApplicationException("The file to download does not exist!"); } using (FileStream fileStream = File.Create(downloader.DownloadPath)) { long createdSize = 0; byte[] buffer = new byte[4096]; while (createdSize < downloader.TotalSize) { int bufferSize = (downloader.TotalSize - createdSize) < 4096 ? (int)(downloader.TotalSize - createdSize) : 4096; fileStream.Write(buffer, 0, bufferSize); createdSize += bufferSize; } } } } }
/// <summary> /// Downloads the specified link. /// </summary> /// <param name="link">The link.</param> public void Download(Subtitle link) { _td = new TaskDialog { Title = "Downloading...", Instruction = link.Release, Content = "Sending request to " + new Uri(link.FileURL ?? link.InfoURL).DnsSafeHost.Replace("www.", string.Empty) + "...", CommonButtons = TaskDialogButton.Cancel, ShowProgressBar = true }; _td.SetMarqueeProgressBar(true); _td.Destroyed += TaskDialogDestroyed; _td.ButtonClick += TaskDialogDestroyed; new Thread(() => _res = _td.Show().CommonButton).Start(); var prm = true; _dl = link.Source.Downloader; _dl.DownloadFileCompleted += DownloadFileCompleted; _dl.DownloadProgressChanged += (s, a) => { if (_td != null && _td.IsShowing) { if (prm) { _td.SetMarqueeProgressBar(false); _td.Navigate(_td); prm = false; } _td.Content = "Downloading file... ({0}%)".FormatWith(a.Data); _td.ProgressBarPosition = a.Data; } }; _dl.Download(link, Utils.GetRandomFileName()); Utils.Win7Taskbar(state: TaskbarProgressBarState.Indeterminate); }
/// <summary> /// Only constructor for DownloaderManager. All parameters required for proper function. /// </summary> /// <param name="downloader">instance of IDownloader implementation</param> /// <param name="validator">instance of IValidator implementation</param> /// <param name="downloadJob">encapsulation of Job information in DownloadJobStatusEntry instance, such as app name job status job id</param> /// <param name="mustStop">an MRE used by Updater to tell this object it must clean up and leave or be Interrupt()ed</param> /// <param name="badExitCbk">delegate instance this class will call on on unrecoverable error, such as an unanticpated exception that compromises too badly to contine</param> public DownloaderManager( IDownloader downloader, IValidator validator, DownloadJobStatusEntry downloadJob, ManualResetEvent mustStop, BadExitCallback badExitCbk ) { _downloader = downloader; _validator = validator; _dnldJob = downloadJob; _mustStopUpdating = mustStop; // using our app's name, look up the application object (ApplicationConfiguration) in array of applications-- // since WE are working on one, but there may be many _application = UpdaterConfiguration.Instance[ _dnldJob.ApplicationName ]; // set out internal delegate instance of BadExitCallback. // this is the function pointer we'll use to notify UpdaterManager of really poor exits (fatal exceptions etc.) _badExitCbk = badExitCbk; }
/// <summary> /// Проверить, существует ли файл назначения. Если нет, то создайте файл с тем же размер файла для загрузки. /// </summary> public static void CheckFileOrCreateFile(IDownloader downloader, object fileLocker) { lock (fileLocker) { FileInfo fileToDownload = new FileInfo(downloader.DownloadPath); if (fileToDownload.Exists) { if (fileToDownload.Length != downloader.TotalSize) { downloader.Status = StateDownload.Nofile; throw new ApplicationException("Загрузка невозможна"); } } // Создать файл else { if (downloader.TotalSize == 0) { downloader.Status = StateDownload.Nofile; throw new ApplicationException("Файл невозможно создать"); } using (FileStream fileStream = File.Create(downloader.DownloadPath)) { long createdSize = 0; byte[] buffer = new byte[4096]; while (createdSize < downloader.TotalSize) { int bufferSize = (downloader.TotalSize - createdSize) < 4096 ? (int)(downloader.TotalSize - createdSize) : 4096; fileStream.Write(buffer, 0, bufferSize); createdSize += bufferSize; } } } } }
public DownloadViewModel(IDownloader downloader, ChapterRecord chapter) : base() { if (downloader == null) throw new ArgumentNullException("downloader"); if (chapter == null) throw new ArgumentNullException("chapter"); _downloader = downloader; _chapter = chapter; // register downloader events _downloader.DownloadProgress += _downloader_DownloadProgress; _downloader.DownloadCompleted += _downloader_DownloadCompleted; _cancelDownloadCommand = new CancelDownloadCommand(this); _removeDownloadCommand = new RemoveDownloadCommand(this); _openDownloadCommand = new OpenDownloadCommand(this); State = DownloadState.Ok; Completed = false; CancelText = ButtonCancelText; }
public YBSquareService(IApiConfiguration configuration, IDownloader downloader) { _configuration = configuration; _downloader = downloader; }
/// <summary> /// 開始任務 /// </summary> public bool Start() { if (BasePlugin == null) { Status = DownloadStatus.Error; throw new Exception("Plugin Not Found"); } if (_downloader == null) { _downloader = BasePlugin.CreateDownloader(); _downloader.TaskInfo = this; } //resourceDownloader = BasePlugin.CreateDownloader(); //resourceDownloader.taskInfo = this; return _downloader.Download(); }
/// <summary> /// Проверить URL /// </summary> /// <param name="downloader"></param> /// <returns></returns> public static string CheckUrl(IDownloader downloader) { string fileName = string.Empty; var webRequest = InitializeHttpWebRequest(downloader); using (var response = webRequest.GetResponse()) { foreach (var header in response.Headers.AllKeys) { if (header.Equals("Accept-Ranges", StringComparison.OrdinalIgnoreCase)) { downloader.IsRangeSupported = true; } if (header.Equals("Content-Disposition", StringComparison.OrdinalIgnoreCase)) { string contentDisposition = response.Headers[header]; string pattern = ".[^;]*;\\s+filename=\"(?<file>.*)\""; Regex r = new Regex(pattern); Match m = r.Match(contentDisposition); if (m.Success) { fileName = m.Groups["file"].Value; } } } //-------- // downloader.IsRangeSupported = true; fileName = MessageClientServer.GenerateFileNameFromUri(downloader.Url.AbsoluteUri); //----------- downloader.TotalSize = response.ContentLength; if (downloader.TotalSize <= 0) { downloader.Status = StateDownload.Nofile; throw new ApplicationException("Файл не найден"); } if (!downloader.IsRangeSupported) { downloader.StartPoint = 0; downloader.EndPoint = int.MaxValue; } } if (downloader.IsRangeSupported && (downloader.StartPoint != 0 || downloader.EndPoint != long.MaxValue)) { webRequest = InitializeHttpWebRequest(downloader); if (downloader.EndPoint != int.MaxValue) { webRequest.AddRange(downloader.StartPoint, downloader.EndPoint); } else { webRequest.AddRange(downloader.StartPoint); } using (var response = webRequest.GetResponse()) { downloader.TotalSize = response.ContentLength; } } return fileName; }
public static HttpWebRequest InitializeHttpWebRequest(IDownloader downloader) { var webRequest = (HttpWebRequest)WebRequest.Create(downloader.Url); if (downloader.Credentials != null) { webRequest.Credentials = downloader.Credentials; } else { webRequest.Credentials = CredentialCache.DefaultCredentials; } if (downloader.Proxy != null) { webRequest.Proxy = downloader.Proxy; } else { webRequest.Proxy = WebRequest.DefaultWebProxy; } return webRequest; }
public void SetWebClient(IDownloader client) { _downloader = client; }
/// <summary> /// Event handler for <c>FileSearch.FileSearchDone</c>. /// </summary> /// <param name="sender">The sender.</param> /// <param name="e">The <see cref="RoliSoft.TVShowTracker.EventArgs<System.Collections.Generic.List<System.String>>"/> instance containing the event data.</param> private void NearVideoFileSearchDone(object sender, EventArgs<List<string>> e) { _files = e.Data ?? new List<string>(); if (_files.Count == 0) { Utils.Win7Taskbar(state: TaskbarProgressBarState.NoProgress); if (_td != null && _td.IsShowing) { _td.SimulateButtonClick(-1); } if (_res == Result.Cancel) { return; } new TaskDialog { CommonIcon = TaskDialogIcon.Stop, Title = "No files found", Instruction = _link.Release, Content = "No files were found for this episode.\r\nUse the first option to download the subtitle and locate the file manually.", CommonButtons = TaskDialogButton.OK }.Show(); return; } _td.Content = "Sending request to " + new Uri(_link.FileURL ?? _link.InfoURL).DnsSafeHost.Replace("www.", string.Empty) + "..."; var prm = true; _dl = _link.Source.Downloader; _dl.DownloadFileCompleted += NearVideoDownloadFileCompleted; _dl.DownloadProgressChanged += (s, a) => { if (_td != null && _td.IsShowing) { if (prm) { _td.SetMarqueeProgressBar(false); _td.Navigate(_td); prm = false; } _td.Content = "Downloading file... ({0}%)".FormatWith(a.Data); _td.ProgressBarPosition = a.Data; } }; _dl.Download(_link, Utils.GetRandomFileName()); }
public RssFeed(string url, IDownloader downloader) { _url = url; _downloader = downloader; }
private bool InstallList(HashSet<string> toInstall, RelationshipResolverOptions options, IDownloader downloader) { if (toInstall.Any()) { // actual magic happens here, we run the installer with our mod list var module_installer = ModuleInstaller.GetInstance(manager.CurrentInstance, GUI.user); module_installer.onReportModInstalled = OnModInstalled; return WasSuccessful( () => module_installer.InstallList(toInstall.ToList(), options, downloader)); } return true; }