public dynamic Load(string url) { using (GZipWebClient client = new GZipWebClient()) { return(client.DownloadString(url)); } }
public XmlDocument GetXml(string url, string referer = null) { _logger.Info($"Getting XML from url: {url}"); var client = new GZipWebClient(); if (!string.IsNullOrEmpty(referer)) client.Headers.Add("Referer", referer); var xmlDoc = new XmlDocument(); try { var xml = client.DownloadString(url); xmlDoc.LoadXml(xml); } catch (Exception ex) { _logger.Error(ex); } return xmlDoc; }
public static HtmlDocument Nacti(String sport, String manualniAdresa) { /*------vrati HTML pro livestranku daneho sportu--------------------------------------------*/ if (sport == "H") { //----pro hokej se to dela jinak--- return null; } //Encoding kodovani = Encoding.GetEncoding(28591); latin1 Encoding kodovani = Encoding.GetEncoding("utf-8"); GZipWebClient mujWebKlient = new GZipWebClient(); mujWebKlient.Headers.Add("Accept-Language", "cs-CZ,cs;q=0.8"); mujWebKlient.Headers.Add("Accept-Encoding", "gzip,deflate,sdch"); mujWebKlient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17"); String liveURL; if (manualniAdresa != "") { liveURL = manualniAdresa; } else { liveURL = knihovna_ser.DejConfig("liveURL_" + sport); } HtmlWeb webovaStranka = new HtmlWeb(); HtmlDocument docHtml = new HtmlDocument(); docHtml.Load(mujWebKlient.OpenRead(liveURL), kodovani); return docHtml; }
public void GetKnownRepositories(Action <ObservableCollection <string> > callback, bool forceUpdate = false) { if (_cache.Count > 0 && !forceUpdate) { callback(new ObservableCollection <string>(_cache)); return; } Log.Info("Update Known Repositories " + Url); Task.Factory.StartNew(() => { try { using (var client = new GZipWebClient()) { var repos = client.DownloadString(Url); var matches = Regex.Matches(repos, "<repo>(.*)</repo>"); _cache.Clear(); foreach (Match match in matches) { _cache.Add(match.Groups[1].ToString()); } callback(new ObservableCollection <string>(_cache)); } } catch (Exception e) { Log.Warn(e); } }); }
private async void codeList_Tap(object sender, System.Windows.Input.GestureEventArgs e) { if (codeList.SelectedItem == null) { return; } CheatText cheatText = (CheatText)codeList.SelectedItem; codeList.SelectedItem = null; if (cheatText.Url != null && cheatText.Url != "") //need to go to the link to obtain the code { GZipWebClient webClient = new GZipWebClient(); string response = null; SystemTray.GetProgressIndicator(this).IsIndeterminate = true; try { //USE THIRD-PARTY GZIPWEBCLIENT webClient.Headers[HttpRequestHeader.Accept] = "text/html, application/xhtml+xml, */*"; webClient.Headers[HttpRequestHeader.Referer] = "http://www.supercheats.com/search.php"; webClient.Headers[HttpRequestHeader.AcceptLanguage] = "en-US"; webClient.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko"; webClient.Headers[HttpRequestHeader.AcceptEncoding] = "gzip, deflate"; webClient.Headers[HttpRequestHeader.Connection] = "Keep-Alive"; webClient.Headers[HttpRequestHeader.Host] = "www.supercheats.com"; response = await webClient.DownloadStringTaskAsync(new Uri(cheatText.Url, UriKind.Absolute)); if (response == null) { SystemTray.GetProgressIndicator(this).IsIndeterminate = false; return; } Match contentMatch = Regex.Match(response, "(?<=<div id='sub).*?(?=</div>)", RegexOptions.Singleline); string content = contentMatch.Value; int index = content.IndexOf(">"); //get the position of the closing bracket content = content.Substring(index + 1); //get rid of the closing bracket cheatText.TextHtml += "<p style=\"font-size:22px\">" + content + "</p></body></html>"; } catch (Exception ex) { MessageBox.Show(ex.Message); } SystemTray.GetProgressIndicator(this).IsIndeterminate = false; } cheatTextStackpanel.DataContext = cheatText; cheatTextBox.NavigateToString(cheatText.TextHtml); gameList.Visibility = Visibility.Collapsed; codeList.Visibility = Visibility.Collapsed; cheatTextStackpanel.Visibility = Visibility.Visible; }
public void LoadData(string lastItem) { if (IsLoading) { return; } IsLoading = true; if (BeginLoading != null) { BeginLoading(); } string uriString = "http://www.reddit.com/r/todayilearned.json"; if (!string.IsNullOrEmpty(lastItem)) { uriString += "?after=" + lastItem; } var uri = new Uri(uriString); var client = new GZipWebClient(); client.Headers["User-Agent"] = "windowsphone:com.trivia.buff:v1.8.9 (by /u/camradal)"; client.DownloadStringCompleted += client_DownloadStringCompleted; client.DownloadStringAsync(uri); }
void Btn_Scrape_Click(object sender, EventArgs e) { /* * HtmlWeb Hw = new HtmlWeb(); * var Hd = Hw.Load(this.Txt_Url.Text); * var Links = Hd.DocumentNode.SelectNodes("//a"); * foreach (var Item_Link in Links) * { * if (Item_Link.Attributes.Contains("href")) * { * if (Item_Link.Attributes["href"].Value.StartsWith(@"http://mangafox.me/manga/")) * { Debug.WriteLine(Item_Link.Attributes["href"].Value); } * } * } */ String Url = this.Txt_Url.Text; String IPEndPoint_IPAddress = this.Txt_EndPointIPAddress.Text; String html; using (var wc = new GZipWebClient(IPEndPoint_IPAddress)) { html = wc.DownloadString(Url); } this.Txt_Chapters.Text = html; }
public void DownloadFile(string url, string fileName) { try { var fileInfo = new FileInfo(fileName); if (fileInfo.Directory != null && !fileInfo.Directory.Exists) { fileInfo.Directory.Create(); } _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); var stopWatch = Stopwatch.StartNew(); var uri = new HttpUri(url); using (var webClient = new GZipWebClient()) { webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); webClient.Proxy = GetProxy(uri); webClient.DownloadFile(uri.FullUri, fileName); stopWatch.Stop(); _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); } } catch (WebException e) { _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); throw; } catch (Exception e) { _logger.Warn(e, "Failed to get response from: " + url); throw; } }
public void Search(string term, string lastItem) { if (IsSearching) { return; } IsSearching = true; if (BeginLoading != null) { BeginLoading(); } string uriString = "http://www.reddit.com/r/todayilearned/search.json?sort=relevance&restrict_sr=on&t=all&q=" + term; if (!string.IsNullOrEmpty(lastItem)) { uriString += "&after=" + lastItem; } else { this.SearchItems.Clear(); } var uri = new Uri(uriString); var client = new GZipWebClient(); client.Headers["User-Agent"] = "windowsphone:com.trivia.buff:v1.8.9 (by /u/camradal)"; client.DownloadStringCompleted += client_DownloadSearchStringCompleted; client.DownloadStringAsync(uri); }
private XDocument retrieveXML(string type, string date, string venue, int raceno) { string xml; using (var webClient = new GZipWebClient()) { webClient.QueryString.Add("type", type); webClient.QueryString.Add("date", date); webClient.QueryString.Add("venue", venue); if (type == "pooltot") { webClient.QueryString.Add("raceno", raceno.ToString()); } if (type == "jcbwracing_winplaodds") { webClient.QueryString.Add("start", raceno.ToString()); webClient.QueryString.Add("end", raceno.ToString()); } //webClient.Encoding = Encoding.UTF8; xml = webClient.DownloadString(BaseUrl); return(XDocument.Parse(xml)); } }
public XmlDocument GetXml(string url, string referer = null) { _logger.Info($"Getting XML from url: {url}"); var client = new GZipWebClient(); if (!string.IsNullOrEmpty(referer)) { client.Headers.Add("Referer", referer); } var xmlDoc = new XmlDocument(); try { var xml = client.DownloadString(url); xmlDoc.LoadXml(xml); } catch (Exception ex) { _logger.Error(ex); } return(xmlDoc); }
public void GetAssemblyDatabase(Action <ObservableCollection <LSharpDbAssembly> > callback, bool forceUpdate = false) { if (_cache.Count > 0 && !forceUpdate) { callback(new ObservableCollection <LSharpDbAssembly>(_cache)); return; } Log.Info("Update Assembly Database " + Url); Task.Factory.StartNew(() => { try { using (var client = new GZipWebClient()) { var data = JsonConvert.DeserializeObject <List <LSharpDbAssembly> >(client.DownloadString(Url)); _cache.Clear(); foreach (var assembly in data) { _cache.Add(assembly); } callback(new ObservableCollection <LSharpDbAssembly>(_cache)); } } catch (Exception e) { Log.Warn(e); } }); }
/// <summary> /// Use this to get the whole page content /// </summary> /// <param name="pageURL"></param> /// <returns></returns> public static PageItem ProcessWebPage(String pageURL) { PageItem pageItem = null; try { // Setup the configuration to support document loading var config = Configuration.Default.WithDefaultLoader(); // Load the names of all The Big Bang Theory episodes from Wikipedia using (GZipWebClient client = new GZipWebClient()) { client.Headers.Add("user-agent", Analyzer.Common.Configuration.ConfigurationManager.AppSettings.UserAgentOptions.GetRandom()); pageItem = new PageItem(); string webLocationContent = client.DownloadString(pageURL); pageItem.Content = Analyzer.Common.HtmlRemoval.StripTagsCharArray(webLocationContent); pageItem.Url = pageURL; } } catch (Exception ex) { Analyzer.Common.Logger.ExceptionLoggingService.Instance.WriteError("Error in processing given wen URL: " + pageURL, ex); pageItem = null; } return(pageItem); }
public override void Download() { // NOTE ultimate guitar service prohibits unauthorized users to download raw gp files // By now, we are do not force users to create any accounts as there is another way to get gp-files // unfortunately, these files are json-serialized representation of gp, so we need to perform additional manipulations if (Entry.Type == "guitar pro") { GetGuitarProDownloadUrl(Entry.Id).ContinueWith(task => { var link = task.Result; if (link != string.Empty) { GZipWebClient client = new GZipWebClient(); client.DownloadStringAsync(new Uri(link)); client.DownloadStringCompleted += (o, e) => { // convert from json to guitar pro format var songReader = new JsonSongReader(e.Result); var song = songReader.ReadSong(); var songWriter = new FifthGuitarProSongWriter(new BinaryWriter(GetOutputStream())); songWriter.WriteSong(song); InvokeDownloadComplete(new DownloadCompletedEventArgs(false)); }; } }); } else { Download(WebRequest.Create(Entry.Url)); } }
public void Run( MainForm form ) { using( WebClient client = new GZipWebClient() ) { WebRequest.DefaultWebProxy = null; int i = 0; DownloadData( classicJarUri, client, "classic.jar", form, ref i ); DownloadData( modernJarUri, client, "1.6.2.jar", form, ref i ); DownloadData( terrainPatchUri, client, "terrain-patch.png", form, ref i ); } reader = new ZipReader(); reader.ShouldProcessZipEntry = ShouldProcessZipEntry_Classic; reader.ProcessZipEntry = ProcessZipEntry_Classic; using( FileStream srcClassic = File.OpenRead( "classic.jar" ), srcModern = File.OpenRead( "1.6.2.jar" ), dst = new FileStream( "default.zip", FileMode.Create, FileAccess.Write ) ) { writer = new ZipWriter( dst ); reader.Extract( srcClassic ); // Grab animations and snow animBitmap = new Bitmap( 1024, 64, PixelFormat.Format32bppArgb ); reader.ShouldProcessZipEntry = ShouldProcessZipEntry_Modern; reader.ProcessZipEntry = ProcessZipEntry_Modern; reader.Extract( srcModern ); writer.WriteNewImage( animBitmap, "animations.png" ); writer.WriteNewString( animationsTxt, "animations.txt" ); animBitmap.Dispose(); writer.WriteCentralDirectoryRecords(); } if( !File.Exists( "terrain-patched.png" ) ) File.Move( "terrain-patch.png", "terrain-patched.png" ); }
private static string getWeb(string url) { GZipWebClient webClient = new GZipWebClient(); webClient.Encoding = System.Text.Encoding.UTF8; return(webClient.DownloadString(url)); }
public void Check(Action <UpdateResponse> callback) { var leagueMd5 = Utility.Md5Checksum(Config.Instance.LeagueOfLegendsExePath); var coreMd5 = Utility.Md5Checksum(Directories.CoreFilePath); if (Config.Instance.DeveloperSettings.IgnoreUpdate) { Log.Info("Ignore Update"); callback(new UpdateResponse { State = UpdateState.UpToDate }); return; } Task.Factory.StartNew(() => { try { using (var client = new GZipWebClient { Timeout = 4000 }) { var info = JsonConvert.DeserializeObject <UpdateInfo>(client.DownloadString(Url + leagueMd5)); if (info.Version == "0") { Log.Info("League Version not Supported " + leagueMd5); callback(new UpdateResponse { State = UpdateState.VersionNotSupported }); return; } if (info.Version != coreMd5 && WhiteList.Any(s => info.Url.StartsWith(s))) { Log.Info("Update available for " + leagueMd5); callback(new UpdateResponse { State = UpdateState.UpdateAvailable, Info = info }); return; } Log.Info("League Version is up to date " + leagueMd5); callback(new UpdateResponse { State = UpdateState.UpToDate }); } } catch (Exception e) { Log.Warn(e); callback(new UpdateResponse { State = UpdateState.UpdateError }); } }); }
private void MapMenuItem_Tap(object sender, RoutedEventArgs e) { FrameworkElement fe = (FrameworkElement) sender; DataBinding g = (DataBinding) fe.DataContext; Plane p = (Plane) g.Attributes["plane"]; GZipWebClient wc = new GZipWebClient(); wc.DownloadStringCompleted += wc_DownloadStringCompleted; wc.DownloadStringAsync(new Uri("http://www.planepictures.net/netsearch4.cgi?srch=" + p.Registration + "&srng=2&stype=reg"),fe); }
/// <summary> /// Methode which downloads the events picture /// </summary> public void DownloadPicture() { WebClient DownloadPictureWC = new GZipWebClient(); DownloadPictureWC.OpenReadCompleted += new OpenReadCompletedEventHandler(DownloadPictureWC_OpenReadCompleted); Uri _Uri = new Uri(person.PictureURL + this.PictureName, UriKind.Absolute); DownloadPictureWC.OpenReadAsync(_Uri); }
public static void ReloadPages() { Debug.Log($"Updating Package listing: {PackageListApi}"); using (var client = new GZipWebClient()) { client.DownloadStringCompleted += Client_DownloadStringCompleted; var address = new Uri(PackageListApi); client.DownloadStringAsync(address); } }
/// <summary> /// Methode which downloads the persons picture /// </summary> public void DownloadPicture() { WebClient DownloadPictureWC = new GZipWebClient(); DownloadPictureWC.OpenReadCompleted += new OpenReadCompletedEventHandler(DownloadPictureWC_OpenReadCompleted); Uri _Uri = new Uri(person.PictureURL + this.PictureName, UriKind.Absolute); System.Diagnostics.Debug.WriteLine(_Uri.ToString()); DownloadPictureWC.OpenReadAsync(_Uri); }
/// <summary> /// The begin request. /// </summary> /// <param name="url">The url.</param> /// <param name="callback">The callback.</param> /// <typeparam name="T">The type that the request is being made for</typeparam> public static void BeginRequest <T>(string url, Action <HttpResponse <T> > callback) { var client = new GZipWebClient(); var timer = new Timer(state => client.CancelAsync(), null, timeout, TimeSpan.FromMilliseconds(-1)); Debug.WriteLine("HTTP Request: {0}", url); client.DownloadStringCompleted += (s, e) => ProcessResponse(callback, e); client.DownloadStringAsync(new Uri(url, UriKind.Absolute), timer); }
private void RetrieveTwitterValues(Uri twitterFeed, ObservableCollectionEx <TwitterStatusModel> twitterStatus) { var wc = new GZipWebClient(); ErrorService service = new ErrorService("Unable to retrieve the twitter feed.", "").ErrorDialog(true); var loadedEventArgs = new LoadEventArgs(); IObservable <IEvent <DownloadStringCompletedEventArgs> > o = Observable.FromEvent <DownloadStringCompletedEventArgs>(wc, "DownloadStringCompleted"); o.Subscribe(s => { if (s.EventArgs != null) { try { string twitterResults = s.EventArgs.Result; if (!String.IsNullOrWhiteSpace(twitterResults)) { XDocument doc = XDocument.Parse(twitterResults); ParseTwitterResults(doc, twitterStatus); } else { service.HandleError(); } } catch (Exception) { service.HandleError(); } finally { loadedEventArgs.IsLoaded = true; OnTwitterLoaded(loadedEventArgs); } } else { service.HandleError(); loadedEventArgs.IsLoaded = true; OnTwitterLoaded(loadedEventArgs); } }, e => { loadedEventArgs.IsLoaded = true; loadedEventArgs.Message = e.Message.ToString(CultureInfo.InvariantCulture); OnTwitterLoaded(loadedEventArgs); service.HandleError(); } ); wc.DownloadStringAsync(twitterFeed); }
public ObservableCollection <UpComingViewSchedule> GetUpcomingSchedule(string url, string pageStart, string pageSize) { _upcoming = new ObservableCollection <UpComingViewSchedule>(); var upcoming2 = new List <UpComingSchedule>(); var loadedEventArgs = new LoadEventArgs(); string queryString = String.Format("{0}?start={1}&pageSize={2}", url, pageStart ?? "", pageSize ?? ""); var wb = new GZipWebClient(); Observable.FromEvent <DownloadStringCompletedEventArgs>(wb, "DownloadStringCompleted") .ObserveOn(Scheduler.ThreadPool) .Select(x => ProcessUpcomingItems(x.EventArgs.Result)) .ObserveOn(Scheduler.Dispatcher) .Subscribe(s => { loadedEventArgs.IsLoaded = true; loadedEventArgs.Message = ""; foreach (UpComingSchedule upComingSchedule in s) { upcoming2.Add(upComingSchedule); _upcoming.Add(ConvertToView(upComingSchedule)); } ThreadPool.QueueUserWorkItem(o => { InsertIntoIS(upcoming2); CacheUpComingSchedule(_upcoming); }); OnUpcomingScheduleLoaded(loadedEventArgs); }, e => { loadedEventArgs.IsLoaded = true; //TODO: LOG Error ErrorService error = new ErrorService( "Unable to retrieve any upcoming events", "") .ErrorDialog(true); error.HandleError(); OnUpcomingScheduleLoaded(loadedEventArgs); } ); wb.DownloadStringAsync(new Uri(queryString)); //TODO: Add this to the service //if (!_upcoming.Any()) //{ // _upcoming.Add(new UpComingViewSchedule{Teams = "Regular season complete", GameDateTime = "Enjoy the playoffs"}); //} return(_upcoming); }
public ObservableCollection <CafeResult> GetCafeResults(CityResult cityResult) { var cafeResults = new ObservableCollection <CafeResult>(); var theForkURL = getUrl(cityResult); var wc = new GZipWebClient(); wc.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.60"); wc.Headers.Add("Accept-Language", "en-US,en;q=0.9"); wc.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); wc.Headers.Add("Accept-Encoding", "gzip"); var test = wc.DownloadString(theForkURL); var htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(test); string typeXpath = "//span[@class = 'enrzupw0 css-1ujxl3z ejesmtr0']"; string nameXpath = "//div[@class = 'css-aycukd e6vs4hd0']/div/h2/a"; string addressExpath = "//p[@class = 'css-axj1nn ejesmtr0']"; string avPriceXpath = "//p[@class = 'css-a7e1wa ejesmtr0']/span[2]"; string rateXpath = "//span[@class = 'css-17f8ytt e1l48fgb0']/span[1]"; try { for (int i = 0; i < 5; i++) // what if there are less than 5? { var type = htmlDoc.DocumentNode.SelectNodes(typeXpath)[i].InnerText; var name = htmlDoc.DocumentNode.SelectNodes(nameXpath)[i].InnerText; var address = htmlDoc.DocumentNode.SelectNodes(addressExpath)[i].InnerText; string averagePrice = ""; if (htmlDoc.DocumentNode.SelectNodes(avPriceXpath).Count > 1) { averagePrice = htmlDoc.DocumentNode.SelectNodes(avPriceXpath)[i].InnerText; } string rate = ""; if (htmlDoc.DocumentNode.SelectNodes(rateXpath).Count > i) { rate = htmlDoc.DocumentNode.SelectNodes(rateXpath)[i].InnerText; //not all cafes have ratings! } cafeResults.Add(new CafeResult(type, name, address, averagePrice, rate)); } } catch (Exception e) { Utils.Log.LogExceptions(e.Message); } //var link = htmlDoc.DocumentNode.SelectSingleNode("//span[@class = 'enrzupw0 css-1ujxl3z ejesmtr0']").InnerText; return(cafeResults); }
void GetChapters(String Url, String IPEndPoint_IPAddress) { //HtmlWeb Hw = new HtmlWeb(); //var Hd = Hw.Load(Url); String html; using (var wc = new GZipWebClient(IPEndPoint_IPAddress)) { html = wc.DownloadString(Url); } HtmlAgilityPack.HtmlDocument Hd = new HtmlAgilityPack.HtmlDocument(); Hd.LoadHtml(html); var Nodes = Hd.DocumentNode.SelectNodes("//div"); var Node_Chapters = Nodes.FirstOrDefault(O => O.Attributes.Contains("id") && O.Attributes["id"].Value == "chapters"); List <String> List_Links = new List <String>(); //var Links = Hd.DocumentNode.SelectNodes("//a"); var Links = Node_Chapters.SelectNodes(".//a"); foreach (var Item_Link in Links) { if (Item_Link.Attributes.Contains("href")) { if (Item_Link.Attributes["href"].Value.StartsWith(@"http://mangafox.me/manga/") || Item_Link.Attributes["href"].Value.StartsWith(@"/mangafox.me/manga/")) { if (Item_Link.Attributes["href"].Value.StartsWith(@"http://mangafox.me/manga/")) { List_Links.Add(Item_Link.Attributes["href"].Value); Debug.WriteLine(Item_Link.Attributes["href"].Value); } else { Uri Uri_Source = new Uri(Url); List_Links.Add("http://" + Uri_Source.Host + Item_Link.Attributes["href"].Value); Debug.WriteLine("http://" + Uri_Source.Host + Item_Link.Attributes["href"].Value); } } } } StringBuilder Sb_Links = new StringBuilder(); List_Links.OrderBy(O => O).ToList().ForEach(O => Sb_Links.AppendLine(O)); this.Txt_Chapters.Text = Sb_Links.ToString().Trim(); }
/// <exclude /> public override Rate Fetch(Pair pair) { string url = pair.Construct(URL); var data = new GZipWebClient().DownloadString(url); if (data == null) { ThrowFormatChanged(); } var value = data.Split(',')[1].Trim(); return(new Rate(double.Parse(value))); }
/// <summary> /// GetFullMessageFromMessageId /// </summary> public void GetFullMessageFromMessageId(Action <MessageModel> callback, Guid conversationId, Guid messageId) { Debug.WriteLine(string.Format("{0} {1} {2}", DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss.fff tt"), "YapperServiceProxy::GetFullMessageFromMessageId ", "start")); Uri uri = new Uri(string.Format( YapperServiceProxy.GetFullMessageFromMessageIdRelativeUrl, YapperServiceProxy.ServicePrefix, conversationId, messageId)); GZipWebClient webClient = new GZipWebClient(); webClient.Headers["AuthTokenCookie"] = this.GetAuthTokenCookie(); webClient.Headers["Accept-Encoding"] = "gzip, deflate"; webClient.DownloadStringCompleted += new DownloadStringCompletedEventHandler(SingleMessageDownloaded); webClient.DownloadStringAsync(uri, callback); }
/// <summary> /// GetAllMessages /// </summary> public void GetAllMessages(Action <List <MessageModel> > callback) { Debug.WriteLine(string.Format("{0} {1} {2}", DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss.fff tt"), "YapperServiceProxy::GetAllMessagesSinceLastSync ", "start")); (Application.Current as App).PerfTrackerStopWatch.Restart(); Uri uri = new Uri(string.Format( YapperServiceProxy.AllMessagesRelativeUrl, YapperServiceProxy.ServicePrefix)); GZipWebClient webClient = new GZipWebClient(); webClient.Headers["AuthTokenCookie"] = this.GetAuthTokenCookie(); webClient.Headers["Accept-Encoding"] = "gzip, deflate"; webClient.DownloadStringCompleted += new DownloadStringCompletedEventHandler(AllMessagesDownloaded); webClient.DownloadStringAsync(uri, callback); }
private static void DownloadTile(object tile) { var parameters = (object[])tile; var tileInfo = (TileInfo)parameters[0]; var doneEvent = (MultipleThreadResetEvent)parameters[1]; var url = ((WebTileProvider)_tileSource.Provider).Request.GetUri(tileInfo); var request = new GZipWebClient(); var bytes = request.DownloadData(url); if (bytes != null) { var stream = new MemoryStream(bytes); Logger.Debug("Start parsing vector tile..."); var layerInfos = VectorTileParser.Parse(stream); tiles[tileInfo] = layerInfos; } doneEvent.SetOne(); }
public static PageItem ProcessWebPage(String pageURL, String cssSelector) { PageItem pageItem = null; try { // Setup the configuration to support document loading var config = Configuration.Default.WithDefaultLoader(); // Load the names of all The Big Bang Theory episodes from Wikipedia using (GZipWebClient client = new GZipWebClient()) { client.Headers.Add("user-agent", Analyzer.Common.Configuration.ConfigurationManager.AppSettings.UserAgentOptions.GetRandom()); string webLocationContent = client.DownloadString(pageURL); var parser = new HtmlParser(); var document = parser.Parse(webLocationContent); // This CSS selector gets the desired content var cellSelector = cssSelector; // Perform the query to get all cells with the content var cells = document.QuerySelectorAll(cellSelector); // We are only interested in the text - select it with LINQ var pageContent = cells.Select(m => m.TextContent); if (pageContent != null && pageContent.Count() > 0) { pageItem = new PageItem(); StringBuilder sb = new StringBuilder(); foreach (var pageContentElement in pageContent) { sb.AppendLine(pageContentElement); } pageItem.Content = Analyzer.Common.HtmlRemoval.StripTagsCharArray(sb.ToString()); pageItem.Url = pageURL; } } } catch (Exception ex) { Analyzer.Common.Logger.ExceptionLoggingService.Instance.WriteError("Error in processing given wen URL: " + pageURL, ex); pageItem = null; } return(pageItem); }
void DownloadChapter(String Url, String IPEndPoint_IPAdress) { String FilePath = this.DownloadChapter_FilePath; //Get Pages String html; using (var wc = new GZipWebClient(IPEndPoint_IPAdress)) { html = wc.DownloadString(Url); } HtmlAgilityPack.HtmlDocument Hd = new HtmlAgilityPack.HtmlDocument(); Hd.LoadHtml(html); //HtmlWeb Hw = new HtmlWeb(); //var Hd = Hw.Load(Url); //var Nds = Hd.DocumentNode.SelectNodes("//form"); //var Nd_Form = Nds.FirstOrDefault(O => O.Attributes.Contains("id") && O.Attributes["id"].Value == "top_bar"); var Nds_Script = Hd.DocumentNode.SelectNodes("//script"); var Nd_Script = Nds_Script.FirstOrDefault(O => O.InnerText.Contains("var total_pages")); Regex R = new Regex(@"var total_pages=[0-9]*;"); var Matches = R.Matches(Nd_Script.InnerText); R = new Regex(@"=[0-9]*"); Int32 Pages = Convert.ToInt32(R.Matches(Matches[0].Value)[0].Value.TrimStart('=')); String Chapter_Url = Strings.Mid(Url, 1, Strings.InStrRev(Url, @"/")); String Tmp = Strings.Mid(Chapter_Url, 1, Strings.InStrRev(Chapter_Url, @"/") - 1); String Chapter = Strings.Mid(Tmp, Strings.InStrRev(Tmp, @"/") + 1); this.mDownloadPage_Params = new DownloadPage_Params() { Chapter = Chapter, Chapter_Url = Chapter_Url, FilePath = FilePath, Url = Url, IPEndPoint_IPAdress = IPEndPoint_IPAdress }; Parallel.For(0, Pages, this.DownloadPage); }
public static PageItem ProcessWordpressArticle(String pageURL) { PageItem pageItem = null; try { // Setup the configuration to support document loading var config = Configuration.Default.WithDefaultLoader(); // Load the names of all The Big Bang Theory episodes from Wikipedia var address = pageURL; using (GZipWebClient client = new GZipWebClient()) { client.Headers.Add("user-agent", Analyzer.Common.Configuration.ConfigurationManager.AppSettings.UserAgentOptions.GetRandom()); string webLocationContent = client.DownloadString(pageURL); var parser = new HtmlParser(); var document = parser.Parse(webLocationContent); // Asynchronously get the document in a new context using the configuration //var document = await BrowsingContext.New(config).OpenAsync(address); var article = document.All.SingleOrDefault(o => o.LocalName == "article"); var title = article.QuerySelectorAll("h1.entry-title"); var content = article.QuerySelectorAll("div.entry-content"); { pageItem = new PageItem(); pageItem.Title = title.First().TextContent; pageItem.Content = Analyzer.Common.HtmlRemoval.StripTagsCharArray(content.First().TextContent); pageItem.Url = pageURL; } } } catch (Exception ex) { Analyzer.Common.Logger.ExceptionLoggingService.Instance.WriteError("Error in processing given wen URL: " + pageURL, ex); pageItem = null; } return(pageItem); }
public static bool downloadToFile(string url, string filepath, WebProxy wp = null) { filepath += ".tmp"; if (File.Exists(filepath) && !removeFile(filepath)) { return false; } Uri downloadurl = new Uri(url); using (var client = new GZipWebClient()) { if (wp != null) { client.Proxy = wp; } // Get the username and password if it has been set if (!string.IsNullOrEmpty(downloadurl.UserInfo)) { var credInfo = downloadurl.UserInfo.Split(':'); if (credInfo.Length == 2) { client.Credentials = new System.Net.NetworkCredential(credInfo[0], credInfo[1]); } // Strip userinfo downloadurl = new Uri(downloadurl.GetComponents(UriComponents.AbsoluteUri & ~UriComponents.UserInfo, UriFormat.UriEscaped)); } try { Log.Trace("Downloading {0} to {1}", downloadurl.ToString(), filepath); client.DownloadFile(downloadurl, filepath); Log.Debug("Downloaded {0} to {1}", downloadurl.ToString(), filepath); return true; } catch (WebException e) { Log.Error("File '" + filepath + "' could not be downloaded from server.", e); return false; } } }
/// <summary> /// 发送请求 /// </summary> /// <param name="url">Url</param> /// <param name="isPost">是否Post</param> /// <param name="timeOut">过期时间(为0则不设置过期时间)</param> /// <returns></returns> private static string SendRequest(string url, string data, bool isPost = true, int timeOut = 0) { GZipWebClient webClient = new GZipWebClient(timeOut); webClient.Headers.Add("Credential", Credential); webClient.Headers.Add("ClientType", ClientType); webClient.Headers.Add("ClientVersion", ClientVersion); webClient.Headers.Add("Referer", "http://www.idfgame.com/New2.swf"); webClient.Headers.Add("ClientSettingVersion", ClientSettingVersion); try { string returnResponse = ""; if (!isPost) { string address = string.Format("{0}?{1}", url, data); StreamReader Reader = new StreamReader(webClient.OpenRead(address), Encoding.UTF8); returnResponse = Reader.ReadToEnd(); } else { returnResponse = webClient.UploadString(url, data); } if (!string.IsNullOrEmpty(webClient.ResponseHeaders["Credential"])) { Credential = webClient.ResponseHeaders["Credential"]; } if (webClient.StatusCode() != HttpStatusCode.OK) { return(""); } return(returnResponse); } catch (Exception ex) { return(ex.Message); } }
public async Task DownloadFileAsync(string url, string filePath, string referer = null) { var webclient = new GZipWebClient(); if(!string.IsNullOrEmpty(referer)) webclient.Headers.Add("Referer", referer); try { _logger.Info($"Downloading file: {url} and saving to {filePath}"); await webclient.DownloadFileTaskAsync(new Uri(url), filePath); _analyticsService.ReportEvent(AnalyticEvent.Download, url); } catch (WebException ex) { _logger.Error(ex); _analyticsService.ReportEvent(AnalyticEvent.DownloadFailed, url); throw; } }