public dynamic Load(string url)
{
using (GZipWebClient client = new GZipWebClient())
{
return(client.DownloadString(url));
}
}
public XmlDocument GetXml(string url, string referer = null)
{
_logger.Info($"Getting XML from url: {url}");
var client = new GZipWebClient();
if (!string.IsNullOrEmpty(referer))
client.Headers.Add("Referer", referer);
var xmlDoc = new XmlDocument();
try
{
var xml = client.DownloadString(url);
xmlDoc.LoadXml(xml);
}
catch (Exception ex)
{
_logger.Error(ex);
}
return xmlDoc;
}
public static HtmlDocument Nacti(String sport, String manualniAdresa)
{
/*------vrati HTML pro livestranku daneho sportu--------------------------------------------*/
if (sport == "H")
{
//----pro hokej se to dela jinak---
return null;
}
//Encoding kodovani = Encoding.GetEncoding(28591); latin1
Encoding kodovani = Encoding.GetEncoding("utf-8");
GZipWebClient mujWebKlient = new GZipWebClient();
mujWebKlient.Headers.Add("Accept-Language", "cs-CZ,cs;q=0.8");
mujWebKlient.Headers.Add("Accept-Encoding", "gzip,deflate,sdch");
mujWebKlient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17");
String liveURL;
if (manualniAdresa != "")
{
liveURL = manualniAdresa;
}
else
{
liveURL = knihovna_ser.DejConfig("liveURL_" + sport);
}
HtmlWeb webovaStranka = new HtmlWeb();
HtmlDocument docHtml = new HtmlDocument();
docHtml.Load(mujWebKlient.OpenRead(liveURL), kodovani);
return docHtml;
}
public void GetKnownRepositories(Action <ObservableCollection <string> > callback, bool forceUpdate = false)
{
if (_cache.Count > 0 && !forceUpdate)
{
callback(new ObservableCollection <string>(_cache));
return;
}
Log.Info("Update Known Repositories " + Url);
Task.Factory.StartNew(() =>
{
try
{
using (var client = new GZipWebClient())
{
var repos = client.DownloadString(Url);
var matches = Regex.Matches(repos, "<repo>(.*)</repo>");
_cache.Clear();
foreach (Match match in matches)
{
_cache.Add(match.Groups[1].ToString());
}
callback(new ObservableCollection <string>(_cache));
}
}
catch (Exception e)
{
Log.Warn(e);
}
});
}
private async void codeList_Tap(object sender, System.Windows.Input.GestureEventArgs e)
{
if (codeList.SelectedItem == null)
{
return;
}
CheatText cheatText = (CheatText)codeList.SelectedItem;
codeList.SelectedItem = null;
if (cheatText.Url != null && cheatText.Url != "") //need to go to the link to obtain the code
{
GZipWebClient webClient = new GZipWebClient();
string response = null;
SystemTray.GetProgressIndicator(this).IsIndeterminate = true;
try
{
//USE THIRD-PARTY GZIPWEBCLIENT
webClient.Headers[HttpRequestHeader.Accept] = "text/html, application/xhtml+xml, */*";
webClient.Headers[HttpRequestHeader.Referer] = "http://www.supercheats.com/search.php";
webClient.Headers[HttpRequestHeader.AcceptLanguage] = "en-US";
webClient.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko";
webClient.Headers[HttpRequestHeader.AcceptEncoding] = "gzip, deflate";
webClient.Headers[HttpRequestHeader.Connection] = "Keep-Alive";
webClient.Headers[HttpRequestHeader.Host] = "www.supercheats.com";
response = await webClient.DownloadStringTaskAsync(new Uri(cheatText.Url, UriKind.Absolute));
if (response == null)
{
SystemTray.GetProgressIndicator(this).IsIndeterminate = false;
return;
}
Match contentMatch = Regex.Match(response, "(?<=<div id='sub).*?(?=</div>)", RegexOptions.Singleline);
string content = contentMatch.Value;
int index = content.IndexOf(">"); //get the position of the closing bracket
content = content.Substring(index + 1); //get rid of the closing bracket
cheatText.TextHtml += "<p style=\"font-size:22px\">" + content + "</p></body></html>";
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
SystemTray.GetProgressIndicator(this).IsIndeterminate = false;
}
cheatTextStackpanel.DataContext = cheatText;
cheatTextBox.NavigateToString(cheatText.TextHtml);
gameList.Visibility = Visibility.Collapsed;
codeList.Visibility = Visibility.Collapsed;
cheatTextStackpanel.Visibility = Visibility.Visible;
}
public void LoadData(string lastItem)
{
if (IsLoading)
{
return;
}
IsLoading = true;
if (BeginLoading != null)
{
BeginLoading();
}
string uriString = "http://www.reddit.com/r/todayilearned.json";
if (!string.IsNullOrEmpty(lastItem))
{
uriString += "?after=" + lastItem;
}
var uri = new Uri(uriString);
var client = new GZipWebClient();
client.Headers["User-Agent"] = "windowsphone:com.trivia.buff:v1.8.9 (by /u/camradal)";
client.DownloadStringCompleted += client_DownloadStringCompleted;
client.DownloadStringAsync(uri);
}
void Btn_Scrape_Click(object sender, EventArgs e)
{
/*
* HtmlWeb Hw = new HtmlWeb();
* var Hd = Hw.Load(this.Txt_Url.Text);
* var Links = Hd.DocumentNode.SelectNodes("//a");
* foreach (var Item_Link in Links)
* {
* if (Item_Link.Attributes.Contains("href"))
* {
* if (Item_Link.Attributes["href"].Value.StartsWith(@"http://mangafox.me/manga/"))
* { Debug.WriteLine(Item_Link.Attributes["href"].Value); }
* }
* }
*/
String Url = this.Txt_Url.Text;
String IPEndPoint_IPAddress = this.Txt_EndPointIPAddress.Text;
String html;
using (var wc = new GZipWebClient(IPEndPoint_IPAddress))
{ html = wc.DownloadString(Url); }
this.Txt_Chapters.Text = html;
}
public void DownloadFile(string url, string fileName)
{
try
{
var fileInfo = new FileInfo(fileName);
if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
_logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
var stopWatch = Stopwatch.StartNew();
var uri = new HttpUri(url);
using (var webClient = new GZipWebClient())
{
webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
webClient.Proxy = GetProxy(uri);
webClient.DownloadFile(uri.FullUri, fileName);
stopWatch.Stop();
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
}
}
catch (WebException e)
{
_logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
throw;
}
catch (Exception e)
{
_logger.Warn(e, "Failed to get response from: " + url);
throw;
}
}
public void Search(string term, string lastItem)
{
if (IsSearching)
{
return;
}
IsSearching = true;
if (BeginLoading != null)
{
BeginLoading();
}
string uriString = "http://www.reddit.com/r/todayilearned/search.json?sort=relevance&restrict_sr=on&t=all&q=" + term;
if (!string.IsNullOrEmpty(lastItem))
{
uriString += "&after=" + lastItem;
}
else
{
this.SearchItems.Clear();
}
var uri = new Uri(uriString);
var client = new GZipWebClient();
client.Headers["User-Agent"] = "windowsphone:com.trivia.buff:v1.8.9 (by /u/camradal)";
client.DownloadStringCompleted += client_DownloadSearchStringCompleted;
client.DownloadStringAsync(uri);
}
private XDocument retrieveXML(string type, string date, string venue, int raceno)
{
string xml;
using (var webClient = new GZipWebClient())
{
webClient.QueryString.Add("type", type);
webClient.QueryString.Add("date", date);
webClient.QueryString.Add("venue", venue);
if (type == "pooltot")
{
webClient.QueryString.Add("raceno", raceno.ToString());
}
if (type == "jcbwracing_winplaodds")
{
webClient.QueryString.Add("start", raceno.ToString());
webClient.QueryString.Add("end", raceno.ToString());
}
//webClient.Encoding = Encoding.UTF8;
xml = webClient.DownloadString(BaseUrl);
return(XDocument.Parse(xml));
}
}
public XmlDocument GetXml(string url, string referer = null)
{
_logger.Info($"Getting XML from url: {url}");
var client = new GZipWebClient();
if (!string.IsNullOrEmpty(referer))
{
client.Headers.Add("Referer", referer);
}
var xmlDoc = new XmlDocument();
try
{
var xml = client.DownloadString(url);
xmlDoc.LoadXml(xml);
}
catch (Exception ex)
{
_logger.Error(ex);
}
return(xmlDoc);
}
public void GetAssemblyDatabase(Action <ObservableCollection <LSharpDbAssembly> > callback,
bool forceUpdate = false)
{
if (_cache.Count > 0 && !forceUpdate)
{
callback(new ObservableCollection <LSharpDbAssembly>(_cache));
return;
}
Log.Info("Update Assembly Database " + Url);
Task.Factory.StartNew(() =>
{
try
{
using (var client = new GZipWebClient())
{
var data = JsonConvert.DeserializeObject <List <LSharpDbAssembly> >(client.DownloadString(Url));
_cache.Clear();
foreach (var assembly in data)
{
_cache.Add(assembly);
}
callback(new ObservableCollection <LSharpDbAssembly>(_cache));
}
}
catch (Exception e)
{
Log.Warn(e);
}
});
}
/// <summary>
/// Use this to get the whole page content
/// </summary>
/// <param name="pageURL"></param>
/// <returns></returns>
public static PageItem ProcessWebPage(String pageURL)
{
PageItem pageItem = null;
try
{
// Setup the configuration to support document loading
var config = Configuration.Default.WithDefaultLoader();
// Load the names of all The Big Bang Theory episodes from Wikipedia
using (GZipWebClient client = new GZipWebClient())
{
client.Headers.Add("user-agent", Analyzer.Common.Configuration.ConfigurationManager.AppSettings.UserAgentOptions.GetRandom());
pageItem = new PageItem();
string webLocationContent = client.DownloadString(pageURL);
pageItem.Content = Analyzer.Common.HtmlRemoval.StripTagsCharArray(webLocationContent);
pageItem.Url = pageURL;
}
}
catch (Exception ex)
{
Analyzer.Common.Logger.ExceptionLoggingService.Instance.WriteError("Error in processing given wen URL: " + pageURL, ex);
pageItem = null;
}
return(pageItem);
}
public override void Download()
{
// NOTE ultimate guitar service prohibits unauthorized users to download raw gp files
// By now, we are do not force users to create any accounts as there is another way to get gp-files
// unfortunately, these files are json-serialized representation of gp, so we need to perform additional manipulations
if (Entry.Type == "guitar pro")
{
GetGuitarProDownloadUrl(Entry.Id).ContinueWith(task =>
{
var link = task.Result;
if (link != string.Empty)
{
GZipWebClient client = new GZipWebClient();
client.DownloadStringAsync(new Uri(link));
client.DownloadStringCompleted += (o, e) =>
{
// convert from json to guitar pro format
var songReader = new JsonSongReader(e.Result);
var song = songReader.ReadSong();
var songWriter = new FifthGuitarProSongWriter(new BinaryWriter(GetOutputStream()));
songWriter.WriteSong(song);
InvokeDownloadComplete(new DownloadCompletedEventArgs(false));
};
}
});
}
else
{
Download(WebRequest.Create(Entry.Url));
}
}
public void Run( MainForm form )
{
using( WebClient client = new GZipWebClient() ) {
WebRequest.DefaultWebProxy = null;
int i = 0;
DownloadData( classicJarUri, client, "classic.jar", form, ref i );
DownloadData( modernJarUri, client, "1.6.2.jar", form, ref i );
DownloadData( terrainPatchUri, client, "terrain-patch.png", form, ref i );
}
reader = new ZipReader();
reader.ShouldProcessZipEntry = ShouldProcessZipEntry_Classic;
reader.ProcessZipEntry = ProcessZipEntry_Classic;
using( FileStream srcClassic = File.OpenRead( "classic.jar" ),
srcModern = File.OpenRead( "1.6.2.jar" ),
dst = new FileStream( "default.zip", FileMode.Create, FileAccess.Write ) ) {
writer = new ZipWriter( dst );
reader.Extract( srcClassic );
// Grab animations and snow
animBitmap = new Bitmap( 1024, 64, PixelFormat.Format32bppArgb );
reader.ShouldProcessZipEntry = ShouldProcessZipEntry_Modern;
reader.ProcessZipEntry = ProcessZipEntry_Modern;
reader.Extract( srcModern );
writer.WriteNewImage( animBitmap, "animations.png" );
writer.WriteNewString( animationsTxt, "animations.txt" );
animBitmap.Dispose();
writer.WriteCentralDirectoryRecords();
}
if( !File.Exists( "terrain-patched.png" ) )
File.Move( "terrain-patch.png", "terrain-patched.png" );
}
private static string getWeb(string url)
{
GZipWebClient webClient = new GZipWebClient();
webClient.Encoding = System.Text.Encoding.UTF8;
return(webClient.DownloadString(url));
}
public void Check(Action <UpdateResponse> callback)
{
var leagueMd5 = Utility.Md5Checksum(Config.Instance.LeagueOfLegendsExePath);
var coreMd5 = Utility.Md5Checksum(Directories.CoreFilePath);
if (Config.Instance.DeveloperSettings.IgnoreUpdate)
{
Log.Info("Ignore Update");
callback(new UpdateResponse {
State = UpdateState.UpToDate
});
return;
}
Task.Factory.StartNew(() =>
{
try
{
using (var client = new GZipWebClient {
Timeout = 4000
})
{
var info = JsonConvert.DeserializeObject <UpdateInfo>(client.DownloadString(Url + leagueMd5));
if (info.Version == "0")
{
Log.Info("League Version not Supported " + leagueMd5);
callback(new UpdateResponse {
State = UpdateState.VersionNotSupported
});
return;
}
if (info.Version != coreMd5 && WhiteList.Any(s => info.Url.StartsWith(s)))
{
Log.Info("Update available for " + leagueMd5);
callback(new UpdateResponse {
State = UpdateState.UpdateAvailable, Info = info
});
return;
}
Log.Info("League Version is up to date " + leagueMd5);
callback(new UpdateResponse {
State = UpdateState.UpToDate
});
}
}
catch (Exception e)
{
Log.Warn(e);
callback(new UpdateResponse {
State = UpdateState.UpdateError
});
}
});
}
private void MapMenuItem_Tap(object sender, RoutedEventArgs e)
{
FrameworkElement fe = (FrameworkElement) sender;
DataBinding g = (DataBinding) fe.DataContext;
Plane p = (Plane) g.Attributes["plane"];
GZipWebClient wc = new GZipWebClient();
wc.DownloadStringCompleted += wc_DownloadStringCompleted;
wc.DownloadStringAsync(new Uri("http://www.planepictures.net/netsearch4.cgi?srch=" + p.Registration + "&srng=2&stype=reg"),fe);
}
/// <summary>
/// Methode which downloads the events picture
/// </summary>
public void DownloadPicture()
{
WebClient DownloadPictureWC = new GZipWebClient();
DownloadPictureWC.OpenReadCompleted += new OpenReadCompletedEventHandler(DownloadPictureWC_OpenReadCompleted);
Uri _Uri = new Uri(person.PictureURL + this.PictureName, UriKind.Absolute);
DownloadPictureWC.OpenReadAsync(_Uri);
}
public static void ReloadPages()
{
Debug.Log($"Updating Package listing: {PackageListApi}");
using (var client = new GZipWebClient())
{
client.DownloadStringCompleted += Client_DownloadStringCompleted;
var address = new Uri(PackageListApi);
client.DownloadStringAsync(address);
}
}
/// <summary>
/// Methode which downloads the persons picture
/// </summary>
public void DownloadPicture()
{
WebClient DownloadPictureWC = new GZipWebClient();
DownloadPictureWC.OpenReadCompleted += new OpenReadCompletedEventHandler(DownloadPictureWC_OpenReadCompleted);
Uri _Uri = new Uri(person.PictureURL + this.PictureName, UriKind.Absolute);
System.Diagnostics.Debug.WriteLine(_Uri.ToString());
DownloadPictureWC.OpenReadAsync(_Uri);
}
/// <summary>
/// The begin request.
/// </summary>
/// <param name="url">The url.</param>
/// <param name="callback">The callback.</param>
/// <typeparam name="T">The type that the request is being made for</typeparam>
public static void BeginRequest <T>(string url, Action <HttpResponse <T> > callback)
{
var client = new GZipWebClient();
var timer = new Timer(state => client.CancelAsync(), null, timeout, TimeSpan.FromMilliseconds(-1));
Debug.WriteLine("HTTP Request: {0}", url);
client.DownloadStringCompleted += (s, e) => ProcessResponse(callback, e);
client.DownloadStringAsync(new Uri(url, UriKind.Absolute), timer);
}
private void RetrieveTwitterValues(Uri twitterFeed, ObservableCollectionEx <TwitterStatusModel> twitterStatus)
{
var wc = new GZipWebClient();
ErrorService service = new ErrorService("Unable to retrieve the twitter feed.", "").ErrorDialog(true);
var loadedEventArgs = new LoadEventArgs();
IObservable <IEvent <DownloadStringCompletedEventArgs> > o = Observable.FromEvent
<DownloadStringCompletedEventArgs>(wc, "DownloadStringCompleted");
o.Subscribe(s =>
{
if (s.EventArgs != null)
{
try
{
string twitterResults = s.EventArgs.Result;
if (!String.IsNullOrWhiteSpace(twitterResults))
{
XDocument doc = XDocument.Parse(twitterResults);
ParseTwitterResults(doc, twitterStatus);
}
else
{
service.HandleError();
}
}
catch (Exception)
{
service.HandleError();
}
finally
{
loadedEventArgs.IsLoaded = true;
OnTwitterLoaded(loadedEventArgs);
}
}
else
{
service.HandleError();
loadedEventArgs.IsLoaded = true;
OnTwitterLoaded(loadedEventArgs);
}
}, e =>
{
loadedEventArgs.IsLoaded = true;
loadedEventArgs.Message = e.Message.ToString(CultureInfo.InvariantCulture);
OnTwitterLoaded(loadedEventArgs);
service.HandleError();
}
);
wc.DownloadStringAsync(twitterFeed);
}
public ObservableCollection <UpComingViewSchedule> GetUpcomingSchedule(string url, string pageStart,
string pageSize)
{
_upcoming = new ObservableCollection <UpComingViewSchedule>();
var upcoming2 = new List <UpComingSchedule>();
var loadedEventArgs = new LoadEventArgs();
string queryString = String.Format("{0}?start={1}&pageSize={2}", url, pageStart ?? "",
pageSize ?? "");
var wb = new GZipWebClient();
Observable.FromEvent <DownloadStringCompletedEventArgs>(wb, "DownloadStringCompleted")
.ObserveOn(Scheduler.ThreadPool)
.Select(x => ProcessUpcomingItems(x.EventArgs.Result))
.ObserveOn(Scheduler.Dispatcher)
.Subscribe(s =>
{
loadedEventArgs.IsLoaded = true;
loadedEventArgs.Message = "";
foreach (UpComingSchedule upComingSchedule in s)
{
upcoming2.Add(upComingSchedule);
_upcoming.Add(ConvertToView(upComingSchedule));
}
ThreadPool.QueueUserWorkItem(o =>
{
InsertIntoIS(upcoming2);
CacheUpComingSchedule(_upcoming);
});
OnUpcomingScheduleLoaded(loadedEventArgs);
}, e =>
{
loadedEventArgs.IsLoaded = true;
//TODO: LOG Error
ErrorService error = new ErrorService(
"Unable to retrieve any upcoming events", "")
.ErrorDialog(true);
error.HandleError();
OnUpcomingScheduleLoaded(loadedEventArgs);
}
);
wb.DownloadStringAsync(new Uri(queryString));
//TODO: Add this to the service
//if (!_upcoming.Any())
//{
// _upcoming.Add(new UpComingViewSchedule{Teams = "Regular season complete", GameDateTime = "Enjoy the playoffs"});
//}
return(_upcoming);
}
public ObservableCollection <CafeResult> GetCafeResults(CityResult cityResult)
{
var cafeResults = new ObservableCollection <CafeResult>();
var theForkURL = getUrl(cityResult);
var wc = new GZipWebClient();
wc.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.60");
wc.Headers.Add("Accept-Language", "en-US,en;q=0.9");
wc.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
wc.Headers.Add("Accept-Encoding", "gzip");
var test = wc.DownloadString(theForkURL);
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(test);
string typeXpath = "//span[@class = 'enrzupw0 css-1ujxl3z ejesmtr0']";
string nameXpath = "//div[@class = 'css-aycukd e6vs4hd0']/div/h2/a";
string addressExpath = "//p[@class = 'css-axj1nn ejesmtr0']";
string avPriceXpath = "//p[@class = 'css-a7e1wa ejesmtr0']/span[2]";
string rateXpath = "//span[@class = 'css-17f8ytt e1l48fgb0']/span[1]";
try
{
for (int i = 0; i < 5; i++) // what if there are less than 5?
{
var type = htmlDoc.DocumentNode.SelectNodes(typeXpath)[i].InnerText;
var name = htmlDoc.DocumentNode.SelectNodes(nameXpath)[i].InnerText;
var address = htmlDoc.DocumentNode.SelectNodes(addressExpath)[i].InnerText;
string averagePrice = "";
if (htmlDoc.DocumentNode.SelectNodes(avPriceXpath).Count > 1)
{
averagePrice = htmlDoc.DocumentNode.SelectNodes(avPriceXpath)[i].InnerText;
}
string rate = "";
if (htmlDoc.DocumentNode.SelectNodes(rateXpath).Count > i)
{
rate = htmlDoc.DocumentNode.SelectNodes(rateXpath)[i].InnerText; //not all cafes have ratings!
}
cafeResults.Add(new CafeResult(type, name, address, averagePrice, rate));
}
}
catch (Exception e)
{
Utils.Log.LogExceptions(e.Message);
}
//var link = htmlDoc.DocumentNode.SelectSingleNode("//span[@class = 'enrzupw0 css-1ujxl3z ejesmtr0']").InnerText;
return(cafeResults);
}
void GetChapters(String Url, String IPEndPoint_IPAddress)
{
//HtmlWeb Hw = new HtmlWeb();
//var Hd = Hw.Load(Url);
String html;
using (var wc = new GZipWebClient(IPEndPoint_IPAddress))
{ html = wc.DownloadString(Url); }
HtmlAgilityPack.HtmlDocument Hd = new HtmlAgilityPack.HtmlDocument();
Hd.LoadHtml(html);
var Nodes = Hd.DocumentNode.SelectNodes("//div");
var Node_Chapters =
Nodes.FirstOrDefault(O =>
O.Attributes.Contains("id") &&
O.Attributes["id"].Value == "chapters");
List <String> List_Links = new List <String>();
//var Links = Hd.DocumentNode.SelectNodes("//a");
var Links = Node_Chapters.SelectNodes(".//a");
foreach (var Item_Link in Links)
{
if (Item_Link.Attributes.Contains("href"))
{
if (Item_Link.Attributes["href"].Value.StartsWith(@"http://mangafox.me/manga/") ||
Item_Link.Attributes["href"].Value.StartsWith(@"/mangafox.me/manga/"))
{
if (Item_Link.Attributes["href"].Value.StartsWith(@"http://mangafox.me/manga/"))
{
List_Links.Add(Item_Link.Attributes["href"].Value);
Debug.WriteLine(Item_Link.Attributes["href"].Value);
}
else
{
Uri Uri_Source = new Uri(Url);
List_Links.Add("http://" + Uri_Source.Host + Item_Link.Attributes["href"].Value);
Debug.WriteLine("http://" + Uri_Source.Host + Item_Link.Attributes["href"].Value);
}
}
}
}
StringBuilder Sb_Links = new StringBuilder();
List_Links.OrderBy(O => O).ToList().ForEach(O => Sb_Links.AppendLine(O));
this.Txt_Chapters.Text = Sb_Links.ToString().Trim();
}
/// <exclude />
public override Rate Fetch(Pair pair)
{
string url = pair.Construct(URL);
var data = new GZipWebClient().DownloadString(url);
if (data == null)
{
ThrowFormatChanged();
}
var value = data.Split(',')[1].Trim();
return(new Rate(double.Parse(value)));
}
/// <summary>
/// GetFullMessageFromMessageId
/// </summary>
public void GetFullMessageFromMessageId(Action <MessageModel> callback, Guid conversationId, Guid messageId)
{
Debug.WriteLine(string.Format("{0} {1} {2}", DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss.fff tt"), "YapperServiceProxy::GetFullMessageFromMessageId ", "start"));
Uri uri = new Uri(string.Format(
YapperServiceProxy.GetFullMessageFromMessageIdRelativeUrl,
YapperServiceProxy.ServicePrefix,
conversationId,
messageId));
GZipWebClient webClient = new GZipWebClient();
webClient.Headers["AuthTokenCookie"] = this.GetAuthTokenCookie();
webClient.Headers["Accept-Encoding"] = "gzip, deflate";
webClient.DownloadStringCompleted += new DownloadStringCompletedEventHandler(SingleMessageDownloaded);
webClient.DownloadStringAsync(uri, callback);
}
/// <summary>
/// GetAllMessages
/// </summary>
public void GetAllMessages(Action <List <MessageModel> > callback)
{
Debug.WriteLine(string.Format("{0} {1} {2}", DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss.fff tt"), "YapperServiceProxy::GetAllMessagesSinceLastSync ", "start"));
(Application.Current as App).PerfTrackerStopWatch.Restart();
Uri uri = new Uri(string.Format(
YapperServiceProxy.AllMessagesRelativeUrl,
YapperServiceProxy.ServicePrefix));
GZipWebClient webClient = new GZipWebClient();
webClient.Headers["AuthTokenCookie"] = this.GetAuthTokenCookie();
webClient.Headers["Accept-Encoding"] = "gzip, deflate";
webClient.DownloadStringCompleted += new DownloadStringCompletedEventHandler(AllMessagesDownloaded);
webClient.DownloadStringAsync(uri, callback);
}
private static void DownloadTile(object tile)
{
var parameters = (object[])tile;
var tileInfo = (TileInfo)parameters[0];
var doneEvent = (MultipleThreadResetEvent)parameters[1];
var url = ((WebTileProvider)_tileSource.Provider).Request.GetUri(tileInfo);
var request = new GZipWebClient();
var bytes = request.DownloadData(url);
if (bytes != null)
{
var stream = new MemoryStream(bytes);
Logger.Debug("Start parsing vector tile...");
var layerInfos = VectorTileParser.Parse(stream);
tiles[tileInfo] = layerInfos;
}
doneEvent.SetOne();
}
public static PageItem ProcessWebPage(String pageURL, String cssSelector)
{
PageItem pageItem = null;
try
{
// Setup the configuration to support document loading
var config = Configuration.Default.WithDefaultLoader();
// Load the names of all The Big Bang Theory episodes from Wikipedia
using (GZipWebClient client = new GZipWebClient())
{
client.Headers.Add("user-agent", Analyzer.Common.Configuration.ConfigurationManager.AppSettings.UserAgentOptions.GetRandom());
string webLocationContent = client.DownloadString(pageURL);
var parser = new HtmlParser();
var document = parser.Parse(webLocationContent);
// This CSS selector gets the desired content
var cellSelector = cssSelector;
// Perform the query to get all cells with the content
var cells = document.QuerySelectorAll(cellSelector);
// We are only interested in the text - select it with LINQ
var pageContent = cells.Select(m => m.TextContent);
if (pageContent != null && pageContent.Count() > 0)
{
pageItem = new PageItem();
StringBuilder sb = new StringBuilder();
foreach (var pageContentElement in pageContent)
{
sb.AppendLine(pageContentElement);
}
pageItem.Content = Analyzer.Common.HtmlRemoval.StripTagsCharArray(sb.ToString());
pageItem.Url = pageURL;
}
}
}
catch (Exception ex)
{
Analyzer.Common.Logger.ExceptionLoggingService.Instance.WriteError("Error in processing given wen URL: " + pageURL, ex);
pageItem = null;
}
return(pageItem);
}
void DownloadChapter(String Url, String IPEndPoint_IPAdress)
{
String FilePath = this.DownloadChapter_FilePath;
//Get Pages
String html;
using (var wc = new GZipWebClient(IPEndPoint_IPAdress))
{ html = wc.DownloadString(Url); }
HtmlAgilityPack.HtmlDocument Hd = new HtmlAgilityPack.HtmlDocument();
Hd.LoadHtml(html);
//HtmlWeb Hw = new HtmlWeb();
//var Hd = Hw.Load(Url);
//var Nds = Hd.DocumentNode.SelectNodes("//form");
//var Nd_Form = Nds.FirstOrDefault(O => O.Attributes.Contains("id") && O.Attributes["id"].Value == "top_bar");
var Nds_Script = Hd.DocumentNode.SelectNodes("//script");
var Nd_Script = Nds_Script.FirstOrDefault(O => O.InnerText.Contains("var total_pages"));
Regex R = new Regex(@"var total_pages=[0-9]*;");
var Matches = R.Matches(Nd_Script.InnerText);
R = new Regex(@"=[0-9]*");
Int32 Pages = Convert.ToInt32(R.Matches(Matches[0].Value)[0].Value.TrimStart('='));
String Chapter_Url = Strings.Mid(Url, 1, Strings.InStrRev(Url, @"/"));
String Tmp = Strings.Mid(Chapter_Url, 1, Strings.InStrRev(Chapter_Url, @"/") - 1);
String Chapter = Strings.Mid(Tmp, Strings.InStrRev(Tmp, @"/") + 1);
this.mDownloadPage_Params = new DownloadPage_Params()
{
Chapter = Chapter,
Chapter_Url = Chapter_Url,
FilePath = FilePath,
Url = Url,
IPEndPoint_IPAdress = IPEndPoint_IPAdress
};
Parallel.For(0, Pages, this.DownloadPage);
}
public static PageItem ProcessWordpressArticle(String pageURL)
{
PageItem pageItem = null;
try
{
// Setup the configuration to support document loading
var config = Configuration.Default.WithDefaultLoader();
// Load the names of all The Big Bang Theory episodes from Wikipedia
var address = pageURL;
using (GZipWebClient client = new GZipWebClient())
{
client.Headers.Add("user-agent", Analyzer.Common.Configuration.ConfigurationManager.AppSettings.UserAgentOptions.GetRandom());
string webLocationContent = client.DownloadString(pageURL);
var parser = new HtmlParser();
var document = parser.Parse(webLocationContent);
// Asynchronously get the document in a new context using the configuration
//var document = await BrowsingContext.New(config).OpenAsync(address);
var article = document.All.SingleOrDefault(o => o.LocalName == "article");
var title = article.QuerySelectorAll("h1.entry-title");
var content = article.QuerySelectorAll("div.entry-content");
{
pageItem = new PageItem();
pageItem.Title = title.First().TextContent;
pageItem.Content = Analyzer.Common.HtmlRemoval.StripTagsCharArray(content.First().TextContent);
pageItem.Url = pageURL;
}
}
} catch (Exception ex)
{
Analyzer.Common.Logger.ExceptionLoggingService.Instance.WriteError("Error in processing given wen URL: " + pageURL, ex);
pageItem = null;
}
return(pageItem);
}
public static bool downloadToFile(string url, string filepath, WebProxy wp = null)
{
filepath += ".tmp";
if (File.Exists(filepath) && !removeFile(filepath))
{
return false;
}
Uri downloadurl = new Uri(url);
using (var client = new GZipWebClient())
{
if (wp != null)
{
client.Proxy = wp;
}
// Get the username and password if it has been set
if (!string.IsNullOrEmpty(downloadurl.UserInfo)) {
var credInfo = downloadurl.UserInfo.Split(':');
if (credInfo.Length == 2)
{
client.Credentials = new System.Net.NetworkCredential(credInfo[0], credInfo[1]);
}
// Strip userinfo
downloadurl = new Uri(downloadurl.GetComponents(UriComponents.AbsoluteUri & ~UriComponents.UserInfo, UriFormat.UriEscaped));
}
try
{
Log.Trace("Downloading {0} to {1}", downloadurl.ToString(), filepath);
client.DownloadFile(downloadurl, filepath);
Log.Debug("Downloaded {0} to {1}", downloadurl.ToString(), filepath);
return true;
}
catch (WebException e)
{
Log.Error("File '" + filepath + "' could not be downloaded from server.", e);
return false;
}
}
}
/// <summary>
/// 发送请求
/// </summary>
/// <param name="url">Url</param>
/// <param name="isPost">是否Post</param>
/// <param name="timeOut">过期时间（为0则不设置过期时间）</param>
/// <returns></returns>
private static string SendRequest(string url, string data, bool isPost = true, int timeOut = 0)
{
GZipWebClient webClient = new GZipWebClient(timeOut);
webClient.Headers.Add("Credential", Credential);
webClient.Headers.Add("ClientType", ClientType);
webClient.Headers.Add("ClientVersion", ClientVersion);
webClient.Headers.Add("Referer", "http://www.idfgame.com/New2.swf");
webClient.Headers.Add("ClientSettingVersion", ClientSettingVersion);
try
{
string returnResponse = "";
if (!isPost)
{
string address = string.Format("{0}?{1}", url, data);
StreamReader Reader = new StreamReader(webClient.OpenRead(address), Encoding.UTF8);
returnResponse = Reader.ReadToEnd();
}
else
{
returnResponse = webClient.UploadString(url, data);
}
if (!string.IsNullOrEmpty(webClient.ResponseHeaders["Credential"]))
{
Credential = webClient.ResponseHeaders["Credential"];
}
if (webClient.StatusCode() != HttpStatusCode.OK)
{
return("");
}
return(returnResponse);
}
catch (Exception ex)
{
return(ex.Message);
}
}
public async Task DownloadFileAsync(string url, string filePath, string referer = null)
{
var webclient = new GZipWebClient();
if(!string.IsNullOrEmpty(referer))
webclient.Headers.Add("Referer", referer);
try
{
_logger.Info($"Downloading file: {url} and saving to {filePath}");
await webclient.DownloadFileTaskAsync(new Uri(url), filePath);
_analyticsService.ReportEvent(AnalyticEvent.Download, url);
}
catch (WebException ex)
{
_logger.Error(ex);
_analyticsService.ReportEvent(AnalyticEvent.DownloadFailed, url);
throw;
}
}
