public static string GetSpaceTrack(string[] noradId, string username, string password) { string uriBase = "https://www.space-track.org"; string requestController = "/basicspacedata"; string requestAction = "/query"; string predicateValues = "/class/tle_latest/ORDINAL/1/NORAD_CAT_ID/" + string.Join(",", noradId) + "/orderby/NORAD_CAT_ID/format/3le"; string request = uriBase + requestController + requestAction + predicateValues; // Create new WebClient object to communicate with the service using (var client = new WebClientEx()) { // Store the user authentication information var data = new NameValueCollection { { "identity", username }, { "password", password }, }; // Generate the URL for the API Query and return the response var response2 = client.UploadValues(uriBase + "/ajaxauth/login", data); var response4 = client.DownloadData(request); return (System.Text.Encoding.Default.GetString(response4)); } }
void disposeWebClient(WebClientEx webClient) { webClient.DownloadDataCompleted -= webClient_DownloadFileCompleted; webClient.DownloadProgressChanged -= webClient_DownloadProgressChanged; webClient.Dispose(); webClient = null; }
public static ArrayList UploadFileToServer(string filePath) { try { //url where we would be uploading our document string upload_url = "https://soap.click2mail.com/UploadDocument.aspx?clientid=" + user_name + "&clientpwd=" + pass_word; string result1; ArrayList result2 = new ArrayList(); try { var clientweb = new WebClientEx(); //System.Net.WebClient wc = new WebClientEx(); clientweb.Timeout = 900000; byte[] result = clientweb.UploadFile(upload_url, "POST", filePath); result1 = Encoding.ASCII.GetString(result); result2.Add(result1); } catch (Exception ex) { throw new Exception(ex.Message); } return result2; } catch (Exception ex) { throw new Exception(ex.Message); } }
public static byte[] DownloadSymbols() { using (var client = new WebClientEx { Timeout = TimeSpan.FromMinutes(3), DecompressionMethods = DecompressionMethods.Deflate | DecompressionMethods.GZip }) { return client.DownloadData("http://www.dtniq.com/product/mktsymbols_v2.zip"); } }
public static UpdateInfo Check(Uri uri, string version) { NameValueCollection data = new NameValueCollection(); data.Add(osField, Environment.OSVersion.VersionString); data.Add(versionField, version); byte[] output = null; try { WebClientEx webClient = new WebClientEx(); output = webClient.UploadValues(uri, data); } catch (WebException e) { Trace.TraceWarning(e.ToString()); return null; } return UpdateInfo.TryParse(output); }
private void backupGallery(string urlbase) { if (textBox1.Text != "") { Directory.CreateDirectory(path + "/Gallery/"); WebClientEx wb = new WebClientEx(); String src = ""; String pages = ""; Regex nbPages = new Regex("@?page=[0-9]*\">[0-9]*</a><a class=\"noborder\""); Regex nom = new Regex("_blank\" title=\".*\""); Regex urls = new Regex("\\('[A-Za-z0-9]*'\\)\""); src = wb.DownloadString("http://puush.me/login/go/?k=" + key); src = wb.DownloadString("http://puush.me" + urlbase); if (nbPages.IsMatch(src)) { pages = nbPages.Match(src).ToString(); pages = pages.Remove(pages.LastIndexOf('>')); pages = pages.Remove(pages.LastIndexOf('<')); pages = pages.Substring(pages.IndexOf('>') + 1); } else { pages = "1"; } for (int i = 0; i < int.Parse(pages); i++) { src = wb.DownloadString("http://puush.me/" + urlbase + "&page=" + (i + 1)); for (int j = 0; j < urls.Matches(src).Count; j++) { String name = nom.Matches(src)[j].ToString(); String url = urls.Matches(src)[j].ToString(); name = name.Remove(0, 15); name = name.Remove(name.LastIndexOf('(') - 1); url = url.Remove(0, 2); url = url.Remove(url.Length - 3); wb.DownloadFile("http://puu.sh/" + url, path + "/Gallery/" + name); } } } }
private void buttonAdd_Click(object sender, System.EventArgs e) { try { var uri = new Uri(comboBoxUrl.Text); string[] segments = uri.Segments; string fileName; if (segments.Length > 0) { fileName = segments[segments.Length - 1]; } else { fileName = string.Empty; } saveFileDialog.FileName = fileName; if (saveFileDialog.ShowDialog() != DialogResult.OK) { return; } var client = new WebClientEx(); client.DownloadFileCompleted += webClient_DownloadFileCompleted; client.DownloadProgressChanged += webClient_DownloadProgressChanged; var file = new FileInfo(saveFileDialog.FileName); saveFileDialog.InitialDirectory = file.DirectoryName; client.Index = addItem(file, uri.AbsoluteUri); webClients.Add(client); client.DownloadFileAsync(uri, saveFileDialog.FileName); } catch (Exception exception) { MessageBox.Show(exception.Message, Text, MessageBoxButtons.OK, MessageBoxIcon.Error); } }
/// <summary> /// DownloadString /// </summary> /// <param name="url"></param> public void WebDownloadString(string url) { string source = string.Empty; try { Utils.SetSecurityProtocol(); WebClientEx ws = new WebClientEx(); Regex rx = new Regex(@"^(?<protocol>.+?//)(?<username>.+?)(?::(?<password>.+?))?@(?<address>.+)$"); Match match = rx.Match(url); if (match.Success) { ws.Credentials = new NetworkCredential(match.Groups["username"].Value, match.Groups["password"].Value); } ws.DownloadStringCompleted += Ws_DownloadStringCompleted; ws.DownloadStringAsync(new Uri(url)); } catch (Exception ex) { Utils.SaveLog(ex.Message, ex); } }
private void DsReQueueJob(string queueName, string jobId, string errorMessage, TimeSpan timeSpan, Dictionary <String, String> parametersToModify) { string pollerResult; using (WebClientEx client = new WebClientEx()) { //TODO: use round robin if a document store is down. var firstUrl = _dsEndpoints.First(); var payload = JsonConvert.SerializeObject(new { QueueName = queueName, JobId = jobId, ErrorMessage = errorMessage, ReQueue = true, ReScheduleTimespanInSeconds = (Int32)timeSpan.TotalSeconds, ParametersToModify = parametersToModify }); Logger.DebugFormat("ReQueuedJob url: {0} with payload {1}", firstUrl.SetJobCompleted, payload); client.Headers[HttpRequestHeader.ContentType] = "application/json"; pollerResult = client.UploadString(firstUrl.SetJobCompleted, payload); Logger.DebugFormat("SetJobExecuted Result: {0}", pollerResult); } }
public void WebLogin(string url, string username, string pwd) { string source = string.Empty; try { ServicePointManager.SecurityProtocol = (SecurityProtocolType)3072; //TLS 1.2 ServicePointManager.DefaultConnectionLimit = 256; WebClientEx ws = new WebClientEx(); var reqParams = new System.Collections.Specialized.NameValueCollection(); reqParams.Add("account", username); reqParams.Add("password", pwd); ws.UploadValuesCompleted += Ws_login_completed; ws.UploadValuesAsync(new Uri(url), "POST", reqParams); } catch (Exception ex) { Utils.SaveLog(ex.Message, ex); } }
private static void getListCoverAndProfileLink() { if (listCoverPhotoLink.Count == 0 || listProfilePhotoLink.Count == 0) { WebClientEx client = new WebClientEx(); client.DoGet("https://500px.com/popular"); if (!string.IsNullOrEmpty(client.ResponseText)) { string strToken = Regex.Match(client.ResponseText, "content=\"(?<val>[^\"]+)\" name=\"csrf-token\"").Groups["val"].Value.Trim(); if (!string.IsNullOrEmpty(strToken)) { client.DoGet("https://api.500px.com/v1/photos?rpp=38&feature=popular&image_size[]=3&image_size[]=5&page=" + new Random().Next(1, 400) + "&sort=&include_states=true&formats=jpeg%2Clytro&only=&authenticity_token=" + strToken.Replace("+", "%2B")); if (!string.IsNullOrEmpty(client.ResponseText)) { Dictionary <string, object> dicData = new JavaScriptSerializer().Deserialize <Dictionary <string, object> >(client.ResponseText); if (dicData.ContainsKey("photos")) { foreach (Dictionary <string, object> dicPhoto in dicData["photos"] as ArrayList) { foreach (Dictionary <string, object> dicImage in dicPhoto["images"] as ArrayList) { if (dicImage["size"].ToString() == "3") { listProfilePhotoLink.Add(dicImage["url"].ToString()); } if (dicImage["size"].ToString() == "5") { listCoverPhotoLink.Add(dicImage["url"].ToString()); } } } } } } } } }
protected internal static TResponse QueryGoogleApi(TRequest _request, TimeSpan _timeout) { if (_request == null) { throw new ArgumentNullException("_request"); } try { var _uri = _request.GetUri(); if (!string.IsNullOrWhiteSpace(_request.Key)) { var keyParam = "&key=" + _request.Key; _uri = new Uri(_uri.ToString() + keyParam); } var _data = new WebClientEx(_timeout).DownloadData(_uri); return(Deserialize(_data)); } catch (WebException _ex) { if (IndicatesAuthenticationFailed(_ex)) { throw new AuthenticationException(AUTHENTICATION_FAILED_MESSAGE, _ex); } if (_ex.Status == WebExceptionStatus.Timeout) { throw new TimeoutException(string.Format("The request has exceeded the timeout limit of {0} and has been aborted.", _timeout)); } throw; } }
public NicoLiveNet() { IsDebug = false; var wc = new WebClientEx(); _wc = wc; _wc.Encoding = Encoding.UTF8; _wc.Proxy = null; _wc.Headers.Add(HttpRequestHeader.UserAgent, Props.UserAgent); _wc.timeout = 30000; _wc.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore); if (IsDebug) { //foreach (Cookie ck in cc.GetCookies(new Uri(Props.NicoDomain))) // Debug.WriteLine(ck.Name.ToString() + ": " + ck.Value.ToString()); for (int i = 0; i < _wc.Headers.Count; i++) { Debug.WriteLine(_wc.Headers.GetKey(i).ToString() + ": " + _wc.Headers.Get(i)); } } }
void IssueTrackerLogout_Completed(object sender, DownloadStringCompletedEventArgs e) { WebClientEx webClient = (sender as WebClientEx); if (webClient == null) // No webClient, nothing to do. { return; } if (e.Result.Equals("Logged Out")) { // Logout Successful MessageBox.Show("Logged Out!"); isLoggedIn = false; // Reset all permissions. canViewTasks = false; canCreateTasks = false; canSetPriority = false; canAttachFiles = false; // Clear set cookies. cookieContainer = new CookieContainer(); } else { // WTF MessageBox.Show("Connection Error?"); } // Update the UI. // Unhook event handler. webClient.DownloadStringCompleted -= IssueTrackerLogout_Completed; webClient.Dispose(); }
/// <summary> /// DownloadString /// </summary> /// <param name="url"></param> public void WebDownloadString(string url, WebProxy webProxy, string userAgent) { string source = string.Empty; try { Utils.SetSecurityProtocol(LazyConfig.Instance.GetConfig().enableSecurityProtocolTls13); WebClientEx ws = new WebClientEx(); ws.Encoding = Encoding.UTF8; if (webProxy != null) { ws.Proxy = webProxy; } if (Utils.IsNullOrEmpty(userAgent)) { userAgent = $"{Utils.GetVersion(false)}"; } ws.Headers.Add("user-agent", userAgent); Uri uri = new Uri(url); //Authorization Header if (!Utils.IsNullOrEmpty(uri.UserInfo)) { ws.Headers.Add(HttpRequestHeader.Authorization, "Basic " + Utils.Base64Encode(uri.UserInfo)); } ws.DownloadStringCompleted += Ws_DownloadStringCompleted; ws.DownloadStringAsync(uri); } catch (Exception ex) { Utils.SaveLog(ex.Message, ex); } }
protected WebClientEx SetupWebClient(Options o, AuthenticationMode validCredentials = AuthenticationMode.UseValidCredentials) { var c = new WebClientEx(); c.Headers.Add(HttpRequestHeader.Accept, "application/json"); if (!String.IsNullOrEmpty(o.ApiKey) && validCredentials != AuthenticationMode.UseNoCredentials) { string headerName = !String.IsNullOrEmpty(o.ApiKeyHeader) ? o.ApiKeyHeader : Options.DefaultApiKeyHeaderName; c.Headers.Add(headerName, validCredentials == AuthenticationMode.UseValidCredentials ? o.ApiKey : Options.InvalidApiKey); } if (!String.IsNullOrEmpty(o.User) && validCredentials != AuthenticationMode.UseNoCredentials) { c.Credentials = validCredentials == AuthenticationMode.UseValidCredentials ? new NetworkCredential(o.User, o.Pass) : new NetworkCredential(Options.InvalidUserName, Options.InvalidPassword); } return(c); }
public static DownloadAdapter WebClientDownload(System.Uri location, System.Net.NetworkCredential credential = null) { if (location == null) { throw new System.ArgumentNullException("location"); } using (WebClientEx webClient = new WebClientEx(location)) { if (credential != null) { webClient.Credentials = credential; } using (var d = new DownloadAdapter(location) { Request = webClient, ResponseOutputStream = webClient.ResponseStream }) { if (webClient.ResponseHeaders != null) { string contentLength = webClient.ResponseHeaders["Content-Length"]; if (false == string.IsNullOrWhiteSpace(contentLength)) { long.TryParse(contentLength, out d.MaximumLength); } contentLength = null; } return(d); } } }
private void Run() { while (true) { var unix = new DateTime(1970, 1, 1); var now = DateTime.Now; var totalSeconds = now.Subtract(unix).TotalSeconds; for (var i = 0; i < _form.Tasks.Count; i++) { if (_form.Tasks[i].state == PwState.Idle && _form.Tasks[i].timestamp <= totalSeconds) { _form.Tasks[i].state = PwState.Processing; UpdateForm(); var wc = new WebClientEx(); wc.Headers.Add("user-agent", MainForm.UserAgent); var url = new Uri($"https://dpm.mini.pw.edu.pl/node/{_form.Tasks[i].id}/register"); // Add session cookie wc.CookieContainer.Add(url, _form.SessionCookie); // Execute try { // Get form_token var get = wc.DownloadString(url); var match = Regex.Match(get, @"name=\""form_token\"" value=\""(.*?)\"""); if (!match.Success) { throw new Exception("form_token not found"); } var post = new NameValueCollection { { "count", _form.Tasks[i].count }, { "form_token", match.Groups[1].Value }, { "form_id", "registration_form" }, }; var result = wc.UploadValues(url, post); var result_str = Encoding.UTF8.GetString(result); if (result_str.Contains("Rejestracja została zapisana")) { // Success :) _form.Tasks[i].state = PwState.Success; UpdateForm(); continue; } } catch (WebException ex) { // Timeout? Network error? Try again! } catch (Exception ex) { Debugger.Break(); } _form.Tasks[i].state = PwState.Idle; UpdateForm(); } } // Sleep for 500ms Thread.Sleep(500); } }
private void ConnectToURL(Uri URI, object token) { if (URI.Host == "192.168.100.1") { c_callback.Invoke(Gdk.Pixbuf.LoadFromResource("RestrictionTrackerGTK.Resources.modem16.png"), Gdk.Pixbuf.LoadFromResource("RestrictionTrackerGTK.Resources.modem32.png"), token, null); return; } try { System.Net.IPAddress[] urlRes = System.Net.Dns.GetHostAddresses(URI.Host); foreach (System.Net.IPAddress addr in urlRes) { if (addr.ToString() == "192.168.100.1") { c_callback.Invoke(Gdk.Pixbuf.LoadFromResource("RestrictionTrackerGTK.Resources.modem16.png"), Gdk.Pixbuf.LoadFromResource("RestrictionTrackerGTK.Resources.modem32.png"), token, null); return; } } } catch (Exception) { } try { WebClientEx wsString = new WebClientEx(); wsString.KeepAlive = false; wsString.ErrorBypass = true; wsString.ManualRedirect = false; string sRet = wsString.DownloadString(URI.OriginalString); if (sRet.StartsWith("Error: ")) { try { ConnectToFile(new Uri(URI.Scheme + "://" + URI.Host + "/favicon.ico"), Path.Combine(modFunctions.AppData, "srt_nettest_favicon.ico"), token, false); } catch (Exception) { } return; } try { string sHTML = sRet; if (sHTML.ToLower().Contains("shortcut icon")) { int startsAt = sHTML.ToLower().IndexOf("shortcut icon"); int endsAt = startsAt + 13; sHTML = sHTML.Substring(0, startsAt) + "icon" + sHTML.Substring(endsAt); } if (sHTML.ToLower().Contains("rel=\"icon\"")) { if (sHTML.Substring(0, sHTML.ToLower().IndexOf("rel=\"icon\"")).Contains("<")) { sHTML = sHTML.Substring(sHTML.Substring(0, sHTML.ToLower().IndexOf("rel=\"icon\"")).LastIndexOf("<")); if (sHTML.Contains(">")) { sHTML = sHTML.Substring(0, sHTML.IndexOf(">") + 1); if (sHTML.ToLower().Contains("href")) { sHTML = sHTML.Substring(sHTML.IndexOf("href")); if (sHTML.Contains("\"")) { sHTML = sHTML.Substring(sHTML.IndexOf("\"") + 1); if (sHTML.Contains("\"")) { string URL = sHTML.Substring(0, sHTML.IndexOf("\"")); if (URL.Contains("://")) { } else if (URL.Contains("//")) { string oldURL = (string)URI.OriginalString; if (oldURL.Contains("://")) { oldURL = oldURL.Substring(0, oldURL.IndexOf("://") + 1); } URL = oldURL + URL; } else { string oldURL = (string)URI.OriginalString; if (!oldURL.EndsWith("/") & oldURL.IndexOf("/", oldURL.IndexOf("//") + 2) > -1) { oldURL = oldURL.Substring(0, oldURL.LastIndexOf("/") + 1); } if (URL.StartsWith("/")) { if (oldURL.IndexOf("/", oldURL.IndexOf("//") + 2) > -1) { oldURL = oldURL.Substring(0, oldURL.IndexOf("/", oldURL.IndexOf("//") + 2)); } URL = oldURL + URL; } else if (oldURL.EndsWith("/")) { URL = oldURL + URL; } else { URL = oldURL + "/" + URL; } } try { ConnectToFile(new Uri(URL), Path.Combine(modFunctions.AppData, "srt_nettest_favicon.ico"), token, true); } catch (Exception) { } return; } } } } } } } catch (Exception) { } try { ConnectToFile(new Uri(URI.Scheme + "://" + URI.Host + "/favicon.ico"), Path.Combine(modFunctions.AppData, "srt_nettest_favicon.ico"), token, true); } catch (Exception) { } } catch (Exception) { c_callback.Invoke(Gdk.Pixbuf.LoadFromResource("RestrictionTrackerGTK.Resources.error.png"), Gdk.Pixbuf.LoadFromResource("RestrictionTrackerGTK.Resources.config.linux.advanced_nettest_error.png"), token, new Exception(" Failed to initialize connection to \"" + URI.OriginalString + "\"!")); } }
/* * private void DeleteServerSideData() * { * if (_abort.WaitOne(0)) * return; * * using (EmailClient ec = new EmailClient()) * { * if (ec.Connect()) * { * ec.DeleteAllMessages(); * * Log($"Server side data deleted..."); * } * } * } */ private void FetchSstFile(DateTime selDate) { if (_abort.WaitOne(0)) { return; } string vid_did_tid = ""; string vid = ConfigurationManager.AppSettings["noaaSst_VID"]; string did = ConfigurationManager.AppSettings["noaaSst_DID"]; string lookupText = ConfigurationManager.AppSettings["noaaSst_Lookup"]; string sstSearchUrl = ConfigurationManager.AppSettings["noaaSstSearchUrl"]; if (string.IsNullOrEmpty(sstSearchUrl) == false) { using (WebClientEx wc = new WebClientEx()) { Log($"Requesting TID data from: {sstSearchUrl}"); string response = wc.DownloadString(sstSearchUrl); using (HtmlLookup lookup = new HtmlLookup(response)) { var elements = lookup.GetElements("a", "href", lookupText); if (elements?.Count > 0) { elements.Sort((s1, s2) => { return(string.Compare(s2, s1, true)); }); foreach (var elem in elements) { var text = elem.ToLowerInvariant(); if (text.Contains($"db_did={did}") && text.Contains($"db_vid={vid}")) { vid_did_tid = elements[0].Replace(lookupText, string.Empty); break; } } } } } } if (string.IsNullOrEmpty(vid_did_tid)) { Log($"Invalid VID_DID_TID data. Cannot continue."); return; } Log($"Using: {vid_did_tid}"); string sstRequestUrlFmt = ConfigurationManager.AppSettings["noaaSstGetUrl"]; if (string.IsNullOrEmpty(sstRequestUrlFmt) == false) { sstRequestUrlFmt = sstRequestUrlFmt .Replace("##YEAR##", selDate.Year.ToString()) .Replace("##MONTH##", selDate.ToString("MMM")) .Replace("##DAY##", selDate.Day.ToString()) .Replace("##VID_PID_TID##", vid_did_tid); using (WebClientEx wc = new WebClientEx()) { Log($"Requesting SST data from: {sstRequestUrlFmt}"); string response = wc.DownloadString(sstRequestUrlFmt); using (HtmlLookup lookup = new HtmlLookup(response)) { List <string> elements = lookup.GetElements("a", "href", ".nc"); if (elements?.Count > 0) { string file = Path.Combine(SimulationData.WorkFolder, "SST.nc"); if (File.Exists(file)) { File.Delete(file); } Log($"Downloading effective SST data from: {elements[0]}"); wc.DownloadFile(elements[0], file); } } } } }
public SessionPageRequest(WebClientEx client, string url, int sessionID) : base(url, client) { this.SessionID = sessionID; this.Session = new Session(); }
public abstract void DownloadDataHandlerAsString(WebClientEx webClient, string result);
private QueuedJobDto DsGetNextJob() { try { QueuedJobDto nextJob = null; string pollerResult; using (WebClientEx client = new WebClientEx()) { //TODO: use round robin if a document store is down. var firstUrl = _dsEndpoints.First(); var payload = JsonConvert.SerializeObject(new { QueueName = this.QueueName, Identity = this._identity, Handle = this._handle, }); Logger.DebugFormat("Polling url: {0} with payload {1}", firstUrl, payload); client.Headers[HttpRequestHeader.ContentType] = "application/json"; pollerResult = client.UploadString(firstUrl.GetNextJobUrl, payload); Logger.DebugFormat("GetNextJobResult: {0}", pollerResult); } if (!pollerResult.Equals("null", StringComparison.OrdinalIgnoreCase)) { nextJob = JsonConvert.DeserializeObject<QueuedJobDto>(pollerResult, _settings); } _lastGoodCommunication = DateTime.Now; return nextJob; } catch (Exception ex) { if (DateTime.UtcNow.AddMinutes(-15) > _lastCommunicationError) { //new error in 15 minutes, we need to log Logger.ErrorFormat(ex, "Unable to contact Document Store at address: {0}", _dsEndpoints.First()); _lastCommunicationError = DateTime.UtcNow; } else { Logger.InfoFormat("Document store cannot be reached, down since {0}", _lastGoodCommunication); } return null; } }
private string getSpaceTrack(string predicateValues) { string requestController = "/basicspacedata"; string requestAction = "/query"; string request = uriBase + requestController + requestAction + predicateValues; // Create new WebClient object to communicate with the service using (var client = new WebClientEx(_cookies)) { // Generate the URL for the API Query and return the response var response = client.DownloadData(request); return (System.Text.Encoding.Default.GetString(response)); } }
///<summary>Returns the status code of the web call made to Podium. ///200-299 indicates a successful response, >=300 indicates a Podium error response, negative status codes indicate an Open Dental error.</summary> private static int MakeWebCall(string apiToken, string apiUrl, string locationId, string phoneNumber, Patient pat) { int retVal = -100; if (pat.TxtMsgOk == YN.No || (pat.TxtMsgOk == YN.Unknown && PrefC.GetBool(PrefName.TextMsgOkStatusTreatAsNo))) //Use email. { retVal = -200; //Our code for 'no email address' phoneNumber = ""; //Ensure that there is no way we send a phone number to Podium if the patient doesn't approve text messages. if (string.IsNullOrEmpty(pat.Email)) { return(retVal); //Will be -200, meaning no email and can't text. } } if (string.IsNullOrWhiteSpace(phoneNumber) && string.IsNullOrWhiteSpace(pat.Email)) { return(retVal); } //We either have a phoneNumber or email (or both), so send it to Podium. string isTestString = "false"; #if DEBUG isTestString = "true"; #endif try { using (WebClientEx client = new WebClientEx()) { client.Headers[HttpRequestHeader.Accept] = "application/json"; client.Headers[HttpRequestHeader.ContentType] = "application/json"; client.Headers[HttpRequestHeader.Authorization] = "Token token=\"" + apiToken + "\""; client.Encoding = UnicodeEncoding.UTF8; string bodyJson = string.Format(@" {{ ""locationId"": ""{0}"", ""lastName"": ""{3}"", ""firstName"": ""{2}"", ""email"": ""{4}"", ""phoneNumber"": ""{1}"", ""integrationName"": ""opendental"", ""test"": {5} }}" , locationId, phoneNumber, pat.FName, pat.LName, pat.Email, isTestString); //Post with Authorization headers and a body comprised of a JSON serialized anonymous type. client.UploadString(apiUrl, "POST", bodyJson); retVal = (int)(client.StatusCode); } } catch (WebException we) { if (we.Response.GetType() == typeof(HttpWebResponse)) { retVal = (int)((HttpWebResponse)we.Response).StatusCode; } } catch (Exception) { //Do nothing because a verbose commlog will be made below if all phone numbers fail. } return(retVal); //Sample Request: //Accept: 'application/json's //Content-Type: 'application/json' //Authorization: 'Token token="my_dummy_token"' //Body: //{ // "location_id": "54321", // "phone_number": "1234567890", // "customer": { // "first_name": "Johnny", // "last_name": "Appleseed", // "email": "*****@*****.**" // }, // "test": true //} //NOTE: There will never be a value after "customer": although it was initially interpreted that there would be a "new" flag there. }
protected string DownloadContent(string url) { return(WebClientEx.DownloadContent(LogManager, url, 5, 15000)); }
private void DownloadAndInstallAsync() { bool exitApp = false; try { string tmpGZFile = Path.GetTempFileName(); string tmpPkgFile = Path.GetTempFileName(); string tmpFolder; do { tmpFolder = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); } while (Directory.Exists(tmpFolder)); Directory.CreateDirectory(tmpFolder); try { #region download update file using (WebClientEx client = new WebClientEx()) { client.Proxy = _proxy; client.UserAgent = GetUserAgent(); client.DownloadFile(_updateInfo.DownloadURI, tmpGZFile); } #endregion #region verify signature, extract & execute using (FileStream sGZ = new FileStream(tmpGZFile, FileMode.Open, FileAccess.Read)) { //verify signature if (!_updateInfo.DownloadSignature.Verify(sGZ, _trustedRootCAs)) { throw new Exception("Update file signature does not match."); } using (FileStream sPkg = new FileStream(tmpPkgFile, FileMode.Create, FileAccess.ReadWrite)) { //unzip sGZ.Position = 0; using (GZipStream unzip = new GZipStream(sGZ, CompressionMode.Decompress, true)) { unzip.CopyTo(sPkg); } //extract sPkg.Position = 0; Package updatePackage = new Package(sPkg, PackageMode.Open); updatePackage.ExtractAll(ExtractLocation.Custom, tmpFolder, true); //execute switch (Environment.OSVersion.Platform) { case PlatformID.Win32NT: { string appFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().CodeBase.Replace("file:///", "").Replace("/", "\\")); ProcessStartInfo pInfo = new ProcessStartInfo(Path.Combine(tmpFolder, "update.exe"), "\"" + _mutexName + "\" \"" + appFolder + "\""); pInfo.UseShellExecute = true; pInfo.Verb = "runas"; Process.Start(pInfo); } break; case PlatformID.Unix: { string appFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().CodeBase.Replace("file://", "")); ProcessStartInfo pInfo = new ProcessStartInfo("mono", "\"" + Path.Combine(tmpFolder, "update.exe") + "\" \"" + _mutexName + "\" \"" + appFolder + "\""); pInfo.UseShellExecute = false; Process.Start(pInfo); } break; default: throw new Exception("Platform not supported."); } exitApp = true; } } #endregion } finally { File.Delete(tmpGZFile); File.Delete(tmpPkgFile); } } catch (Exception ex) { RaiseEventUpdateError(ex); } _downloadInstall = null; if (exitApp) { RaiseEventExitApplication(); } }
private void CheckForUpdateAsync(object state) { if (state == null) { if (DateTime.UtcNow < _lastUpdateCheckedOn.AddDays(_checkUpdateIntervalDays)) { return; } } //update last check time _lastUpdateCheckedOn = DateTime.UtcNow; try { using (WebClientEx client = new WebClientEx()) { client.Proxy = _proxy; client.UserAgent = GetUserAgent(); client.IfModifiedSince = _lastModifiedGMT; using (Stream s = client.OpenRead(_checkUpdateURL)) { _updateInfo = new UpdateInfo(s); } _lastModifiedGMT = DateTime.Parse(client.ResponseHeaders["Last-Modified"]); if (_updateInfo.IsUpdateAvailable(_currentVersion)) { //update available, stop timer _checkTimer.Change(Timeout.Infinite, Timeout.Infinite); //raise event to user UI RaiseEventUpdateAvailable(); } else { //if manual check then raise event if (state != null) { RaiseEventNoUpdateAvailable(); } } } } catch (WebException ex) { HttpWebResponse response = ex.Response as HttpWebResponse; if (response != null) { switch (response.StatusCode) { case HttpStatusCode.NotModified: case HttpStatusCode.NotFound: //if manual check then raise event if (state != null) { RaiseEventNoUpdateAvailable(); } break; default: //if manual check then raise event if (state != null) { RaiseEventUpdateError(ex); } break; } } else { //if manual check then raise event if (state != null) { RaiseEventUpdateError(ex); } } } catch (Exception ex) { //if manual check then raise event if (state != null) { RaiseEventUpdateError(ex); } } if (state != null) { _checkUpdate = null; } }
protected override void UpdateTracker(TrackerClientEvent @event, IPEndPoint clientEP) { string queryString; queryString = "?info_hash=" + Uri.EscapeDataString(Encoding.ASCII.GetString(_infoHash)) + "&peer_id=" + Uri.EscapeDataString(Encoding.ASCII.GetString(_clientID.PeerID)); switch (clientEP.Address.ToString()) { case "0.0.0.0": case "127.0.0.1": case "::": case "::1": break; default: if (clientEP.Address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) { queryString += "&ipv6=" + clientEP.Address.ToString(); } else { queryString += "&ip=" + clientEP.Address.ToString(); } break; } queryString += "&port=" + clientEP.Port + "&uploaded=0&downloaded=0&left=0&corrupt=0" + "&key=" + BitConverter.ToString(_clientID.ClientKey).Replace("-", ""); switch (@event) { case TrackerClientEvent.Started: queryString += "&event=started"; break; case TrackerClientEvent.Stopped: queryString += "&event=stopped"; break; case TrackerClientEvent.Completed: queryString += "&event=completed"; break; } queryString += "&numwant=" + _clientID.NumWant; if (_clientID.Compact) { queryString += "&compact=1"; } else { queryString += "&compact=0"; } if (_clientID.NoPeerID) { queryString += "&no_peer_id=1"; } using (WebClientEx webClient = new WebClientEx()) { webClient.Proxy = _proxy; webClient.Timeout = 30000; //30 sec timeout webClient.UserAgent = _clientID.HttpUserAgent; webClient.AddHeader("Accept-Encoding", _clientID.HttpAcceptEncoding); webClient.KeepAlive = false; using (Stream responseStream = webClient.OpenRead(_trackerURI.AbsoluteUri + queryString)) { switch (@event) { case TrackerClientEvent.None: case TrackerClientEvent.Started: Bencoding x = Bencoding.Decode(responseStream); switch (x.Type) { case BencodingType.Dictionary: _peers.Clear(); foreach (var item in x.ValueDictionary) { switch (item.Key) { case "peers": switch (item.Value.Type) { case BencodingType.String: ParseCompactPeersIPv4(item.Value.Value as byte[], _peers); break; case BencodingType.List: foreach (var peerObj in item.Value.ValueList) { var peer = peerObj.ValueDictionary; _peers.Add(new IPEndPoint(IPAddress.Parse(peer["ip"].ValueString), Convert.ToInt32(peer["port"].ValueInteger))); } break; } break; case "peers_ipv6": case "peers6": switch (item.Value.Type) { case BencodingType.String: ParseCompactPeersIPv6(item.Value.Value as byte[], _peers); break; } break; case "interval": if (item.Value.Type == BencodingType.Integer) { _interval = Convert.ToInt32(item.Value.Value); } break; case "min interval": if (item.Value.Type == BencodingType.Integer) { _minInterval = Convert.ToInt32(item.Value.Value); } break; } } break; default: throw new TrackerClientException("Invalid data received from tracker. Expected bencoded dictionary."); } break; } } } }
public abstract void DownloadStringHandler(WebClientEx webClient, string result);
private void DsReQueueJob(string queueName, string jobId, string message, TimeSpan timeSpan) { string pollerResult; using (WebClientEx client = new WebClientEx()) { //TODO: use round robin if a document store is down. var firstUrl = _dsEndpoints.First(); var payload = JsonConvert.SerializeObject(new { QueueName = queueName, JobId = jobId, ErrorMessage = message, ReQueue = true, ReScheduleTimespanInSeconds = (Int32) timeSpan.TotalSeconds }); Logger.DebugFormat("ReQueuedJob url: {0} with payload {1}", firstUrl.SetJobCompleted, payload); client.Headers[HttpRequestHeader.ContentType] = "application/json"; pollerResult = client.UploadString(firstUrl.SetJobCompleted, payload); Logger.DebugFormat("SetJobExecuted Result: {0}", pollerResult); } }
public static WebClient GetWebClient() { WebClientEx localClient = new WebClientEx(((WebClientEx)Client).CookieContainer); return(localClient); }
/// <summary> /// Main method /// </summary> /// <param name="serviceProvider">Service Provider</param> public void Execute(IServiceProvider serviceProvider) { if (serviceProvider == null) { return; } ITracingService tracer = (ITracingService)serviceProvider.GetService(typeof(ITracingService)); context = (IPluginExecutionContext)serviceProvider.GetService(typeof(IPluginExecutionContext)); IOrganizationServiceFactory factory = (IOrganizationServiceFactory)serviceProvider.GetService(typeof(IOrganizationServiceFactory)); IOrganizationService service = factory.CreateOrganizationService(context.UserId); try { EntityReference currentRecordReference = (EntityReference)context.InputParameters["Target"]; if (currentRecordReference.Equals(null) || currentRecordReference.LogicalName != Model.MailChimpSync.LOGICAL_NAME) { tracer.Trace("target record: " + currentRecordReference.LogicalName.ToString()); tracer.Trace("Does not contain target or logical name is different."); return; } //// Retrieve the mail chimp sync record Entity currentRecord = service.Retrieve(currentRecordReference.LogicalName, currentRecordReference.Id, new ColumnSet(Model.MailChimpSync.ATTR_BATCHID, Model.MailChimpSync.ATTR_MARKETING_LIST)); string batchId = string.Empty; Guid marketingListId = Guid.Empty; if (string.IsNullOrEmpty(currentRecord.GetAttributeValue <string>(Model.MailChimpSync.ATTR_BATCHID))) { tracer.Trace("Mail chimp Sync Batch ID is not present"); throw new InvalidPluginExecutionException("Mail chimp Sync Batch ID is not present"); } else { batchId = currentRecord.GetAttributeValue <string>(Model.MailChimpSync.ATTR_BATCHID); } if ((currentRecord.GetAttributeValue <EntityReference>(Model.MailChimpSync.ATTR_MARKETING_LIST)) != null) { marketingListId = currentRecord.GetAttributeValue <EntityReference>(Model.MailChimpSync.ATTR_MARKETING_LIST).Id; } string getBatchResponseJSON = string.Empty; Entity mailChimpconfig = null; mailChimpconfig = this.RetrieveMailChimpConfiguration(tracer, service); string username = string.Empty; string password = string.Empty; string api = string.Empty; string basicURL = string.Empty; if (mailChimpconfig.Contains(Model.MailChimpConfiguration.ATTR_APIKEY)) { api = mailChimpconfig.Attributes[Model.MailChimpConfiguration.ATTR_APIKEY].ToString(); tracer.Trace("API key present" + api); } if (mailChimpconfig.Contains(Model.MailChimpConfiguration.ATTR_MAILCHIMPURL)) { basicURL = mailChimpconfig.Attributes[Model.MailChimpConfiguration.ATTR_MAILCHIMPURL].ToString(); tracer.Trace("URL present" + basicURL); } if (mailChimpconfig.Contains(Model.MailChimpConfiguration.ATTR_MAILCHIMP_USERNAME)) { username = mailChimpconfig.Attributes[Model.MailChimpConfiguration.ATTR_MAILCHIMP_USERNAME].ToString(); tracer.Trace("UserName present" + username); } if (mailChimpconfig.Contains(Model.MailChimpConfiguration.ATTR_MAILCHIMP_PASSWORD)) { password = mailChimpconfig.Attributes[Model.MailChimpConfiguration.ATTR_MAILCHIMP_PASSWORD].ToString(); tracer.Trace("Password present" + password); } //// Call the web service using (WebClientEx client = new WebClientEx()) { string authorizationKey = string.Empty; authorizationKey = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "{0}:{1}", username, api))); basicURL = basicURL + "/" + batchId; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(basicURL); request.Accept = "application/json"; request.Method = "GET"; request.Headers.Add("Authorization", "Basic " + authorizationKey); using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) using (Stream stream = response.GetResponseStream()) using (StreamReader reader = new StreamReader(stream)) { getBatchResponseJSON = reader.ReadToEnd(); } } tracer.Trace("createBatchResponse :" + getBatchResponseJSON); Model.MailChimpContactCreateBatchResponse createBatchResponse = GetInfoFromJSON(getBatchResponseJSON); UpdateMailChimpSyncRecord(createBatchResponse, tracer, service, currentRecord.Id); } catch (FaultException <OrganizationServiceFault> ex) { throw new InvalidPluginExecutionException("An error occurred in the Follow up Plugin plug-in.", ex); } catch (Exception ex) { tracer.Trace("Message: {0}", ex.ToString()); throw; } }
public abstract void DownloadDataHandlerAsIntArray(WebClientEx webClient, int[] result);
public static String DownloadString(WebClientEx client, String uri) { client.Headers["Content-Type"] = "application/json; charset=utf-8"; return(client.DownloadString(uri)); }
public BaseHtmlRequest(string url, WebClientEx Client) : base(url, Client) { InitializeEvent(); }
public WebDatabase(Uri webDatabaseUri, WebClientEx webClient) { _webDatabaseUri = webDatabaseUri; _webClient = webClient; }
private void button2_Click(object sender, EventArgs e) { WebClientEx wb = new WebClientEx(); Regex r = new Regex(@"/account/\?pool=[0-9]*"); String src = wb.DownloadString("http://puush.me/login/go/?k=" + key); if (checkBoxPublic.Checked) { backupPublic(r.Matches(src)[0].ToString()); } if (checkBoxPrivate.Checked) { backupPrivate(r.Matches(src)[1].ToString()); } if (checkBoxGallery.Checked) { backupGallery(r.Matches(src)[2].ToString()); } }
public bool Next(System.Threading.CancellationToken ct, out System.Collections.Generic.IEnumerable <IArticle> elems) { WebClientEx client = new WebClientEx(); string result = client.DownloadStringAsyncTask( new Uri(string.Format("http://clien.career.co.kr/cs2/bbs/board.php?bo_table={0}&page={1}&{2}", id, page + 1, DateTime.Now.Ticks), UriKind.Absolute), ct).GetResult(); StringEngine se = new StringEngine(result); List <IArticle> articles = new List <IArticle>(); bool curBool = true; while (true) { Match match; if (!se.Next(new Regex("<tr class=\"mytr\">"), out match)) { break; } string line; if (!se.GetNextLine(out line)) { continue; } match = Regex.Match(line, @"<td>(\d+)</td>"); if (!match.Success) { continue; } string articleID = match.Groups[1].Value; int curArticleID = int.Parse(articleID); if (recentArticle <= curArticleID) { continue; } recentArticle = curArticleID; ClienArticle article = new ClienArticle(board, articleID); article.HasImage = curBool; curBool = !curBool; // 글 제목과 댓글 개수 if (!se.GetNextLine(out line)) { continue; } match = Regex.Match(line, @"<a[^>]*?>(.*?)</a>\s*(<span>\[(\d+)\]</span>)?"); if (!match.Success) { continue; } article.Title = match.Groups[1].Value; if (match.Groups[3].Success) { article.CommentCount = int.Parse(match.Groups[3].Value); } else { article.CommentCount = 0; } // 이름 if (!se.GetNextLine(out line)) { continue; } match = Regex.Match(line, @"<span class='member'>(.*?)</span>"); if (match.Success) { article.Name = match.Groups[1].Value; } else { match = Regex.Match(line, @"<img src='/cs2/data/member/.*?/(.*?).gif"); if (!match.Success) { continue; } article.Name = match.Groups[1].Value; } // 시간 if (!se.GetNextLine(out line)) { continue; } match = Regex.Match(line, "<span title=\"([^\"]*?)\">"); if (!match.Success) { continue; } article.Date = DateTime.Parse(match.Groups[1].Value); articles.Add(article); } elems = articles; page++; return(true); }
/// <summary> /// Initializes a new instance of the MainViewModel class. /// </summary> public MainViewModel(IDataAccessService servPxy) { try { _serviceProxy = servPxy; Database.SetInitializer(new AvitoModelInitializer()); //ID приложения: 2355847 //Создание браузерной группы BrowserClientEx = new WebClientEx(); BrowserClientEx.Encoding = new UTF8Encoding(); BrowserClientEx.DefaultHeaders[HttpRequestHeader.UserAgent] = Settings.Default.WebClientAgent; BrowserClientEx.RequestTimeOut = Settings.Default.WebClientTimeOut * 1000; //Настройки CefSharp CefSharp.Wpf.CefSettings cfsettings = new CefSharp.Wpf.CefSettings { CachePath = Environment.CurrentDirectory + @"\browserCache", UserAgent = Settings.Default.WebClientAgent, }; Cef.Initialize(cfsettings); ChBrowser = new CefSharpWrapper(); _requestHandler = new RequestHandler(); ChBrowser.RequestHandler = _requestHandler; BrowserClientEx.CHW = ChBrowser; AvitoUsers = servPxy.GetUsers; CurrentDetachedUser = (AvitoUsers.LastOrDefault() != null) ? _serviceProxy.GetDetachedEntity(AvitoUsers.LastOrDefault()) : new AvitoUser(); } catch (Exception exception) { Console.WriteLine(exception); MessageBox.Show(exception.Message, "Ошибка инициализации приложения", MessageBoxButton.OK, MessageBoxImage.Error); } // WINDOW COMMANDS WindowClosing = new RelayCommand <CancelEventArgs>((s) => { Cef.Shutdown(); }); AuthenticateCommand = new RelayCommand(DoAvitoLogin); CallUserListCommand = new RelayCommand(OpenUserListWindow); CallProfileNavigateCommand = new RelayCommand(() => { ChBrowser.Address = "https://www.avito.ru/profile"; }); CallProfileRegisterCommand = new RelayCommand(AddNewProfile); CallAvitoTechnicalCommand = new RelayCommand(() => { ChBrowser.Address = "https://www.avito.ru/info/show_technical"; }); CallHidePanelCommand = new RelayCommand(HideLeftPanel); CallViewPhotosCommand = new RelayCommand(() => { Process.Start(Environment.CurrentDirectory + @"\images"); }); CallLicenseInfoCommand = new RelayCommand(OpenLicenseInfo); CallDebugInfoCommand = new RelayCommand(OpenDebugWindow); CallReloadBrowserCommand = new RelayCommand(() => CefSharp.WebBrowserExtensions.Reload(ChBrowser, true)); //TODO: вынести в отдельную viewmodel LogInAsUserCommand = new RelayCommand <AvitoUser>(LogInAsUser); RefreshUserInformationCommand = new RelayCommand <AvitoUser>(RefreshUserInformation); DropCredentialsCommand = new RelayCommand <AvitoUser>(DropCredentials); DeleteUserCommand = new RelayCommand <AvitoUser>(DeleteUser); SetDefaultPhoneCommand = new RelayCommand <AvitoUser>(SetDefaultPhone); }
public void event_btnUpload_Activated(object sender, EventArgs e) { if (string.IsNullOrEmpty (txtUserName.StringValue) || string.IsNullOrEmpty (txtPassWord.StringValue)) { var alert = new NSAlert { MessageText = "Username or password empty.", AlertStyle = NSAlertStyle.Informational }; alert.AddButton ("Ok"); alert.RunModal(); return; } FileInfo fInfo = new FileInfo(txtFullSceneName.StringValue); long numBytes = fInfo.Length; FileStream fStream = new FileStream(txtFullSceneName.StringValue, FileMode.Open, FileAccess.Read); BinaryReader br = new BinaryReader(fStream); FormUpload.FileParameter fp = new FormUpload.FileParameter (br.ReadBytes((int)numBytes), Path.GetFileName (txtFullSceneName.StringValue), "application/octet-stream"); br.Close(); fStream.Close(); string releaseType = string.Empty; if (cmbType.StringValue == "App") releaseType = "1"; if (cmbType.StringValue == "Game") releaseType = "2"; var client = new WebClientEx (); var values = new NameValueCollection { { "username", txtUserName.StringValue }, { "password", txtPassWord.StringValue }, }; // Authenticate client.UploadValues("https://mac-torrents.me/login.php", values); string tempResponse = client.DownloadString ("https://mac-torrents.me/upload.php"); string searchFor = "<input type=\"hidden\" name=\"auth\" value=\""; int indexSearchFor = tempResponse.IndexOf (searchFor) + searchFor.Length; string authString = tempResponse.Substring (indexSearchFor, 32); // Upload torrent Dictionary<string, object> paramsDic = new Dictionary<string, object> (); paramsDic.Add ("submit", "true"); paramsDic.Add ("auth", authString); paramsDic.Add ("type", releaseType); paramsDic.Add ("title", txtTitleCode.StringValue); paramsDic.Add ("tags", txtTagsCode.StringValue); paramsDic.Add ("image", txtImageCode.StringValue); paramsDic.Add ("desc", txtDescriptionCode.Value); paramsDic.Add ("file_input", fp); if (cmbComp.StringValue == "10.6") { paramsDic.Add ("macos[]", new string[] {"1", "2", "3", "4"}); } if (cmbComp.StringValue == "10.7") { paramsDic.Add ("macos[]", new string[] {"2", "3", "4"}); } if (cmbComp.StringValue == "10.8") { paramsDic.Add ("macos[]", new string[] {"3", "4"}); } if (cmbComp.StringValue == "10.9") { paramsDic.Add ("macos[]", "4"); } //Upload torrent FormUpload.MultipartFormDataPost ( "https://mac-torrents.me/upload.php", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36", paramsDic, client.CookieContainer); try { tempResponse = client.DownloadString ("https://mac-torrents.me/logout.php?auth=" + authString); } catch { } txtNfoFilename.StringValue = ""; txtDescription.Value = ""; txtDescriptionCode.Value = ""; txtTagsCode.StringValue = ""; txtImageCode.StringValue = ""; var alertEnd = new NSAlert { MessageText = "Done!", AlertStyle = NSAlertStyle.Informational }; alertEnd.AddButton ("Ok"); alertEnd.RunModal(); }
public void authenticate() { using (var client = new WebClientEx(_cookies)) { var data = new NameValueCollection { { "identity", _username }, { "password", _password }, }; var response = client.UploadValues(uriBase + "/ajaxauth/login", data); } }
private static async Task <string> CrawlGitHubForToc(string url, StringBuilder sb = null, string rootUrl = null) { lock (CrawledGitHubRepositories) if (CrawledGitHubRepositories.ContainsKey(url)) { return(CrawledGitHubRepositories[url]); } if (rootUrl == null) { rootUrl = url; } var sbWasNull = false; if (sb == null) { sb = new StringBuilder(); sb.Append($"{{ \"title\": \"{url}\", \"topics\": ["); sbWasNull = true; } try { var html = await WebClientEx.GetStringAsync(url); var htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(html); var navigationItems = htmlDoc.QuerySelectorAll("tr.js-navigation-item"); foreach (var navigationItem in navigationItems) { // First, we look for all folders var icon = navigationItem.QuerySelector("td.icon"); if (icon != null) { var svg = icon.QuerySelector("svg.octicon-file-directory"); if (svg != null) { // This is a folder var content = navigationItem.QuerySelector("td.content span a"); if (content != null) { var slug = GetSlugFromLink(url, rootUrl, content.InnerText); sb.Append("{"); sb.Append($"\"title\": \"{content.InnerText}\", \"slug\": \"{slug}\""); sb.Append(", \"topics\": ["); var linkUrl = "https://github.com" + content.GetAttributeValue("href", string.Empty); await CrawlGitHubForToc(linkUrl, sb, rootUrl); sb.Append("]},"); } } } // Now, we look for all the local documents var icon2 = navigationItem.QuerySelector("td.icon"); if (icon2 != null) { var svg = icon2.QuerySelector("svg.octicon-file"); if (svg != null) { // This is a file var content = navigationItem.QuerySelector("td.content span a"); if (content != null) { var text = content.InnerText; if (ShouldFileBeIncludedInCrawl(text)) { var title = StringHelper.JustFileName(text); var slug = GetSlugFromLink(url, rootUrl, title); var link = content.GetAttributeValue("href", string.Empty); var blobMasterIndex = link.IndexOf("/blob/master/", StringComparison.InvariantCultureIgnoreCase); if (blobMasterIndex > -1) { link = link.Substring(blobMasterIndex + 13); } sb.Append("{"); sb.Append($"\"title\": \"{title}\", \"link\": \"{link}\", \"slug\": \"{slug}\""); sb.Append("},"); } } } } } } catch { // Oh well! Nothing we can do } if (sbWasNull) { sb.Append("]}"); var resultToc = sb.ToString(); lock (CrawledGitHubRepositories) if (CrawledGitHubRepositories.ContainsKey(url)) { CrawledGitHubRepositories[url] = resultToc; } else { CrawledGitHubRepositories.Add(url, resultToc); } return(resultToc); } return(string.Empty); }
public List <byte> Send(String sessionPayload, String status, List <byte> payload, out bool commandChannelDead) { commandChannelDead = false; if (String.IsNullOrWhiteSpace(status)) { status = "nochange"; } var sessionAndStatus = sessionPayload + ":" + status; var encryptedSessionPayload = _encryption.Encrypt(UTF8Encoding.UTF8.GetBytes(sessionAndStatus).ToList()); var cookies = new CookieContainer(); WebClientEx wc = null; if (!String.IsNullOrWhiteSpace(_config.HostHeader)) { wc = new WebClientEx(cookies, _config.HostHeader, _config.InsecureSSL) { UserAgent = _config.UserAgent } } ; else { wc = new WebClientEx(cookies, _config.InsecureSSL) { UserAgent = _config.UserAgent } }; if (_config.UseProxy) { if (null == _config.WebProxy) { wc.Proxy = HttpWebRequest.GetSystemWebProxy(); } else { wc.Proxy = _config.WebProxy; } } wc.Headers.Add("Host", _config.HostHeader); cookies.Add(new Cookie($"{_config.SessionCookieName}", $"{encryptedSessionPayload}") { Domain = (!String.IsNullOrWhiteSpace(_config.HostHeader)) ? _config.HostHeader.Split(':')[0] : _config.URL.Host }); string encPayload = null; if (null != payload && payload.Count > 0) { try { encPayload = _encryption.Encrypt(payload); if (String.IsNullOrWhiteSpace(encPayload)) { _error.LogError("Encrypted payload was null, it shouldn't be"); if (!InitialConnectionSucceded.HasValue) { InitialConnectionSucceded = false; } return(null); } } catch (Exception ex) { _error.LogError(ex.Message); return(null); } } bool retryRequired = false; Int32 retryInterval = 2000; UInt16 retryCount = 0; Guid errorId = Guid.NewGuid(); //This is only if the command channel has failed first time do { try { String response = null; if (encPayload != null && encPayload.Count() > 4096) { response = wc.UploadString(BuildServerURI(), encPayload); } else { if (null != _config.HostHeader) { if (wc.Headers.AllKeys.Contains("Host")) { if (wc.Headers["Host"] != _config.HostHeader) { wc.Headers["Host"] = _config.HostHeader; } } else { wc.Headers.Add("Host", _config.HostHeader); } } if (payload != null && payload.Count() > 0) { cookies.Add(new Cookie($"{_config.PayloadCookieName}", $"{encPayload}") { Domain = (!String.IsNullOrWhiteSpace(_config.HostHeader)) ? _config.HostHeader.Split(':')[0] : _config.URL.Host }); } response = wc.DownloadString(BuildServerURI()); } if (!InitialConnectionSucceded.HasValue) { InitialConnectionSucceded = true; } if (null != response && response.Count() > 0) { return(_encryption.Decrypt(response)); } else { return(new List <byte>()); } } catch (System.Net.WebException ex) { var lst = new List <String>(); if (WebExceptionAnalyzer.IsTransient(ex)) { if (15 > retryCount++) { _error.LogError($"Error has occured and looks like it's transient going to retry in {retryInterval} milliseconds: {ex.Message}"); retryRequired = true; if (retryInterval++ > 2) { retryInterval += retryInterval; } Timeout.WaitOne(retryInterval); } else { _error.FailError($"Kept trying but afraid error isn't going away {retryInterval} {ex.Message} {ex.Status.ToString()} {_config.CommandServerUI.ToString()} {errorId.ToString()}"); commandChannelDead = true; return(null); } } else if (sessionPayload == _config.CommandChannelSessionId) { if (!RetryUntilFailure(ref retryCount, ref retryRequired, ref retryInterval)) { lst.Add("Command channel re-tried connection 5 times giving up"); ReportErrorWebException(ex, lst, errorId); commandChannelDead = true; return(null); } retryRequired = true; } else { ReportErrorWebException(ex, lst, errorId); if (HttpStatusCode.NotFound == ((HttpWebResponse)ex.Response).StatusCode) { if (_error.VerboseErrors) { _error.LogError(String.Format($"Connection on server has been killed")); } } else { _error.LogError(String.Format($"Send to {_config.URL} failed with {ex.Message}")); } return(null); } } } while (retryRequired); if (!InitialConnectionSucceded.HasValue) { commandChannelDead = true; InitialConnectionSucceded = false; } return(null); } bool RetryUntilFailure(ref UInt16 retryCount, ref bool retryRequired, ref Int32 retryInterval) { if (5 <= retryCount++) { return(retryRequired = false); } _error.LogError($"Command Channel failed to connect : retry interval {retryInterval} ms"); Timeout.WaitOne(retryInterval); retryInterval += retryInterval; return(true); } Uri BuildServerURI(String payload = null) { if (null != _config.Tamper) { return(new Uri(_config.Tamper.TamperUri(_config.CommandServerUI, payload))); } if (_config.URLPaths.Count() == 0) { return(new Uri(_config.URL, "Upload")); } else { var path = _config.URLPaths[_urlRandomizer.Next(0, _config.URLPaths.Count())]; return(new Uri(_config.URL, path)); } } void ReportErrorWebException(System.Net.WebException ex, List <String> lst, Guid errorId) { lst.Add(ex.Message); lst.Add(ex.Status.ToString()); lst.Add(_config.CommandServerUI.ToString()); lst.Add(errorId.ToString()); _error.LogError(lst); } }
public ModulesPageRequest(WebClientEx client, string url) : base(url, client) { }