public void Convert(ufDateNode node, Stream stream, bool multiples, Urls urls, Errors errors, bool reporting)
{
m_bReporting = reporting;
m_oErrors = errors;
m_oUrls = urls;
Convert(node, stream, multiples);
}
protected internal void OnDeleteFile(IFileModel file)
{
Urls = Urls.Where(x => x != file.Url).ToArray();
if (UrlsChanged.HasDelegate)
{
_ = UrlsChanged.InvokeAsync(Urls);
}
RequireRender = true;
SetFieldValue(Urls, true);
}
public ProxyCrawler_Iphai()
{
Urls.AddRange(new string[] {
"http://www.iphai.com/free/ng",
"http://www.iphai.com/free/wg",
"http://www.iphai.com/free/np",
"http://www.iphai.com/free/wp",
"http://www.iphai.com/"
});
}
public bool ForceUrl()
{
if (!string.IsNullOrEmpty(m_websiteString))
{
Urls.Add(m_websiteString);
SetThirdLevelRequest();
return(true);
}
return(false);
}
public static PayResponse Request(string merchantId, long amount, string description, string callbackUrl)
{
bool sandBoxMode = merchantId.Equals(TestMerchantId);
HttpCore httpCore = new HttpCore(Urls.GetPaymentRequestUrl(sandBoxMode), "POST",
JsonConvert.SerializeObject(new PayRequest(merchantId, amount, description, callbackUrl)));
var res = JsonConvert.DeserializeObject <PayResponse>(httpCore.GetResponse());
res.Authority = res.Authority.TrimStart('0');
return(res);
}
protected override Task OnInitializedAsync()
{
Sessions.SetData(NatsService.Configuration.Sessions);
Urls.SetData(new UrlData[0]);
Subjects.SetData(new SubjectData[0]);
Sessions.SelectedItemChanged += OnSelectedItemChanged;
Sessions.ItemDoubleClicked += OnItemDoubleClicked;
Sessions.ItemClicked += OnItemClicked;
return(Task.CompletedTask);
}
public ProxyCrawlerData5u()
{
Urls.AddRange(new string[] {
"http://www.data5u.com/",
"http://www.data5u.com/free/gngn/index.shtml",
"http://www.data5u.com/free/gnpt/index.shtml",
"http://www.data5u.com/free/gwgn/index.shtml",
"http://www.data5u.com/free/gwpt/index.shtml"
});
}
public ProxyCrawler_Xdaili()
{
List <string> urls = new List <string>();
for (int i = 1; i <= 20; ++i)
{
urls.Add(string.Format("http://www.xdaili.cn/ipagent/freeip/getFreeIps?page={0}&rows=10", i));
}
Urls.AddRange(urls.ToArray());
}
public void isURLStringValid(string value)
{
var logger = new Mock <ILogger>();
Urls url = new Urls(value);
bool result = url.UrlValid();
Assert.False(result);
}
public FileUploadTestsSteps(IConfiguration config)
{
_filePath = config
.GetSection("Files")
.Get <Files>();
_url = config.GetSection("Urls")
.Get <Urls>();
WebDriver.driver.Navigate().GoToUrl(_url.FileUploadUrl);
}
public ApiClient(ISimpleLogger logger, IHttpClientFactory httpClientFactory, ICustomHttpClientFactory factory)
{
_logger = logger;
var httpClient = httpClientFactory.CreateClient(Name);
_client = factory.CreateClient(httpClient);
Urls = new Urls(_client.BaseAddress, 1);
}
private async Task SetActionUpdateMessages(Guid actionId, int propertyUpdated)
{
var action = await _dssReader.GetActionDetails(ViewModel.CustomerId.ToString(),
ViewModel.InteractionId.ToString(), ViewModel.ActionPlanId.ToString(), actionId.ToString());
ViewModel.Name = $"{action.ActionSummary} - {action.ActionType.GetDisplayName()}";
ViewModel.ObjLink = Urls.GetViewActionUrl(ViewModel.CompositeSettings.Path, actionId);
ViewModel.ObjLinkText = "view or update this action";
SetActionStatusMessages(propertyUpdated, action);
}
private void OnOk()
{
TextBoxBinding.ForceSourceUpdate();
Notification.TrySetConfirmed(true);
Notification.Content = Urls.ToList();
FinishInteraction();
}
/// <summary>
/// Marks a file as deleted.
/// </summary>
/// <param name="fileId"> Resource UUID </param>
public void DeleteFile(Guid fileId)
{
var url = Urls.ApiFile(fileId);
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "DELETE";
_requestHelper.ExecuteCommand(request, true);
}
/// <summary>
/// Marks a file as saved.
///
/// This has to be done for all files you want to keep.
/// Unsaved files are eventually purged.
/// </summary>
/// <param name="fileId"> Resource UUID </param>
public void SaveFile(Guid fileId)
{
var url = Urls.ApiFileStorage(fileId);
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "POST";
_requestHelper.ExecuteCommand(request, true);
}
public override void Cleanup()
{
// Clean up if needed
Urls.Clear();
if (_onNextSubscription != null)
{
_onNextSubscription.Dispose();
}
base.Cleanup();
}
private Task RunHost(CancellationToken token)
{
try
{
var appLifetime = (IApplicationLifetime)_host.Services.GetService(typeof(IApplicationLifetime));
appLifetime.ApplicationStarted.Register(() =>
{
var addresses = _host.ServerFeatures
.Get <Microsoft.AspNetCore.Hosting.Server.Features.IServerAddressesFeature>()
.Addresses;
foreach (string address in addresses)
{
Urls.Add(address.Replace("0.0.0.0", "localhost").Replace("[::]", "localhost"));
PortUtils.TryExtract(address, out bool isHttps, out string protocol, out string host, out int port);
Ports.Add(port);
}
IsStarted = true;
});
#if NETSTANDARD1_3
_logger.Info("WireMock.Net server using netstandard1.3");
#elif NETSTANDARD2_0
_logger.Info("WireMock.Net server using netstandard2.0");
#elif NETSTANDARD2_1
_logger.Info("WireMock.Net server using netstandard2.1");
#elif NETCOREAPP3_1
_logger.Info("WireMock.Net server using .NET Core 3.1");
#elif NET5_0
_logger.Info("WireMock.Net server using .NET 5.0");
#elif NET46
_logger.Info("WireMock.Net server using .NET Framework 4.6.1 or higher");
#endif
#if NETSTANDARD1_3
return(Task.Run(() =>
{
_host.Run(token);
}));
#else
return(_host.RunAsync(token));
#endif
}
catch (Exception e)
{
_runningException = e;
_logger.Error(e.ToString());
IsStarted = false;
return(Task.CompletedTask);
}
}
public ActionResult ActivityRss(DialoguePage page)
{
using (UnitOfWorkManager.NewUnitOfWork())
{
// get an rss lit ready
var rssActivities = new List <RssItem>();
var activities = ServiceFactory.ActivityService.GetAll(20).OrderByDescending(x => x.ActivityMapped.Timestamp);
var activityLink = Urls.GenerateUrl(Urls.UrlType.Activity);
// Now loop through the topics and remove any that user does not have permission for
foreach (var activity in activities)
{
if (activity is BadgeActivity)
{
var badgeActivity = activity as BadgeActivity;
rssActivities.Add(new RssItem
{
Description = badgeActivity.Badge.Description,
Title = string.Concat(badgeActivity.User.UserName, " ", Lang("Activity.UserAwardedBadge"), " ", badgeActivity.Badge.DisplayName, " ", Lang("Activity.Badge")),
PublishedDate = badgeActivity.ActivityMapped.Timestamp,
RssImage = AppHelpers.ReturnBadgeUrl(badgeActivity.Badge.Image).Replace("~", ""),
Link = activityLink
});
}
else if (activity is MemberJoinedActivity)
{
var memberJoinedActivity = activity as MemberJoinedActivity;
rssActivities.Add(new RssItem
{
Description = string.Empty,
Title = Lang("Activity.UserJoined"),
PublishedDate = memberJoinedActivity.ActivityMapped.Timestamp,
RssImage = memberJoinedActivity.User.MemberImage(AppConstants.GravatarPostSize),
Link = activityLink
});
}
else if (activity is ProfileUpdatedActivity)
{
var profileUpdatedActivity = activity as ProfileUpdatedActivity;
rssActivities.Add(new RssItem
{
Description = string.Empty,
Title = Lang("Activity.ProfileUpdated"),
PublishedDate = profileUpdatedActivity.ActivityMapped.Timestamp,
RssImage = profileUpdatedActivity.User.MemberImage(AppConstants.GravatarPostSize),
Link = activityLink
});
}
}
return(new RssResult(rssActivities, Lang("Rss.LatestActivity.Title"), Lang("Rss.LatestActivity.Description")));
}
}
/// <summary>
/// Start a run
/// </summary>
public void Run()
{
if (Receivers.Count == 0)
{
Exception ex = new Exception($"No receiver declared");
Logger.Warn(ex.Message, ex);
return;
}
List <Runner> runners;
switch (StartingRunningLevel.BaseRunningLevel)
{
case BaseRunningLevel.Tenant:
// If Tenant, must be only one url
runners = new List <Runner>()
{
GetTenantRunner(Urls.FirstOrDefault())
};
break;
case BaseRunningLevel.SiteCollection:
runners = new List <Runner>(GetSiteCollectionRunners(Urls));
break;
case BaseRunningLevel.Site:
runners = new List <Runner>(GetSiteRunners(Urls));
break;
case BaseRunningLevel.List:
runners = new List <Runner>(GetListRunners(Urls));
break;
default:
throw new Exception($"Run cannot start at '{StartingRunningLevel.ToString()}' level");
}
Logger.Info("RunningManager initialized");
Logger.Info($"Receivers count: {Receivers.Count}");
Logger.Info($"StartingRunningLevel: {StartingRunningLevel}");
Logger.Info($"Runners count: {runners.Count} for URLs: '{string.Join(", ", Urls)}'");
// OnStart
Logger.Debug("RunningManager OnStart()");
Receivers.ForEach(r => r.OnStart());
// Launch runners
runners.ForEach(r => r.Process());
// OnEnd
Logger.Debug("RunningManager OnEnd()");
Receivers.ForEach(r => r.OnEnd());
Logger.Info("RunningManager finished");
}
public bool IsOnTheLoginScreen()
{
string url = Urls.UrlFor(new LoginRequest(), "GET");
var actual = Driver.Url;
StoryTellerAssert.Fail(!actual.Matches(url), "The actual url was " + actual);
return(true);
}
public async Task <IActionResult> Create([Bind("ID,href,text,status,sortId,catalog")] Urls urls)
{
if (ModelState.IsValid)
{
_context.Add(urls);
await _context.SaveChangesAsync();
return(RedirectToAction(nameof(Index)));
}
return(View(urls));
}
public IEnumerable <Uri> GetRemotes()
{
if ((Config.Remotes == null) || !Config.Remotes.Any())
{
return(Urls.Distinct());
}
lock (Config.Remotes) {
return(Config.Remotes.Values
.SelectMany(x => x).Distinct());
}
}
private async Task SetGoalUpdateMessages(Guid goalId, int propertyUpdated)
{
ViewModel.PageTitle = "Goal Updated";
var goal = await _dssReader.GetGoalDetails(ViewModel.CustomerId.ToString(),
ViewModel.InteractionId.ToString(), ViewModel.ActionPlanId.ToString(), goalId.ToString());
ViewModel.Name = $"{goal.GoalSummary} - {goal.GoalType.GetDisplayName()}";
ViewModel.ObjLink = Urls.GetViewGoalUrl(ViewModel.CompositeSettings.Path, goalId);
ViewModel.ObjLinkText = "view or update this goal";
SetGoalMessagesForProperty(propertyUpdated, goal);
}
public string Convert(ufDateNode node,
ufFormatDescriber ufFormat,
bool multiplesFormats,
string callBack,
Urls urls)
{
m_sCallBack = callBack;
m_sCallBack = m_sCallBack.Replace("(", "").Replace(")", "").Trim();
m_oUrls = urls;
return Convert(node, ufFormat, multiplesFormats);
}
private static async Task HandleAsync(string serverUrl, string accessToken, string username, string password, string timelineKey)
{
using var client = HttpClientFactory.Create(username, password, accessToken);
serverUrl ??= await client.GetCloudServerUrlAsync();
JsonElement home = await client.GetAsJsonAsync(serverUrl, ManicTimeMediaType.V3);
JsonElement timelines = await client.GetAsJsonAsync(Urls.Timelines(home.GetLinkHref("manictime/timelines"), timelineKey), ManicTimeMediaType.V3);
Console.WriteLine(timelines.FormatForDisplay());
}
public void Get()
{
int transactionId = 345345;
string password = "******";
int amount = 34065;
string orderId = "545646474";
int currencyAsInt = 208;
var url = Urls.GetAuthorizeSubscriptionUrl(transactionId, password, amount, orderId, currencyAsInt);
Assert.AreEqual("/webservices/public/subscriptionmanager.asmx/AuthorizeSubscription3?subscriptionID=345345&password=Password&amount=34065&orderID=545646474¤cy=208", url);
}
private void Listen()
{
Urls.ForEach(url => _httpListener.Prefixes.Add(url));
while (_running)
{
var context = _httpListener.GetContext();
Task.Run(() => StartWorker(context));
}
}
public void PostSpecUpdate(string message, string picture)
{
if (token.IsEmpty())
{
return;
}
var path = Urls.WebToSys(picture);
var photoId = File.Exists(path) ? PostPhoto(path) : null;
PostToApi2("wall.post", new { message, owner_id = -SpecGroupId, from_group = 1, attachments = photoId });
}
private void ParseUrls(System.Xml.XmlReader reader)
{
reader.Read();
while (reader.LocalName == "url")
{
PhotoInfoUrl url = new PhotoInfoUrl();
((IFlickrParsable)url).Load(reader);
Urls.Add(url);
}
}
public string Convert(ufDateNode node,
ufFormatDescriber ufFormat,
bool multiplesFormats,
string callBack,
Urls urls)
{
m_sCallBack = callBack;
m_sCallBack = m_sCallBack.Replace("(", "").Replace(")", "").Trim();
m_oUrls = urls;
return(Convert(node, ufFormat, multiplesFormats));
}
public void SetUrl(string site, string url)
{
if (Urls.ContainsKey(site))
{
Urls[site] = url;
}
else
{
Urls.Add(site, url);
}
}
public async Task WhenBodyCalled_ReturnHtml()
{
var viewActionUrl = Urls.GetViewActionUrl("", new Guid());
var changeActionUrl = Urls.GetChangeUrl("", CompositeViewModel.PageId.ChangeActionStatus, new Guid(), "actionId");
var result = await _controller.Body(new Guid()) as ViewResult;
result.Should().NotBeNull();
result.Should().BeOfType <ViewResult>();
result.ViewName.Should().BeNull();
}
public async Task<IActionResult> GetReutersTechRss()
{
var request = new ApplicationRSSToJSON();
var data = await request.GetReutersTechRss();
var model = new Urls()
{
Links = data
};
return PartialView(model);
}
public string Convert(ufDateNode node,
ufFormatDescriber ufFormat,
bool multiplesFormats,
string callBack,
Urls urls,
Errors errors,
bool reporting)
{
m_bReporting = reporting;
m_oErrors = errors;
m_sCallBack = callBack;
m_sCallBack = m_sCallBack.Replace("(", "").Replace(")", "").Trim();
m_oUrls = urls;
return Convert(node, ufFormat, multiplesFormats);
}
/// <summary>
/// Takes multiple formatting objects - hAtom
/// </summary>
/// <param name="node"></param>
/// <param name="formatArray"></param>
/// <param name="multiplesFormats"></param>
/// <param name="callBack"></param>
/// <param name="urls"></param>
/// <param name="errors"></param>
/// <param name="reporting"></param>
/// <returns></returns>
public string Convert(ufDateNode node,
ArrayList formatArray,
bool multiplesFormats,
string callBack,
Urls urls,
Errors errors,
bool reporting)
{
m_sCallBack = callBack;
m_sCallBack = m_sCallBack.Replace("(", "").Replace(")", "").Trim();
m_oUrls = urls;
m_oErrors = errors;
m_bReporting = reporting;
foreach (ufFormatDescriber ufFormat in formatArray)
{
foreach (ufDateNode childNode in node.Nodes)
{
foreach (ufDateNode grandChildNode in childNode.Nodes)
{
if (grandChildNode.Name == ufFormat.BaseElement.Name)
{
ufDateNode xChild = xTree.Nodes.Append(grandChildNode.Name, grandChildNode.Value, grandChildNode.SourceUrl, grandChildNode.RepresentativeNode);
if (grandChildNode.Nodes.Count > 0)
AddChildNodes(xChild, grandChildNode, ufFormat.BaseElement);
}
}
}
}
string output = "// ufXtract \n";
if (m_sCallBack != string.Empty)
output += m_sCallBack + "( ";
output += "{";
foreach (ufDateNode childNode in xTree.Nodes)
output += BuildDataString(childNode, true, false);
if (xTree.Nodes.Count > 0)
output = output.Substring(0, output.Length - 2);
output += AddErrors();
output += AddReporting(node);
// End whole block
output += "}";
if (m_sCallBack != string.Empty)
output += " )";
//return output.Replace(",", ",\n").Replace("}", "}\n").Replace("{", "{\n").Replace("]", "]\n").Replace("[", "[\n"); ;
return output;
}
/// <summary>
/// Provides string version of data object
/// </summary>
/// <param name="node">The data object</param>
/// <param name="urls">The Urls object created by spider</param>
/// <returns></returns>
public string Convert(ufDateNode node, bool multiples, Urls urls, Errors errors, bool reporting)
{
m_bReporting = reporting;
m_oErrors = errors;
m_oUrls = urls;
return Convert(node, multiples);
}
/// <summary>
/// Provides string version of data object
/// </summary>
/// <param name="node">The data object</param>
/// <param name="urls">The Urls object created by spider</param>
/// <returns></returns>
public string Convert(ufDateNode node, bool multiples, Urls urls)
{
m_oUrls = urls;
return Convert(node, multiples);
}
public string Convert(ufDateNode node,
ufFormatDescriber ufFormat,
bool multiplesFormats,
Urls urls)
{
m_oUrls = urls;
return Convert(node, ufFormat, multiplesFormats);
}
public void Convert(ufDateNode node, Stream stream, bool multiples, Urls urls)
{
m_oUrls = urls;
Convert(node, stream, multiples);
}
private void LoadPage(string url, ufFormatDescriber format)
{
if (url != string.Empty)
{
Url urlReport = new Url();
urlReport.Address = url;
//m_oFormatDescriber = format;
UrlModule urlModule = new UrlModule();
Uri uri = new Uri(url);
urlModule.DocumentContentType = UrlModule.ContentType.Html;
urlModule.DocumentRequestType = UrlModule.RequestType.Get;
urlModule.Load(uri);
urlReport.Status = urlModule.StatusCode;
DateTime started = DateTime.Now;
m_AllParsedUrls.Add(urlReport);
if (urlModule.StatusCode == 200 && urlModule.Html != null)
ParseUf(urlModule.Html, uri.ToString(), format);
//----------------------
Urls newUrls = new Urls();
m_oMe.Nodes.Add(m_oData);
foreach(ufDateNode node in m_oData.Nodes)
{
if (node.Nodes["link"] != null)
{
string link = node.Nodes["link"].Value;
if (ShouldParseUrl(link))
newUrls.Add(new Url(link));
}
}
foreach (Url newUrl in newUrls)
{
LoadPage(newUrl.Address, m_oMeDescriber);
}
//----------------------
DateTime ended = DateTime.Now;
urlReport.LoadTime = ended.Subtract(started);
}
}
private static void UsingSyndicationFeed(List<NewsComponents> listNews, Dictionary<string, List<NewsComponents>> newsdictionary, Urls item, XmlReader reader)
{
SyndicationFeed feed = SyndicationFeed.Load(reader);
var fed = (from f in feed.Items
orderby f.LastUpdatedTime descending
select f).Take(10);
foreach (var newsItem in fed)
{
NewsComponents newsComponents = new NewsComponents();
newsComponents.NewsID = Guid.NewGuid();
newsComponents.NewsHeadline = newsItem.Title.Text;
newsComponents.Summary = string.IsNullOrWhiteSpace(newsItem.Summary.Text) ? "" : newsItem.Summary.Text;
newsComponents.Source = item.key;
newsComponents.Section = "";
//newsComponents. = "";
newsComponents.Category = item.Category;
newsComponents.NewsAdded = newsItem.PublishDate.DateTime;
newsComponents.TimeStamp = DateTime.Now;
try
{
foreach (var link in newsItem.Links)
{
if (link.MediaType == null)
{
var stream = GetStreamInternet(link);
StreamReader strReader = new StreamReader(stream);
string newsContent = strReader.ReadToEnd();
newsComponents.NewsItem = StripHtmlStuff(newsContent, newsComponents.NewsHeadline);
}
else if (link.MediaType.Contains("image"))
{
var stream = GetStreamInternet(link);
Images image = new Images();
image.photostreams = stream;
image.fileName = Guid.NewGuid() + ".jpg";
image.Url = URL + image.fileName;
newsComponents.Images.Add(image);
newsComponents.NewsPhotoUrl = image.Url + ";" + newsComponents.NewsPhotoUrl;
}
}
SaveImageToSC(newsComponents.Images);
}
catch (Exception e)
{
}
finally
{
listNews.Add(newsComponents);
//}
}
}
newsdictionary.Add(item.key, listNews);
}
//public static void LoadNewsintoTables(List<Urls> newsurls, string domainName)
//{
// List<NewsComponents> listNews = new List<NewsComponents>();
// Dictionary<string, List<NewsComponents>> newsdictionary = new Dictionary<string, List<NewsComponents>>();
// //Zambian watchdog
// foreach (var item in newsurls)
// {
// try
// {
// LoadXml(listNews, newsdictionary, item);
// }
// catch (XmlException e)
// {
// var xmlDoc = XDocument.Load(reader);
// var newsitems = (from channel in xmlDoc.Root.Elements("rdf")
// from itemx in channel.Elements("item")
// select new NewsComponents
// {
// Source = item.key,
// Section = "",
// NewsItem = LoadNewsContent(itemx.Element("Link").Value),
// NewsHeadline = itemx.Element("title").Value,
// NewsAdded = Convert.ToDateTime(itemx.Element("dc:date").Value),
// NewsPhotoUrl = "",
// Summary = itemx.Element("description").Value,
// Category = item.Category
// }).Take(10);
// }
// catch (WebException ex)
// {
// Thread.Sleep(1000);
// LoadXml(listNews, newsdictionary, item);
// }
// }
// FindOccurence(newsdictionary, domainName);
//}
private static void LoadXml(List<NewsComponents> listNews, Dictionary<string, List<NewsComponents>> newsdictionary, Urls item)
{
reader = XmlReader.Create(item.url);
UsingSyndicationFeed(listNews, newsdictionary, item, reader);
}
