public async Task CanGetStringFromExternalSite()
{
var requestor = new WebRequestor(this.httpClient);
var res = await requestor.GetString(new Uri("http://yandex.ru")).ConfigureAwait(false);
Assert.IsNotEmpty(res);
}
static void VisitPage(string url)
{
Console.WriteLine("Submitting request...");
var response = WebRequestor.Get(url);
Console.WriteLine("Response received.");
}
public async Task CanGetStringFromFinamResource()
{
var requestor = new WebRequestor(this.httpClient);
var res = await requestor.GetString(new Uri("https://www.finam.ru/cache/icharts/icharts.js")).ConfigureAwait(false);
#if DEBUG
Console.WriteLine(res);
#endif
Assert.IsNotEmpty(res);
}
private IVehicleTypeDocumentRequestor createVehicleTypeDRObj(string vehicleTypeUrl, string content)
{
Mock <VehicleTypeDocumentRequestor> mockDRVehicleType = new Mock <VehicleTypeDocumentRequestor>(vehicleTypeUrl);
WebRequestor webRequestor = new WebRequestor();
HtmlDocument vehicleTypePage = webRequestor.CreateHtmlDocument(content);
mockDRVehicleType.Setup(dr => dr.GetHtml()).Returns(vehicleTypePage);
IVehicleTypeDocumentRequestor documentRequester = mockDRVehicleType.Object;
return(documentRequester);
}
/*
* [TestMethod]
* public void CreatePlan_NoTypeGiven_VehicleTypesLengthEquals4()
* {
* try
* {
* //Arrange
* IInitialPageDocumentRequestor drInitialPage = createInitialPageDRObj();
* string vehicleType1Url = "http://catalog.mann-filter.com/EU/tur/vehicle/MANN-FILTER%20Katalog%20Europa/Ara%C3%A7lar/Otomobiller%20%2B%20Ticari%20Ara%C3%A7lar";
* string vehicleType2Url = "http://catalog.mann-filter.com/EU/tur/vehicle/MANN-FILTER%20Katalog%20Europa/Ara%C3%A7lar/Kamyon%20%2B%20Otob%C3%BCs";
* string vehicleType3Url = "http://catalog.mann-filter.com/EU/tur/vehicle/MANN-FILTER%20Katalog%20Europa/Ara%C3%A7lar/Off%20Highway%20uygulamalar%C4%B1";
* string vehicleType4Url = "http://catalog.mann-filter.com/EU/tur/vehicle/MANN-FILTER%20Katalog%20Europa/Ara%C3%A7lar/Motosiklet";
* IVehicleTypeDocumentRequestor drVehicleType1 = createVehicleTypeDRObj(vehicleType1Url, Resources.MANNFILTER_Araclar_Otomobiller_Ticari_Araclar);
* IVehicleTypeDocumentRequestor drVehicleType2 = createVehicleTypeDRObj(vehicleType2Url, Resources.MANNFILTER_Araclar_Kamyon_Otobus);
* IVehicleTypeDocumentRequestor drVehicleType3 = createVehicleTypeDRObj(vehicleType3Url, Resources.MANNFILTER_Araclar_Off_Highway_uygulamalari);
* IVehicleTypeDocumentRequestor drVehicleType4 = createVehicleTypeDRObj(vehicleType4Url, Resources.MANNFILTER_Araclar_Motosiklet);
*
* //Act
* var executionPlan = new MannDBExecutionPlan(drInitialPage, new[]{ "Kategori", "Üretici", "Model Adı", "Tipi", "Motor Kodu", "kW", "PS", "Yıl", "Hava Filtresi", "Ölçü Hava", "Yağ Filtresi", "Ölçü Yağ", "Benzin Filtresi", "Ölçü Benzin", "İç Mekan Filtresi", "Ölçü İç Mekan", "Diğer Filtreler", "Ölçü Diğer" });
* executionPlan.CreatePlan();
* executionPlan.ExecutePlan();
*
* //Assert
* Assert.IsTrue(executionPlan.VehicleTypes.Length == 4);
* Assert.IsTrue(executionPlan.VehicleTypeProducers.Count == 4);
* Assert.IsTrue(executionPlan.VehicleTypeProducers["Otomobiller + Ticari Araçlar"].Length == 123);
* Assert.IsTrue(executionPlan.VehicleTypeProducers["Kamyon + Otobüs"].Length == 129);
* Assert.IsTrue(executionPlan.VehicleTypeProducers["Off Highway uygulamaları"].Length == 404);
* Assert.IsTrue(executionPlan.VehicleTypeProducers["Motosiklet"].Length == 48);
* }
* catch (Exception exc)
* {
* Assert.Fail(exc.Message);
* }
* }
*/
private IInitialPageDocumentRequestor createInitialPageDRObj()
{
string initialPageUrl = "http://catalog.mann-filter.com/EU/tur/vehicle/MANN-FILTER%20Katalog%20Europa/Ara%C3%A7lar";
Mock <InitialPageDocumentRequestor> mockDocumentRequester = new Mock <InitialPageDocumentRequestor>(initialPageUrl);
WebRequestor webRequestor = new WebRequestor();
HtmlDocument initialPage = webRequestor.CreateHtmlDocument(Resources.mannfilter_initial_page);
mockDocumentRequester.Setup(dr => dr.GetHtml()).Returns(initialPage);
IInitialPageDocumentRequestor documentRequester = mockDocumentRequester.Object;
return(documentRequester);
}
public async Task <string> LoadData(Symbol symbol, Period period, DateTime fromDate, DateTime toDate)
{
var requestUrl = new QueryBuilder()
.WithDateRange(fromDate, toDate, period)
.WithSymbol(symbol)
.GetUrl(this.config.ExportDataUrl);
using (var client = new HttpClient())
{
var requestor = new WebRequestor(client);
return(await requestor.GetString(requestUrl).ConfigureAwait(false));
}
}
public void Update()
{
using (var client = new HttpClient())
{
var requestor = new WebRequestor(client);
var updater = new MetadataUpdater(requestor, config.MarketsFeedUrl, config.SymbolsFeedUrl);
var markets = updater.LoadMarkets().Result;
var symbols = updater.LoadSymbols().Result;
this.persister.SaveMarkets(markets);
this.persister.SaveSymbols(symbols);
}
}
/// <summary>
/// Tell the reporter to use magic AWS REST address to get the AWS instance Id
/// </summary>
/// <returns>this</returns>
public SignalFxReporterBuilder WithAWSInstanceIdDimension(IWebRequestor awsRequestor = null)
{
if (awsRequestor == null)
{
awsRequestor = new WebRequestor("http://169.254.169.254/latest/meta-data/instance-id")
.WithTimeout(1000 * 60)
.WithMethod("GET");
}
using (var resp = awsRequestor.Send())
{
string source = new StreamReader(resp).ReadToEnd();
defaultDimensions[INSTANCE_ID_DIMENSION] = source;
return(this);
}
}
static void Main(string[] args)
{
WebRequestor.Init();
ProgramInit();
NewItemNotifierService newItemService = new NewItemNotifierService();
newItemService.CheckAllNewItemsFromGumtree();
//var items = GumtreeHtmlReader.GetAdvertItems("https://www.gumtree.com.au/s-miscellaneous-goods/perth/cardboard+boxes/k0c18319l3008303");
//InsertAdvertItems(items);
//DoDatabaseStuff();
//VisitPage("https://www.gumtree.com.au/s-miscellaneous-goods/perth/cardboard+boxes/k0c18319l3008303");
//VisitPage("https://intranet.health.wa.gov.au");
}
public async Task CanDownloadDataFromFinam()
{
// TODO: Refactor this: Load from Metadata Repo
var sber = new Symbol()
{
MarketId = 8,
Id = 81075,
Code = "SBER"
};
var request = new QueryBuilder()
.WithDateRange(new DateTime(2016, 01, 01), new DateTime(2016, 08, 31), Period.H1)
.WithSymbol(sber)
.GetUrl(CONST_FinamExportHost);
using (var client = new HttpClient())
{
var requestor = new WebRequestor(client);
var data = await requestor.GetString(request).ConfigureAwait(false);
Console.WriteLine(data);
}
}
private SignalFxBackendConfiguration parseConfig(XElement configElement)
{
var apiToken = getRequiredStringConfig(configElement, "apiToken");
var defaultDimensionsNode = configElement.Element("defaultDimensions");
IDictionary <string, string> defaultDimensions = new Dictionary <string, string>();
if (defaultDimensionsNode != null)
{
foreach (var defaultDimensionNode in defaultDimensionsNode.Elements("defaultDimension"))
{
string name = getRequiredStringConfig(defaultDimensionNode, "name");
string value = getRequiredStringConfig(defaultDimensionNode, "value");
if (!String.IsNullOrEmpty(name) && !String.IsNullOrEmpty(value))
{
defaultDimensions[name] = value;
}
}
}
var awsIntegrationNode = configElement.Attribute("awsIntegration");
if (awsIntegrationNode != null && configElement.ToBoolean("awsIntegration"))
{
var awsRequestor = new WebRequestor("http://169.254.169.254/latest/meta-data/instance-id")
.WithTimeout(1000 * 60)
.WithMethod("GET");
using (var resp = awsRequestor.Send())
{
defaultDimensions[INSTANCE_ID_DIMENSION] = new StreamReader(resp).ReadToEnd();
}
}
var baseURI = getOptionalStringConfig(configElement, "baseURI", DEFAULT_URI);
var sourceType = getRequiredStringConfig(configElement, "sourceType");
string source;
switch (sourceType)
{
case "netbios":
source = System.Environment.MachineName;
break;
case "dns":
source = System.Net.Dns.GetHostName();
break;
case "fqdn":
string domainName = System.Net.NetworkInformation.IPGlobalProperties.GetIPGlobalProperties().DomainName;
string hostName = System.Net.Dns.GetHostName();
if (!hostName.EndsWith(domainName)) // if hostname does not already include domain name
{
hostName += "." + domainName; // add the domain name part
}
source = hostName;
break;
case "custom":
source = getRequiredStringConfig(configElement, "sourceValue", " when \"sourceType\" is \"custom\"");
break;
default:
throw new Exception("sourceType attribute must be one of netbios, dns, fqdn, or custom(with source attribute set) on " + configElement);
}
var maxBatchSize = getOptionalIntConfig(configElement, "maxBatchSize", MAX_DATAPOINTS_PER_MESSAGE);
var maxTimeBetweenBatches = getOptionalTimeConfig(configElement, "maxWaitBetweenBatches", Utility.ConvertToTimespan("5s"));
var retryDelay = getOptionalTimeConfig(configElement, "retryDelay", Utility.ConvertToTimespan("1s"));
var postTimeout = getOptionalTimeConfig(configElement, "postTimeout", Utility.ConvertToTimespan("1s"));
return(new SignalFxBackendConfiguration(apiToken, defaultDimensions, baseURI, maxBatchSize, source,
maxTimeBetweenBatches, retryDelay, postTimeout));
}
public MetadataUpdater(WebRequestor requestor, string marketsFeedUrl, string symbolsFeedUrl)
{
this.requestor = requestor;
this.marketsFeedUrl = new Uri(marketsFeedUrl);
this.symbolsFeedUrl = new Uri(symbolsFeedUrl);
}
public VehicleTypeDocumentRequestor(WebRequestor webRequestor, string url) : base(webRequestor, url)
{
}
public ModelDocumentRequestor(WebRequestor webRequestor, string url) : base(webRequestor, url)
{
}
static void Main(string[] args)
{
var builder = new ConfigurationBuilder()
.AddJsonFile($"appsettings.json", true, true);
var configuration = builder.Build();
var webClient = new WebClient();
var fileProvider = new FileSystemProvider();
var telegramBotSettings = configuration.GetSection("TelegramBotSettings").Get <TelegramBotSettings>();
var webRequestor = new WebRequestor(webClient);
var chatUpdateReader = new TelegramUpdatesReader(telegramBotSettings, webRequestor);
var publishBotReaction = new TelegramChatMessagePublisher(telegramBotSettings, webRequestor);
var chatBotState = new ChatBotState(fileProvider);
chatBotState.LoadState();
var pollingWatcher = new TelegramChatUpdatesPollingWatcher(chatUpdateReader, chatBotState);
var cancellationTokenSource = new CancellationTokenSource();
var hellow = new HellowAnswer(publishBotReaction);
var Say = new SayUserInformation(publishBotReaction);
var weather = new WeatherAnswer(publishBotReaction);
var vedomosti = new Vedomosti(publishBotReaction);
var dw = new EconomicsDw(publishBotReaction);
var cbr = new Cbr(publishBotReaction);
var news = new NewsRu(publishBotReaction);
var prime = new PrimeRu(publishBotReaction);
var sport = new Sport(publishBotReaction);
var utkin = new Utkin(publishBotReaction);
var nike = new Nike(publishBotReaction);
var legislat = new Legislation(publishBotReaction);
var reebok = new Reebok(publishBotReaction);
var newbalance = new Newbalance(publishBotReaction);
var answers = new List <IBotAnswer>()
{
hellow, Say, weather, vedomosti, dw, cbr, news, prime, sport, utkin, nike, legislat, reebok, newbalance
};
var asker = new Asker(answers);
pollingWatcher.OnMessageArrived += (msg) =>
{
Console.WriteLine($"{msg.text} from {msg.from.username} with id { msg.chat.id} {msg.from.id} ");
if (msg.text == "/start")
{
publishBotReaction.SendMessage("Приветствую вас! Я готов к работе. Команды бота:\n1) Скажи погоду: [место].\n2)Скажи информацию про состояние атмосферы в городе: [название города].\n3)Сколько времени в городе: [название города].\nСобытия в мире экономики по версии:\n4)Газета ведомости -\n/EconomicsVedomosti.\n5)Cайт dw.com\n/EconomicsDw\n6)Сайт ЦБ РФ-\n/EconomicsCbr.\n7)Сайт news.ru-\n/EconomicsNewsRu.\n8)Сайт 1prime.ru-\n/EconomicsPrimeRu.\nНОВОСТИ СПОРТА:\n9)Cайт sports.ru-\n/SportsNews\n10)Блог Уткина-\n/UtkinSportsNews\nНОВЫЕ ЗАКОНЫ:\n11)Сайт с обновлениями законодательства РФ-\n/LegislationChanges.\nНОВОСТИ БРЕНДОВ ОДЕЖДЫ:\n12)Nike-\n/NikenewsFeed\n13)Reebok-\n/ReebokNewsGlobal\n14)NewBalance-\n/NewBalanceNewsMarket.", msg.chat.id);
}
TimeAnswer response = new TimeAnswer(publishBotReaction, msg);
response.CanWork(msg.text);
if (response.CanWork(msg.text) == true)
{
response.Answer();
}
asker.Ask(msg.text, msg);
};
pollingWatcher.StartWatch(cancellationTokenSource.Token);
var stop = Console.ReadLine();
if (stop == "stop bot")
{
Console.WriteLine("Работа Бота остановлена. Сохранение.");
cancellationTokenSource.Cancel();
chatBotState.SaveState();
}
}
public InitialPageDocumentRequestor(WebRequestor webRequestor, string url, VehicleTypes?vehicleType = null) : base(webRequestor, url)
{
this.vehicleType = vehicleType;
}
public InitialPageDocumentRequestor(WebRequestor webRequestor, string url) : base(webRequestor, url)
{
}
public FilterPageDocumentRequestor(WebRequestor webRequestor, string url) : base(webRequestor, url)
{
}
public void OneTimeSetup()
{
this.requestor = new WebRequestor(new HttpClient());
}
