private async Task <IDocument> LoadAsync(string address) { IConfiguration configuration; if (GenerateRandomUserAgent) { Request request = new Request(); request.Headers["User-Agent"] = FfeWeb.GetValidUserAgent(); configuration = Configuration.Default.With(request).WithDefaultLoader(); } else { configuration = Configuration.Default.WithDefaultLoader(); } IBrowsingContext browsingContext = BrowsingContext.New(configuration); IDocument document = await browsingContext.OpenAsync(address); if (string.IsNullOrEmpty(document.Body.InnerHtml)) { throw new WebException($"The URL {address} returned no HTML content."); } return(document); }
private static string LoadJson(Uri uri) { string json = null; if (SsqSetting.Default.AutoUpdate) { if (uri.Scheme.Equals("http") || uri.Scheme.Equals("https")) { json = FfeWeb.GetHttpResponseContent(uri); } else if (uri.Scheme.Equals("file")) { json = File.ReadAllText(uri.LocalPath); } else { throw new Ssq.SsqException("Not supported URI scheme. Supported schemes are http(s) and file."); } } else { log.Debug("Auto Update of SSQ JSON is disabled. Use the embedded local one."); json = SsqResource.SsqUdf; } log.Debug("Loaded SSQ JSON text.", json); log.Verbose("SSQ JSON text: {@SsqJsonText}", json); return(json); }
public void QAVMAWebTest(QAVTest test) { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_MONTHLY_ADJUSTED&symbol={test.Symbol}&apikey={test.ApiKey}"); object actual = Avq.QAVMA(test.Symbol, info: test.Info, tradingMonth: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch); QAVAssert(test, actual); }
public static object QBIC([ExcelArgument(Name = "IBAN", Description = "the IBAN for which the BIC should be returned.")] string iban) { const string FUNCTON_NAME = "QBIC"; // IBAN is mandatory. if (string.IsNullOrEmpty(iban)) { return(OpenibanExcelError(FUNCTON_NAME, "NULL!")); } // Check on valid IBAN. // https://stackoverflow.com/questions/44656264/iban-regex-design if (!System.Text.RegularExpressions.Regex.IsMatch(iban, @"^([A-Z]{2}[ \-]?[0-9]{2})(?=(?:[ \-]?[A-Z0-9]){9,30}$)((?:[ \-]?[A-Z0-9]{3,5}){2,7})([ \-]?[A-Z0-9]{1,3})?$")) { log.Error("Invalid {@IBAN}", iban); return(OpenibanExcelError(FUNCTON_NAME, "INVALID_IBAN")); } JObject json = null; try { //var ffeAttribute = (FfeFunctionAttribute)Attribute.GetCustomAttribute(typeof(Openiban).GetMethod(System.Reflection.MethodBase.GetCurrentMethod().Name), typeof(FfeFunctionAttribute)); log.Debug("Querying the BIC for IBAN {@IBAN} from {@Provider}", iban, "openiban.com"); var endpoint = $"{URL_OPEN_IBAN}/validate/{iban}?getBIC=true&validateBankCode=true"; log.Debug("Openiban endpoint URL: {@OpenibanEndpointUrl}.", endpoint); // TODO: Newtonsoft.Json.JsonConvert.DeserializeObject<List<MyType>(jsonData); //faster with typed object if (UrlContenResult == null) { json = JObject.Parse(FfeWeb.GetHttpResponseContent(endpoint)); } else // else running tests. { json = JObject.Parse(UrlContenResult); } if ((bool)json["valid"]) { return((string)json["bankData"]["bic"]); } else { log.Error((string)json["messages"][0]); return(OpenibanExcelError(FUNCTON_NAME, "INVALID_IBAN")); } } catch (Exception ex) { log.Error(ex, FUNCTON_NAME); if (json != null) { log.Error("Openiban JSON response: {@OpenibanResponse}", json.ToString()); } return(ExcelError.ExcelErrorGettingData); } }
public static object QIBAN([ExcelArgument(Name = "Country Code", Description = "the country code (ISO 3166-1 alpha-2 code) for which the IBAN should be calculated.\nUse =QCOUNTRIES() to get an array of supported countries.")] string countryCode, [ExcelArgument(Name = "Bank Code", Description = "the bank code for which the IBAN should be calculated.")] string bankCode, [ExcelArgument(Name = "Account Number", Description = "the account number for which the IBAN should be calculated.")] string accountNumber) { const string FUNCTON_NAME = "QIBAN"; // All arguments are mandatory (Prevalidation). if (string.IsNullOrEmpty(countryCode) //|| !System.Text.RegularExpressions.Regex.IsMatch(countryCode, @"^[a-zA-Z]{2}$") || string.IsNullOrEmpty(bankCode) || string.IsNullOrEmpty(accountNumber)) { return(OpenibanExcelError(FUNCTON_NAME, "NULL!")); } JObject json = null; try { log.Debug("Querying the IBAN from {@Provider}", "openiban.com"); var endpoint = $"{URL_OPEN_IBAN}/calculate/{countryCode}/{bankCode}/{accountNumber}"; log.Debug("Openiban endpoint URL: {@OpenibanEndpointUrl}.", endpoint); if (UrlContenResult == null) { json = JObject.Parse(FfeWeb.GetHttpResponseContent(endpoint)); } else // else running tests. { json = JObject.Parse(UrlContenResult); } if ((bool)json["valid"]) { return((string)json["iban"]); } else { log.Error((string)json["message"]); return(OpenibanExcelError(FUNCTON_NAME, "INVALID")); } } catch (Exception ex) { log.Error(ex, FUNCTON_NAME); if (json != null) { log.Error("Openiban JSON response: {@OpenibanResponse}", json.ToString()); } return(ExcelError.ExcelErrorGettingData); } }
public void QAVIDWebTest(QAVTest test) { if (test.OutputSize.Equals("full")) { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={test.Symbol}&interval={test.Interval}&outputsize={test.OutputSize}&apikey={test.ApiKey}"); } else { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={test.Symbol}&interval={test.Interval}&apikey={test.ApiKey}"); } object actual = Avq.QAVID(test.Symbol, info: test.Info, dataPointIndex: test.DatePart, interval: test.Interval, outputSize: test.OutputSize); QAVAssert(test, actual); }
public void QAVDAWebTest(QAVTest test) { if (test.OutputSize.Equals("full")) { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={test.Symbol}&outputsize={test.OutputSize}&apikey={test.ApiKey}"); } else { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={test.Symbol}&apikey={test.ApiKey}"); } object actual = Avq.QAVDA(test.Symbol, info: test.Info, tradingDay: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch, outputSize: test.OutputSize); QAVAssert(test, actual); }
public void QAVDWebNegativeTest(QAVTest test) { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol={test.Symbol}&apikey={test.ApiKey}"); if (test.OutputSize.Equals("n/a")) { Assert.Throws <ArgumentException>(() => Avq.QAVD(test.Symbol, info: test.Info, tradingDay: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch, outputSize: test.OutputSize)); } else { object actual = Avq.QAVD(test.Symbol, info: test.Info, tradingDay: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch, outputSize: test.OutputSize); Assert.Equal(test.Expected, actual); } }
public dynamic Load(string url) { /*using (Stream stream = FfeWeb.GetHttpResponseContentAsStreamReader(url, GenerateRandomUserAgent)) * using (StreamReader streamReader = new StreamReader(stream)) * using (JsonReader reader = new JsonTextReader(streamReader)) * { * //return JObject.Load(reader); * return JArray.Load(reader); * }*/ string json = FfeWeb.GetHttpResponseContent(url, GenerateRandomUserAgent); //return JObject.Parse(json); return(JArray.Parse(json)); }
public static object[,] QCOUNTRIES() { const string FUNCTON_NAME = "QCOUNTRIES"; string json = null; try { log.Debug("Querying the supported countries from {@Provider}", "openiban.com"); var endpoint = $"{URL_OPEN_IBAN}/countries"; log.Debug("Openiban endpoint URL: {@OpenibanEndpointUrl}.", endpoint); if (UrlContenResult == null) { json = FfeWeb.GetHttpResponseContent(endpoint); } else // else running tests. { json = UrlContenResult; } Dictionary <string, string> countries = JsonConvert.DeserializeObject <Dictionary <string, string> >(json); int numberOfCountries = countries.Count; object[,] countryArray = new object[numberOfCountries, 2]; for (int i = 0; i < numberOfCountries; i++) { KeyValuePair <string, string> country = countries.ElementAt(i); countryArray[i, 0] = country.Key; countryArray[i, 1] = country.Value; } return(countryArray); } catch (Exception ex) { log.Error(ex, FUNCTON_NAME); if (json != null) { log.Error("Openiban JSON response: {@OpenibanResponse}", json.ToString()); } return(new object[0, 0]); } }
public void QAVTSWebTest(QAVTest test) { string api; string interval = test.Interval.ToLower(); if (interval.Equals("daily") || interval.Equals("weekly") || interval.Equals("monthly")) { api = $"TIME_SERIES_{interval.ToUpper()}" + (test.Adjusted ? "_ADJUSTED" : ""); // Remove not valid API parameters (for demo API key). if (!interval.Equals("daily") || test.OutputSize.Equals("compact")) { test.OutputSize = null; } test.Interval = null; } else { api = "TIME_SERIES_INTRADAY"; // Remove not valid TIME_SERIES_INTRADAY API parameter (for demo API key). test.OutputSize = null; } Avq.AvStockTimeSeriesOutputSize?outputSize = null; if (test.OutputSize != null) { outputSize = (Avq.AvStockTimeSeriesOutputSize)Enum.Parse(typeof(Avq.AvStockTimeSeriesOutputSize), test.OutputSize); } string avUri = Avq.AvUrlBuilder(api, apiKey: test.ApiKey, symbol: test.Symbol, outputSize: outputSize, interval: test.Interval); Avq.UrlContenResult = FfeWeb.GetHttpResponseContent(avUri); object actual = Avq.QAVTS(test.Symbol, info: test.Info, interval: test.Interval, tradingDay: test.DatePart, tradingDate: test.QuoteDate, adjusted: test.Adjusted, outputSize: test.OutputSize, bestMatch: test.BestMatch); QAVAssert(test, actual); }
public void QAVQWebTest(QAVTest test) { Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol={test.Symbol}&apikey={test.ApiKey}"); object actual = Avq.QAVQ(test.Symbol, info: test.Info); // If a date specific argument was given, then stock info may not available when trading date is on weekend or a public holiday. DateTime tradingDate = DateTime.Today.AddDays(test.DatePart); if (tradingDate.DayOfWeek == DayOfWeek.Saturday || tradingDate.DayOfWeek == DayOfWeek.Sunday || DateSystem.IsPublicHoliday(tradingDate, CountryCode.US)) { // No quotes are available on the weekend or on public holidays. Assert.True(ExcelError.ExcelErrorNA.Equals(actual) || actual is DateTime || actual is string || actual is decimal); } else { switch (test.Info) { case "latest trading day": Assert.IsType <DateTime>(actual); break; case "change percent": Assert.IsType <string>(actual); break; default: Assert.IsType <decimal>(actual); break; } } }
public static (string value, string rawSource) GetValueFromWeb(string url, string xPath = null, string cssSelector = null, string regExPattern = null, string regExGroup = null, int regExMatchIndex = 0, string jsonPath = null, Parser parser = Parser.Auto) { if (xPath == null && cssSelector == null && regExPattern == null && jsonPath == null) { throw new ArgumentException("No selection criteria were provided. Provide a least one of the following selection criteria: xPath, cssSelector, regExPattern or jsonPath."); } string value = null; string rawSource = null; Uri uri = new Uri(url); if (parser != Parser.Newtonsoft) { IFfeWebParser ffeWebParser = null; switch (parser) { case Parser.Auto: ffeWebParser = AutoWebParserSelection(uri, xPath, cssSelector, regExPattern); break; case Parser.HAP: ffeWebParser = new FfeWebHap(uri); break; case Parser.AngleSharp: ffeWebParser = new FfeWebAngleSharp(uri); break; case Parser.HttpClient: ffeWebParser = new FfeWebHttpClient(uri); break; case Parser.WebClient: ffeWebParser = new FfeWebClient(uri); break; default: ffeWebParser = new FfeWebHap(uri); break; } rawSource = ffeWebParser.GetHtml(); if (log.IsEnabled(Serilog.Events.LogEventLevel.Debug)) { rawSource.WriteToFile($"PageSource_{uri.Host}.html", parser.ToString()); } // Input for RegEx (if set). string input = null; if (!String.IsNullOrEmpty(xPath)) { value = ffeWebParser.SelectByXPath(xPath); input = value; } else if (!String.IsNullOrEmpty(cssSelector)) { value = ffeWebParser.SelectByCssSelector(cssSelector); input = value; } else // Select by RegEx (HTML source code = RegEx input). { //HACK: AngelSharp does not provide full HTML source code. if (parser == Parser.AngleSharp) { log.Warning("Regular Expression with AngleSharp WebParser does not work well. HttpClient is used instead."); rawSource = FfeWeb.GetHttpResponseContent(uri); } input = rawSource; } if (!String.IsNullOrEmpty(regExPattern)) { value = FfeRegEx.RegExByIndexAndGroup(input, regExPattern, regExMatchIndex, regExGroup); if (String.IsNullOrEmpty(value)) { if (log.IsEnabled(Serilog.Events.LogEventLevel.Debug)) { input.WriteToFile("RegExInput.html", "RegEx"); } throw new RegExException() { Input = input, Pattern = regExPattern }; } } } else { IFfeJsonParser ffeJsonParser = new FfeJsonNewtonsoft(uri); value = ffeJsonParser.SelectByJsonPath(jsonPath); rawSource = ffeJsonParser.GetJson(); if (log.IsEnabled(Serilog.Events.LogEventLevel.Debug)) { rawSource.WriteToFile($"JsonSource_{uri.Host}.json", parser.ToString()); } } return(value, rawSource); }