private async Task <IDocument> LoadAsync(string address)
{
IConfiguration configuration;
if (GenerateRandomUserAgent)
{
Request request = new Request();
request.Headers["User-Agent"] = FfeWeb.GetValidUserAgent();
configuration = Configuration.Default.With(request).WithDefaultLoader();
}
else
{
configuration = Configuration.Default.WithDefaultLoader();
}
IBrowsingContext browsingContext = BrowsingContext.New(configuration);
IDocument document = await browsingContext.OpenAsync(address);
if (string.IsNullOrEmpty(document.Body.InnerHtml))
{
throw new WebException($"The URL {address} returned no HTML content.");
}
return(document);
}
private static string LoadJson(Uri uri)
{
string json = null;
if (SsqSetting.Default.AutoUpdate)
{
if (uri.Scheme.Equals("http") ||
uri.Scheme.Equals("https"))
{
json = FfeWeb.GetHttpResponseContent(uri);
}
else if (uri.Scheme.Equals("file"))
{
json = File.ReadAllText(uri.LocalPath);
}
else
{
throw new Ssq.SsqException("Not supported URI scheme. Supported schemes are http(s) and file.");
}
}
else
{
log.Debug("Auto Update of SSQ JSON is disabled. Use the embedded local one.");
json = SsqResource.SsqUdf;
}
log.Debug("Loaded SSQ JSON text.", json);
log.Verbose("SSQ JSON text: {@SsqJsonText}", json);
return(json);
}
public void QAVMAWebTest(QAVTest test)
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_MONTHLY_ADJUSTED&symbol={test.Symbol}&apikey={test.ApiKey}");
object actual = Avq.QAVMA(test.Symbol, info: test.Info, tradingMonth: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch);
QAVAssert(test, actual);
}
public static object QBIC([ExcelArgument(Name = "IBAN", Description = "the IBAN for which the BIC should be returned.")]
string iban)
{
const string FUNCTON_NAME = "QBIC";
// IBAN is mandatory.
if (string.IsNullOrEmpty(iban))
{
return(OpenibanExcelError(FUNCTON_NAME, "NULL!"));
}
// Check on valid IBAN.
// https://stackoverflow.com/questions/44656264/iban-regex-design
if (!System.Text.RegularExpressions.Regex.IsMatch(iban, @"^([A-Z]{2}[ \-]?[0-9]{2})(?=(?:[ \-]?[A-Z0-9]){9,30}$)((?:[ \-]?[A-Z0-9]{3,5}){2,7})([ \-]?[A-Z0-9]{1,3})?$"))
{
log.Error("Invalid {@IBAN}", iban);
return(OpenibanExcelError(FUNCTON_NAME, "INVALID_IBAN"));
}
JObject json = null;
try
{
//var ffeAttribute = (FfeFunctionAttribute)Attribute.GetCustomAttribute(typeof(Openiban).GetMethod(System.Reflection.MethodBase.GetCurrentMethod().Name), typeof(FfeFunctionAttribute));
log.Debug("Querying the BIC for IBAN {@IBAN} from {@Provider}", iban, "openiban.com");
var endpoint = $"{URL_OPEN_IBAN}/validate/{iban}?getBIC=true&validateBankCode=true";
log.Debug("Openiban endpoint URL: {@OpenibanEndpointUrl}.", endpoint);
// TODO: Newtonsoft.Json.JsonConvert.DeserializeObject<List<MyType>(jsonData); //faster with typed object
if (UrlContenResult == null)
{
json = JObject.Parse(FfeWeb.GetHttpResponseContent(endpoint));
}
else // else running tests.
{
json = JObject.Parse(UrlContenResult);
}
if ((bool)json["valid"])
{
return((string)json["bankData"]["bic"]);
}
else
{
log.Error((string)json["messages"][0]);
return(OpenibanExcelError(FUNCTON_NAME, "INVALID_IBAN"));
}
}
catch (Exception ex)
{
log.Error(ex, FUNCTON_NAME);
if (json != null)
{
log.Error("Openiban JSON response: {@OpenibanResponse}", json.ToString());
}
return(ExcelError.ExcelErrorGettingData);
}
}
public static object QIBAN([ExcelArgument(Name = "Country Code", Description = "the country code (ISO 3166-1 alpha-2 code) for which the IBAN should be calculated.\nUse =QCOUNTRIES() to get an array of supported countries.")]
string countryCode,
[ExcelArgument(Name = "Bank Code", Description = "the bank code for which the IBAN should be calculated.")]
string bankCode,
[ExcelArgument(Name = "Account Number", Description = "the account number for which the IBAN should be calculated.")]
string accountNumber)
{
const string FUNCTON_NAME = "QIBAN";
// All arguments are mandatory (Prevalidation).
if (string.IsNullOrEmpty(countryCode)
//|| !System.Text.RegularExpressions.Regex.IsMatch(countryCode, @"^[a-zA-Z]{2}$")
|| string.IsNullOrEmpty(bankCode) ||
string.IsNullOrEmpty(accountNumber))
{
return(OpenibanExcelError(FUNCTON_NAME, "NULL!"));
}
JObject json = null;
try
{
log.Debug("Querying the IBAN from {@Provider}", "openiban.com");
var endpoint = $"{URL_OPEN_IBAN}/calculate/{countryCode}/{bankCode}/{accountNumber}";
log.Debug("Openiban endpoint URL: {@OpenibanEndpointUrl}.", endpoint);
if (UrlContenResult == null)
{
json = JObject.Parse(FfeWeb.GetHttpResponseContent(endpoint));
}
else // else running tests.
{
json = JObject.Parse(UrlContenResult);
}
if ((bool)json["valid"])
{
return((string)json["iban"]);
}
else
{
log.Error((string)json["message"]);
return(OpenibanExcelError(FUNCTON_NAME, "INVALID"));
}
}
catch (Exception ex)
{
log.Error(ex, FUNCTON_NAME);
if (json != null)
{
log.Error("Openiban JSON response: {@OpenibanResponse}", json.ToString());
}
return(ExcelError.ExcelErrorGettingData);
}
}
public void QAVIDWebTest(QAVTest test)
{
if (test.OutputSize.Equals("full"))
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={test.Symbol}&interval={test.Interval}&outputsize={test.OutputSize}&apikey={test.ApiKey}");
}
else
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={test.Symbol}&interval={test.Interval}&apikey={test.ApiKey}");
}
object actual = Avq.QAVID(test.Symbol, info: test.Info, dataPointIndex: test.DatePart, interval: test.Interval, outputSize: test.OutputSize);
QAVAssert(test, actual);
}
public void QAVDAWebTest(QAVTest test)
{
if (test.OutputSize.Equals("full"))
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={test.Symbol}&outputsize={test.OutputSize}&apikey={test.ApiKey}");
}
else
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={test.Symbol}&apikey={test.ApiKey}");
}
object actual = Avq.QAVDA(test.Symbol, info: test.Info, tradingDay: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch, outputSize: test.OutputSize);
QAVAssert(test, actual);
}
public void QAVDWebNegativeTest(QAVTest test)
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol={test.Symbol}&apikey={test.ApiKey}");
if (test.OutputSize.Equals("n/a"))
{
Assert.Throws <ArgumentException>(() => Avq.QAVD(test.Symbol, info: test.Info, tradingDay: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch, outputSize: test.OutputSize));
}
else
{
object actual = Avq.QAVD(test.Symbol, info: test.Info, tradingDay: test.DatePart, tradingDate: test.QuoteDate, bestMatch: test.BestMatch, outputSize: test.OutputSize);
Assert.Equal(test.Expected, actual);
}
}
public dynamic Load(string url)
{
/*using (Stream stream = FfeWeb.GetHttpResponseContentAsStreamReader(url, GenerateRandomUserAgent))
* using (StreamReader streamReader = new StreamReader(stream))
* using (JsonReader reader = new JsonTextReader(streamReader))
* {
* //return JObject.Load(reader);
* return JArray.Load(reader);
* }*/
string json = FfeWeb.GetHttpResponseContent(url, GenerateRandomUserAgent);
//return JObject.Parse(json);
return(JArray.Parse(json));
}
public static object[,] QCOUNTRIES()
{
const string FUNCTON_NAME = "QCOUNTRIES";
string json = null;
try
{
log.Debug("Querying the supported countries from {@Provider}", "openiban.com");
var endpoint = $"{URL_OPEN_IBAN}/countries";
log.Debug("Openiban endpoint URL: {@OpenibanEndpointUrl}.", endpoint);
if (UrlContenResult == null)
{
json = FfeWeb.GetHttpResponseContent(endpoint);
}
else // else running tests.
{
json = UrlContenResult;
}
Dictionary <string, string> countries = JsonConvert.DeserializeObject <Dictionary <string, string> >(json);
int numberOfCountries = countries.Count;
object[,] countryArray = new object[numberOfCountries, 2];
for (int i = 0; i < numberOfCountries; i++)
{
KeyValuePair <string, string> country = countries.ElementAt(i);
countryArray[i, 0] = country.Key;
countryArray[i, 1] = country.Value;
}
return(countryArray);
}
catch (Exception ex)
{
log.Error(ex, FUNCTON_NAME);
if (json != null)
{
log.Error("Openiban JSON response: {@OpenibanResponse}", json.ToString());
}
return(new object[0, 0]);
}
}
public void QAVTSWebTest(QAVTest test)
{
string api;
string interval = test.Interval.ToLower();
if (interval.Equals("daily") ||
interval.Equals("weekly") ||
interval.Equals("monthly"))
{
api = $"TIME_SERIES_{interval.ToUpper()}" + (test.Adjusted ? "_ADJUSTED" : "");
// Remove not valid API parameters (for demo API key).
if (!interval.Equals("daily") ||
test.OutputSize.Equals("compact"))
{
test.OutputSize = null;
}
test.Interval = null;
}
else
{
api = "TIME_SERIES_INTRADAY";
// Remove not valid TIME_SERIES_INTRADAY API parameter (for demo API key).
test.OutputSize = null;
}
Avq.AvStockTimeSeriesOutputSize?outputSize = null;
if (test.OutputSize != null)
{
outputSize = (Avq.AvStockTimeSeriesOutputSize)Enum.Parse(typeof(Avq.AvStockTimeSeriesOutputSize), test.OutputSize);
}
string avUri = Avq.AvUrlBuilder(api, apiKey: test.ApiKey, symbol: test.Symbol, outputSize: outputSize, interval: test.Interval);
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent(avUri);
object actual = Avq.QAVTS(test.Symbol, info: test.Info, interval: test.Interval, tradingDay: test.DatePart, tradingDate: test.QuoteDate, adjusted: test.Adjusted, outputSize: test.OutputSize, bestMatch: test.BestMatch);
QAVAssert(test, actual);
}
public void QAVQWebTest(QAVTest test)
{
Avq.UrlContenResult = FfeWeb.GetHttpResponseContent($"https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol={test.Symbol}&apikey={test.ApiKey}");
object actual = Avq.QAVQ(test.Symbol, info: test.Info);
// If a date specific argument was given, then stock info may not available when trading date is on weekend or a public holiday.
DateTime tradingDate = DateTime.Today.AddDays(test.DatePart);
if (tradingDate.DayOfWeek == DayOfWeek.Saturday ||
tradingDate.DayOfWeek == DayOfWeek.Sunday ||
DateSystem.IsPublicHoliday(tradingDate, CountryCode.US))
{
// No quotes are available on the weekend or on public holidays.
Assert.True(ExcelError.ExcelErrorNA.Equals(actual) ||
actual is DateTime ||
actual is string ||
actual is decimal);
}
else
{
switch (test.Info)
{
case "latest trading day":
Assert.IsType <DateTime>(actual);
break;
case "change percent":
Assert.IsType <string>(actual);
break;
default:
Assert.IsType <decimal>(actual);
break;
}
}
}
public static (string value, string rawSource) GetValueFromWeb(string url,
string xPath = null,
string cssSelector = null,
string regExPattern = null, string regExGroup = null, int regExMatchIndex = 0,
string jsonPath = null,
Parser parser = Parser.Auto)
{
if (xPath == null &&
cssSelector == null &&
regExPattern == null &&
jsonPath == null)
{
throw new ArgumentException("No selection criteria were provided. Provide a least one of the following selection criteria: xPath, cssSelector, regExPattern or jsonPath.");
}
string value = null;
string rawSource = null;
Uri uri = new Uri(url);
if (parser != Parser.Newtonsoft)
{
IFfeWebParser ffeWebParser = null;
switch (parser)
{
case Parser.Auto:
ffeWebParser = AutoWebParserSelection(uri, xPath, cssSelector, regExPattern);
break;
case Parser.HAP:
ffeWebParser = new FfeWebHap(uri);
break;
case Parser.AngleSharp:
ffeWebParser = new FfeWebAngleSharp(uri);
break;
case Parser.HttpClient:
ffeWebParser = new FfeWebHttpClient(uri);
break;
case Parser.WebClient:
ffeWebParser = new FfeWebClient(uri);
break;
default:
ffeWebParser = new FfeWebHap(uri);
break;
}
rawSource = ffeWebParser.GetHtml();
if (log.IsEnabled(Serilog.Events.LogEventLevel.Debug))
{
rawSource.WriteToFile($"PageSource_{uri.Host}.html", parser.ToString());
}
// Input for RegEx (if set).
string input = null;
if (!String.IsNullOrEmpty(xPath))
{
value = ffeWebParser.SelectByXPath(xPath);
input = value;
}
else if (!String.IsNullOrEmpty(cssSelector))
{
value = ffeWebParser.SelectByCssSelector(cssSelector);
input = value;
}
else // Select by RegEx (HTML source code = RegEx input).
{
//HACK: AngelSharp does not provide full HTML source code.
if (parser == Parser.AngleSharp)
{
log.Warning("Regular Expression with AngleSharp WebParser does not work well. HttpClient is used instead.");
rawSource = FfeWeb.GetHttpResponseContent(uri);
}
input = rawSource;
}
if (!String.IsNullOrEmpty(regExPattern))
{
value = FfeRegEx.RegExByIndexAndGroup(input, regExPattern, regExMatchIndex, regExGroup);
if (String.IsNullOrEmpty(value))
{
if (log.IsEnabled(Serilog.Events.LogEventLevel.Debug))
{
input.WriteToFile("RegExInput.html", "RegEx");
}
throw new RegExException()
{
Input = input, Pattern = regExPattern
};
}
}
}
else
{
IFfeJsonParser ffeJsonParser = new FfeJsonNewtonsoft(uri);
value = ffeJsonParser.SelectByJsonPath(jsonPath);
rawSource = ffeJsonParser.GetJson();
if (log.IsEnabled(Serilog.Events.LogEventLevel.Debug))
{
rawSource.WriteToFile($"JsonSource_{uri.Host}.json", parser.ToString());
}
}
return(value, rawSource);
}
