protected override NasdaqEarningForecastResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
List <NasdaqEarningForecastData> yearly = new List <NasdaqEarningForecastData>(10);
List <NasdaqEarningForecastData> quarterly = new List <NasdaqEarningForecastData>(10);
System.Globalization.CultureInfo culture = Factory.DownloadCultureInfo;
string pattern = @"<title>.*\((\w*)\).*</title>";
if (stream != null)
{
var content = MyHelper.StreamToString(stream, System.Text.Encoding.UTF8);
var matchPattern = "(<div class=\"genTable\">.*?</div>)";
var match = Regex.Matches(content, matchPattern, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant);
XParseDocument year = MyHelper.ParseXmlDocument(match[0].Groups[0].Value);
XParseDocument quarter = MyHelper.ParseXmlDocument(match[1].Groups[0].Value);
var symbol = Regex.Match(content, pattern).Groups[1].Value;
var resultNode = XPath.GetElement("//table", year);
ParseTable(yearly, resultNode, symbol, "");
resultNode = XPath.GetElement("//table", quarter);
ParseTable(quarterly, resultNode, symbol, "");
return(new NasdaqEarningForecastResult(yearly.ToArray(), quarterly.ToArray()));
}
return(null);
}
protected override QuoteOptionsResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
List <QuoteOptionsDataChain> options = new List <QuoteOptionsDataChain>();
System.Globalization.CultureInfo culture = new System.Globalization.CultureInfo("en-US");
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement[] mainLst = XPath.GetElements("//optionsChain", doc);
foreach (XParseElement chain in mainLst)
{
string idAtt = MyHelper.GetXmlAttributeValue(chain, "symbol");
string expirationDateAtt = MyHelper.GetXmlAttributeValue(chain, "expiration");
DateTime expirationDate = default(DateTime);
if (!System.DateTime.TryParseExact(expirationDateAtt, "yyyy-MM-dd", culture, System.Globalization.DateTimeStyles.None, out expirationDate))
{
System.DateTime.TryParseExact(expirationDateAtt, "yyyy-MM", culture, System.Globalization.DateTimeStyles.None, out expirationDate);
}
List <QuoteOptionsData> lst = new List <QuoteOptionsData>();
foreach (XParseElement optionNode in chain.Elements())
{
if (optionNode.Name.LocalName == "option")
{
QuoteOptionsData opt = ImportExport.ToQuoteOption(optionNode, culture);
if (opt != null)
{
lst.Add(opt);
}
}
}
options.Add(new QuoteOptionsDataChain(idAtt, expirationDate, lst));
}
return(new QuoteOptionsResult(options.ToArray()));
}
protected override TickerEarningDate ConvertResult(string contentStr, string ticker = "")
{
XParseDocument doc = MyHelper.ParseXmlDocument(contentStr);
#if DEBUG
Console.WriteLine(contentStr);
#endif
XParseElement resultNode = XPath.GetElement("//results/td/strong", doc);
if (resultNode == null)
{
resultNode = XPath.GetElement("//results/td/b", doc);
}
if (resultNode == null)
{
return new TickerEarningDate {
Ticker = ticker
}
}
;
var dateStr = resultNode.Value;
var date = MyHelper.ConvertEarningDate(dateStr);
return(new TickerEarningDate {
Ticker = ticker, EarningDate = date
});
}
}
protected sealed override NasdaqEarningForecastAggregate ConvertResult(string contentStr, string ticker = "")
{
List <NasdaqEarningForecastData> yearly = new List <NasdaqEarningForecastData>(10);
List <NasdaqEarningForecastData> quarterly = new List <NasdaqEarningForecastData>(10);
string pattern = @"<title>.*\((\w*)\).*</title>";
if (!string.IsNullOrEmpty(contentStr))
{
var content = contentStr;
var matchPattern = "(<div class=\"genTable\">.*?</div>)";
var match = Regex.Matches(content, matchPattern, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant);
XParseDocument year = MyHelper.ParseXmlDocument(match[0].Groups[0].Value);
XParseDocument quarter = MyHelper.ParseXmlDocument(match[1].Groups[0].Value);
var symbol = Regex.Match(content, pattern).Groups[1].Value;
var resultNode = XPath.GetElement("//table", year);
ParseTable(yearly, resultNode, symbol, "", () => CreateData(Frequency.Yearly));
resultNode = XPath.GetElement("//table", quarter);
ParseTable(quarterly, resultNode, symbol, "", () => CreateData(Frequency.Quarterly));
return(new NasdaqEarningForecastAggregate(yearly.ToArray(), quarterly.ToArray(), ticker));
}
return(null);
}
protected override HoldingsResult ConvertResult(ConnectionInfo connInfo, System.IO.Stream stream, SettingsBase settings)
{
HoldingsDownloadSettings set = (HoldingsDownloadSettings)settings;
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
Holding[] result = this.ConvertHtmlDoc(doc, set.PortfolioID);
return(new HoldingsResult(result));
}
protected override AnalystRatingsDataAggregate ConvertResult(string contentStr, string ticker = "")
{
XParseDocument doc = MyHelper.GetResultTable(contentStr, 1, @"<table id='ratingstable'");
XParseElement[] rows = XPath.GetElements("//tr", doc);
AnalystRatingsData[] data = GetResult <AnalystRatingsData>(rows);
return(GetAggregate(data, _ => new AnalystRatingsDataAggregate(data)));
}
protected override CpiDataAggregate ConvertResult(string content, string ticker = "")
{
//List<CpiData> dataList = new List<CpiData>(100);
XParseDocument doc = MyHelper.GetResultTable(content, 1, "<table");
XParseElement[] results = XPath.GetElements("//tr", doc);
CpiData[] data = GetResult <CpiData>(results);
return(GetAggregate(data, _ => new CpiDataAggregate(data)));
}
public static XParseDocument GetResultTable(string content, int nthTable, string tableStart, string tableEnd = "</table>")
{
var startIndex = content.IndexOfOccurence(tableStart, nthTable);
var endIndex = content.IndexOf(tableEnd, startIndex, StringComparison.InvariantCultureIgnoreCase);
var targetTableStr = content.Substring(startIndex, endIndex + tableEnd.Length - startIndex);
XParseDocument doc = MyHelper.ParseXmlDocument(targetTableStr);
return(doc);
}
/// <summary>
/// Loads a list of default currencies from market.xml
/// </summary>
/// <returns></returns>
/// <remarks></remarks>
public static CurrencyInfo[] GetDefaultCurrencies()
{
List <CurrencyInfo> currencies = new List <CurrencyInfo>();
XParseDocument xmlDoc = MyHelper.ParseXmlDocument(Properties.Resources.market);
XParseElement[] curs = XPath.GetElements("//Resources/Currencies/Currency", xmlDoc);
foreach (XParseElement curNode in curs)
{
currencies.Add(new CurrencyInfo(MyHelper.GetXmlAttributeValue(curNode, "ID"), MyHelper.GetXmlAttributeValue(curNode, "Name")));
}
return(currencies.ToArray());
}
/// <summary>
/// Loads default market information from market.xml
/// </summary>
/// <remarks></remarks>
public static void FillCountriesWithIndices()
{
XParseDocument xmlDoc = MyHelper.ParseXmlDocument(Properties.Resources.market);
XParseElement[] countryNodes = XPath.GetElements("//Resources/Countries/Country", xmlDoc);
foreach (XParseElement countryNode in countryNodes)
{
CountryInfo ctr = null;
string ctrIDStr = MyHelper.GetXmlAttributeValue(countryNode, "ID");
foreach (CountryInfo defaultCtr in DefaultCountries)
{
if (defaultCtr.ID.ToString() == ctrIDStr)
{
ctr = defaultCtr;
break;
}
}
if (ctr != null)
{
ctr.Indices.Clear();
XParseElement indicesNode = XPath.GetElement("Indices", countryNode);
foreach (XParseElement indexNode in indicesNode.Elements())
{
if (indexNode.Name.LocalName == "Index")
{
string name = MyHelper.GetXmlAttributeValue(indexNode, "Name");
string id = MyHelper.GetXmlAttributeValue(indexNode, "ID");
string seStr = MyHelper.GetXmlAttributeValue(indexNode, "StockExchange");
StockExchange se = null;
foreach (StockExchange defaultExc in mStockExchanges)
{
if (defaultExc.ID == seStr)
{
se = defaultExc;
break;
}
}
ctr.Indices.Add(new YIndexID(id)
{
Name = name,
StockExchange = se
});
}
}
}
}
}
protected override EarningHistoryDataAggregate ConvertResult(string content, string ticker = "")
{
List <EarningHistoryData> history = new List <EarningHistoryData>(10);
var matchPattern = "(<div class=\"genTable\">.*?</div>)";
var match = Regex.Matches(content, matchPattern, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant);
XParseDocument doc = MyHelper.ParseXmlDocument(match[0].Groups[0].Value);
var resultNode = XPath.GetElement("//table", doc);
ParseTable(history, resultNode, ticker, "");
return(new EarningHistoryDataAggregate(history.ToArray(), ticker));
}
private void dl_DownloadAsyncCompleted(DownloadClient <System.IO.Stream> sender, DownloadCompletedEventArgs <System.IO.Stream> e)
{
sender.AsyncDownloadCompleted -= this.dl_DownloadAsyncCompleted;
XParseDocument doc = null;
if (e.Response.Result != null)
{
doc = MyHelper.ParseXmlDocument(e.Response.Result);
}
if (this.AsyncUploadCompleted != null)
{
this.AsyncUploadCompleted(this, ((DefaultDownloadCompletedEventArgs <System.IO.Stream>)e).CreateNew(doc));
}
}
protected override FeedResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
List <Feed> feeds = new List <Feed>();
XParseDocument xmlDoc = MyHelper.ParseXmlDocument(stream);
if (xmlDoc != null)
{
foreach (XParseElement f in XPath.GetElements("//channel", xmlDoc))
{
feeds.Add(ImportExport.XML.ToFeed(f));
}
}
return(new FeedResult(feeds.ToArray(), (FeedDownloadSettings)settings));
}
protected override SuggestionResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
List <string> lst = new List <string>();
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement[] results = XPath.GetElements("//s", doc);
foreach (XParseElement resultNode in results)
{
string att = MyHelper.GetXmlAttributeValue(resultNode, "k");
if (att != string.Empty)
{
lst.Add(att);
}
}
return(new SuggestionResult(lst.ToArray(), (SuggestionDownloadSettings)settings));
}
protected override PlacesResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
if (settings is PlacesDownloadSettings)
{
List <PlacesData> places = new List <PlacesData>();
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement[] results = XPath.GetElements("//place", doc);
foreach (XParseElement node in results)
{
places.Add(this.ToPlace(node));
}
return(new PlacesResult(places.ToArray()));
}
else
{
return(null);
}
}
protected override WeatherFeedResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
List <WeatherFeed> feeds = new List <WeatherFeed>();
XParseDocument xmlDoc = MyHelper.ParseXmlDocument(stream);
if (xmlDoc != null)
{
foreach (XParseElement feedNode in XPath.GetElements("//channel", xmlDoc))
{
WeatherFeed feed = this.ToWeatherFeed(feedNode);
if (feed != null)
{
feeds.Add(feed);
}
}
}
return(new WeatherFeedResult(feeds.ToArray()));
}
/// <summary>
/// Loads as list of default countries from market.xml
/// </summary>
/// <returns></returns>
/// <remarks></remarks>
public static CountryInfo[] GetDefaultCountries()
{
List <CountryInfo> countries = new List <CountryInfo>();
XParseDocument xmlDoc = MyHelper.ParseXmlDocument(Properties.Resources.market);
XParseElement[] cntNodes = XPath.GetElements("//Resources/Countries/Country", xmlDoc);
System.Globalization.CultureInfo convCulture = new System.Globalization.CultureInfo("en-US");
foreach (XParseElement cntNode in cntNodes)
{
for (Country cnt = 0; cnt <= Country.VN; cnt++)
{
if (cnt.ToString() == MyHelper.GetXmlAttributeValue(cntNode, "ID"))
{
CurrencyInfo cntCur = null;
string curID = MyHelper.GetXmlAttributeValue(cntNode, "Currency");
foreach (CurrencyInfo cur in DefaultCurrencies)
{
if (cur.ID.ToString() == curID)
{
cntCur = cur;
break;
}
}
XParseElement dstNodes = XPath.GetElement("DaylightSavingTimes", cntNode);
List <DaylightSavingTime> dstList = new List <DaylightSavingTime>();
foreach (XParseElement dstNode in dstNodes.Elements())
{
if (dstNode.Name.LocalName == "DST")
{
DateTime dstStart = Convert.ToDateTime(MyHelper.GetXmlAttributeValue(dstNode, "Start"), convCulture);
DateTime dstEnd = Convert.ToDateTime(MyHelper.GetXmlAttributeValue(dstNode, "End"), convCulture);
dstList.Add(new DaylightSavingTime(dstStart, dstEnd));
}
}
countries.Add(new CountryInfo(cnt, MyHelper.GetXmlAttributeValue(cntNode, "Name"), cntCur, dstList.ToArray()));
break;
}
}
}
return(countries.ToArray());
}
public static XParseElement GetResultTable(string content, string matchPattern, string targetXpath)
{
try
{
var match = Regex.Matches(content, matchPattern, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant);
var targetTableStr = match[0].Groups[0].Value;
XParseDocument doc = MyHelper.ParseXmlDocument(targetTableStr);
var resultNode = XPath.GetElement(targetXpath, doc);
return(resultNode);
}
catch (Exception)
{
Debug.Write(matchPattern);
Debug.Write(content);
}
return(null);
}
protected override LocationIDSearchResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
List <LocationIDData> lst = new List <LocationIDData>();
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement[] results = XPath.GetElements("//loc", doc);
foreach (XParseElement locNode in results)
{
LocationIDData loc = new LocationIDData();
loc.Name = locNode.Value;
XParseAttribute att = locNode.Attribute(XParseName.Get("id"));
if (att != null)
{
loc.ID = att.Value;
}
lst.Add(loc);
}
return(new LocationIDSearchResult(lst.ToArray()));
}
protected override PositionChangeDataPointAggregate ConvertResult(string contentStr, string ticker = "")
{
string startPattern = "<table class=\"tableFile\" summary=\"Document Format Files\">";
string endPattern = "</table>";
int startIndex = contentStr.IndexOfOccurence(startPattern, 1);
int endIndex = contentStr.IndexOf(endPattern, startIndex);
string normalizeContent = contentStr.Substring(startIndex, endIndex - startIndex + endPattern.Length);
XParseDocument doc = MyHelper.ParseXmlDocument(normalizeContent);
XParseElement resultNode = XPath.GetElement("//table/tr[3]/td[3]/a", doc);
var xmlLink = resultNode.Attribute("href").Value;
var xmlResult = ParseForm4(GetSecPath(xmlLink));
var aggregate = new PositionChangeDataPointAggregate(xmlResult);
aggregate.PopulatePoints();
Console.WriteLine(xmlResult);
////*[@id="formDiv"]/div/table/tbody/tr[3]/td[3]/a
return(aggregate);
}
protected override ValuationDataPointAggregate ConvertResult(string contentStr, string ticker = "")
{
var aggregate = new ValuationDataPointAggregate(ticker);
if (!string.IsNullOrEmpty(contentStr))
{
var normalizeContent = contentStr;//.Replace("\r\n", "").Replace("\t", "");
/*due to attribute without quotes, can't be parsed it as xml*/
string startPattern = "<table";
string endPattern = "</table>";
int startIndex = normalizeContent.IndexOfOccurence(startPattern, 5);
int endIndex = normalizeContent.IndexOf(endPattern, startIndex);
normalizeContent = normalizeContent.Substring(startIndex, endIndex - startIndex + endPattern.Length);
var tempnormalizeContent = normalizeContent.Replace("\r\n", " ").Replace("\t", " ").Replace("\n", " ");
normalizeContent = MyHelper.FixAttributes(tempnormalizeContent);
XParseDocument doc = MyHelper.ParseXmlDocument(normalizeContent);
var resultNode = XPath.GetElement("/table", doc);
ParseTable(aggregate, resultNode, ticker, "");
return(aggregate);
}
return(null);
}
protected override HoldingDataPointAggregate ConvertResult(string contentStr, string ticker = "")
{
var aggregate = new HoldingDataPointAggregate(ticker);
if (!string.IsNullOrEmpty(contentStr))
{
var normalizeContent = contentStr;//.Replace("\r\n", "").Replace("\t", "");
/*due to attribute without quotes, can't be parsed it as xml*/
string startPattern = "<form";
string endPattern = "</form>";
string startTablePattern = "<table>";
string endTablePattern = "</table>";
int startIndex = normalizeContent.IndexOfOccurence(startPattern, 4);
int endIndex = normalizeContent.IndexOf(endPattern, startIndex);
normalizeContent = normalizeContent.Substring(startIndex, endIndex - startIndex + endPattern.Length);
var tempnormalizeContent = normalizeContent.Replace("\r\n", " ").Replace("\t", " ").Replace("\n", " ");
normalizeContent = MyHelper.FixAttributes(tempnormalizeContent);
XParseDocument doc = MyHelper.ParseXmlDocument(normalizeContent);
string temp;
for (int i = 1; i <= 4; i++)
{
startIndex = normalizeContent.IndexOfOccurence(startTablePattern, i);
endIndex = normalizeContent.IndexOf(endTablePattern, startIndex);
temp = normalizeContent.Substring(startIndex, endIndex - startIndex + endTablePattern.Length);
doc = MyHelper.ParseXmlDocument(temp);
var ownershipSummaryTable = XPath.GetElement("//table", doc);
ParseTable(aggregate, ownershipSummaryTable);
}
//var activePositionTable = XPath.GetElement("//table[2]", doc);
//var newSoldoutTable = XPath.GetElement("//table[3]", doc);
//var holdingDetail = XPath.GetElement("//table[4]", doc);
return(aggregate);
}
return(null);
}
internal PortfolioInfoResult ConvertHtml(XParseDocument doc)
{
XParseElement resultsNode = XPath.GetElement("//div[@id=\"yfi-main\"]/div/div[2]/form/div/table/tbody", doc);
List <PortfolioInfo> lst = new List <PortfolioInfo>();
if (resultsNode != null)
{
foreach (XParseElement trNode in resultsNode.Elements())
{
XParseElement a = XPath.GetElement("/td[2]/a", trNode);
if (a != null)
{
string name = a.Value;
XParseAttribute att = a.Attribute(XParseName.Get("href"));
if (att != null)
{
string id = att.Value.Split(';')[0].Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries)[1];
lst.Add(new PortfolioInfo(name, id));
}
}
}
}
return(new PortfolioInfoResult(lst.ToArray()));
}
protected override PortfolioInfoResult ConvertResult(ConnectionInfo connInfo, System.IO.Stream stream, SettingsBase settings)
{
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
return(this.ConvertHtml(doc));
}
protected override AlphabeticIDIndexResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
AlphabeticIDIndexSettings s = (AlphabeticIDIndexSettings)settings;
System.Globalization.CultureInfo convCulture = new System.Globalization.CultureInfo("en-US");
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement[] resultsNodes = null;
switch (s.Type)
{
case AlphabeticalIndexDownloadType.TopIndex:
resultsNodes = XPath.GetElements("//results", doc);
List <AlphabeticalTopIndex> lstTopIndex = new List <AlphabeticalTopIndex>();
if (resultsNodes.Length > 0)
{
XParseElement resultNode = resultsNodes[0];
foreach (XParseElement tdNode in resultNode.Elements())
{
foreach (XParseElement aNode in tdNode.Elements())
{
if (aNode.Name.LocalName == "a")
{
string att = MyHelper.GetXmlAttributeValue(aNode, "href");
if (att != string.Empty)
{
lstTopIndex.Add(new AlphabeticalTopIndex(aNode.Value, "http://biz.yahoo.com" + att));
}
}
}
}
}
return(new AlphabeticIDIndexResult(lstTopIndex.ToArray()));
case AlphabeticalIndexDownloadType.Index:
resultsNodes = XPath.GetElements("//results", doc);
List <AlphabeticalIndex> lstIndex = new List <AlphabeticalIndex>();
if (resultsNodes.Length > 0)
{
XParseElement resultNode = resultsNodes[0];
foreach (XParseElement tdNode in resultNode.Elements())
{
foreach (XParseElement tableNode in tdNode.Elements())
{
if (tableNode.Name.LocalName == "table")
{
XParseElement[] chdLst = MyHelper.EnumToArray(MyHelper.EnumToArray(tableNode.Elements())[0].Elements());
if (chdLst.Length >= 3)
{
lstIndex.Add(new AlphabeticalIndex(chdLst[1].Value, s.TopIndex.URL));
if (chdLst.Length > 3)
{
for (int i = 3; i <= chdLst.Length - 1; i++)
{
XParseElement tdTbNode = chdLst[i];
XParseElement[] enm = MyHelper.EnumToArray(tdTbNode.Elements());
if (enm.Length > 0 && enm[0].Name.LocalName == "a")
{
string name = enm[0].Value.Replace("\n", "").Replace(" ", "").Trim();
string url = string.Empty;
string att = MyHelper.GetXmlAttributeValue(enm[0], "href");
if (att != string.Empty)
{
url = "http://biz.yahoo.com" + att.Trim();
}
if (url != string.Empty & name != string.Empty)
{
lstIndex.Add(new AlphabeticalIndex(name, url));
}
}
}
}
}
}
}
}
}
s.TopIndex.SetIndices(lstIndex.ToArray());
return(new AlphabeticIDIndexResult(s.TopIndex.SubIndices));
default:
return(null);
}
}
//http:/table/regexhero.net/tester/
protected override FinvizCompanyDetails ConvertResult(string contentStr, string ticker = "")
{
#if DEBUG
Console.WriteLine(contentStr);
#endif
var normalizeContent = contentStr.Replace("\r\n", "").Replace("\t", "");
string startPattern = "<table width=\"100%\" cellpadding=\"3\" cellspacing=\"0\" border=\"0\" class=\"snapshot-table2\">";
string endPattern = "</table>";
int startIndex = normalizeContent.IndexOf(startPattern);
int endIndex = normalizeContent.IndexOf("</table>", startIndex);
var tableStr = normalizeContent.Substring(startIndex, endPattern.Length + endIndex - startIndex);
//var tableStr = MyHelper.ExtractPattern(normalizeContent, ".*<table.*class=\"snapshot-table2\">(?<table>.*)</table>");
XParseDocument doc = MyHelper.ParseXmlDocument(tableStr);
XParseElement resultNode;// = XPath.GetElement("/table/tr[1]/td[2]/b", doc);
object value = null;
dynamic normalizeStr;
XPathAttribute xpath;
RegularExpressionAttribute regex;
var details = new FinvizCompanyDetails {
Ticker = Setting.Ticker
};
foreach (var property in details.GetType().GetProperties())
{
xpath = property.GetCustomAttributes(typeof(XPathAttribute), false).FirstOrDefault() as XPathAttribute;
if (xpath == null)
{
continue;
}
resultNode = XPath.GetElement(xpath.Path, doc);
var type = Nullable.GetUnderlyingType(property.PropertyType);
normalizeStr = resultNode.Value;
regex = property.GetCustomAttributes(typeof(RegularExpressionAttribute), false).FirstOrDefault() as RegularExpressionAttribute;
try
{
if (regex != null)
{
normalizeStr = MyHelper.NumerizeString(MyHelper.ExtractPattern(normalizeStr, regex.Pattern));
}
else if (property.PropertyType == typeof(DateTime))
{
normalizeStr = MyHelper.ConvertEarningDate(normalizeStr);
}
else if (property.PropertyType != typeof(string))
{
normalizeStr = MyHelper.NumerizeString(resultNode.Value);
}
if (type != null)
{
value = Convert.ChangeType(normalizeStr, type, MyHelper.DefaultCulture.NumberFormat);
}
else
{
value = Convert.ChangeType(normalizeStr, property.PropertyType);
}
}
catch (Exception ex)
{
string msg = ex.Message;
}
property.SetValue(details, value);
}
return(details);
}
protected override TrefisCompanyCoveredInfoAggregate ConvertResult(string contentStr, string ticker = "")
{
List <TrefisCompanyCoveredInfoData> companies = new List <TrefisCompanyCoveredInfoData>(100);
var normalizeContent = contentStr.Replace("\r\n", "").Replace("\t", "");
XParseDocument doc = MyHelper.ParseXmlDocument(normalizeContent);
int startIndex = normalizeContent.IndexOf("initCompanies(") + "initCompanies(".Length;
int endIndex = normalizeContent.IndexOf(");});</script>");
var jsonStr = normalizeContent.Substring(startIndex, endIndex - startIndex);
// jsonStr = MyHelper.ExtractPattern(normalizeContent, @".*initCompanies\((?<content>.*)\);\}\);</script>");
// var serializer = new DataContractJsonSerializer(typeof(TrefisCompanyCoveredInfoData));// JavaScriptSerializer();
List <TrefisCompanyCoveredInfoData> dataList = MyHelper.FromJson <List <TrefisCompanyCoveredInfoData> >(jsonStr);
//serializer..Deserialize<List<TrefisCompanyCoveredInfoData>>(scriptContent);
//XDocument xdoc = XDocument.Parse(normalizeContent);
//XElement tempElement = null;
//var rows = xdoc.XPathSelectElements("//tbody[@class=\"cmpTblBody\")]/tr");
//foreach (var row in rows)
//{
// var data = new TrefisCompanyCoveredInfoData();
// tempElement = row.Document.XPathSelectElement("/td[1]/a");
// data.CompanyName = tempElement.Value;
// data.Ticker = this.Setting.GetTickerFromUrl(tempElement.Attribute(XName.Get("href")).Value);
// tempElement = row.Document.XPathSelectElement("/td[2]/a");
// data.TrefisTarget = double.Parse(tempElement.Value, NumberStyles.Currency);
// tempElement = row.Document.XPathSelectElement("/td[3]");
// data.PriceGap = float.Parse(tempElement.Value);
// tempElement = row.Document.XPathSelectElement("/td[4]");
// data.Sector = tempElement.Value;
// tempElement = row.Document.XPathSelectElement("/td[5]");
// data.Industry = tempElement.Value;
// tempElement = row.Document.XPathSelectElement("/td[6]");
// data.Bearishness = float.Parse(tempElement.Value);
// companies.Add(data);
//}
var culture = MyHelper.DefaultCulture;
XParseElement[] results = XPath.GetElements("//tbody[@class=\"cmpTblBody\")]/tr", doc);
XParseElement tempNode = null;
foreach (XParseElement row in results)
{
tempNode = XPath.GetElement("/td[1]/a", row);
var data = dataList.Where(company => company.CompanyName == HttpUtility.HtmlDecode(tempNode.Value)).FirstOrDefault();
tempNode = XPath.GetElement("/td[4]", row);
data.Sector = tempNode.Value;
tempNode = XPath.GetElement("/td[5]", row);
data.Industry = tempNode.Value;
}
return(new TrefisCompanyCoveredInfoAggregate(dataList.ToArray()));
}
/// <summary>
/// Loads a list of default stock exchanges from market.xml
/// </summary>
/// <returns></returns>
/// <remarks></remarks>
public static StockExchange[] GetDefaultStockExchanges()
{
List <StockExchange> lst = new List <StockExchange>();
XParseDocument xmlDoc = MyHelper.ParseXmlDocument(Properties.Resources.market);
XParseElement[] exchanges = XPath.GetElements("//Resources/StockExchanges/StockExchange", xmlDoc);
foreach (XParseElement exchangeNode in exchanges)
{
string seID = MyHelper.GetXmlAttributeValue(exchangeNode, "ID");
string seSuffix = MyHelper.GetXmlAttributeValue(exchangeNode, "Suffix");
string seName = MyHelper.GetXmlAttributeValue(exchangeNode, "Name");
CountryInfo seCountry = null;
string ctrID = MyHelper.GetXmlAttributeValue(exchangeNode, "Country");
foreach (CountryInfo ctr in DefaultCountries)
{
if (ctr.ID.ToString() == ctrID)
{
seCountry = ctr;
break;
}
}
//TradingTimeInfo
int seDelayMinutes = Convert.ToInt32(MyHelper.GetXmlAttributeValue(exchangeNode, "DelayMinutes"));
int seRelativeToUTC = Convert.ToInt32(MyHelper.GetXmlAttributeValue(exchangeNode, "UtcOffsetStandardTime"));
DateTime seOpeningTimeLocal = Convert.ToDateTime(MyHelper.GetXmlAttributeValue(exchangeNode, "OpeningTimeLocal"));
DateTime seClosingTimeLocal = Convert.ToDateTime(MyHelper.GetXmlAttributeValue(exchangeNode, "ClosingTimeLocal"));
TimeSpan seTradingSpan = seClosingTimeLocal - seOpeningTimeLocal;
List <DayOfWeek> seTradingDaysList = new List <DayOfWeek>();
string trdDays = MyHelper.GetXmlAttributeValue(exchangeNode, "TradingDays");
foreach (string day in trdDays.Split(','))
{
switch (day)
{
case "Mo":
seTradingDaysList.Add(DayOfWeek.Monday);
break;
case "Tu":
seTradingDaysList.Add(DayOfWeek.Tuesday);
break;
case "We":
seTradingDaysList.Add(DayOfWeek.Wednesday);
break;
case "Th":
seTradingDaysList.Add(DayOfWeek.Thursday);
break;
case "Fr":
seTradingDaysList.Add(DayOfWeek.Friday);
break;
case "Sa":
seTradingDaysList.Add(DayOfWeek.Saturday);
break;
case "Su":
seTradingDaysList.Add(DayOfWeek.Sunday);
break;
}
}
DaylightSavingTime[] seDaylightSavingTimes = null;
if (seCountry != null)
{
seDaylightSavingTimes = seCountry.DaylightSavingTimes;
}
TradingTimeInfo seTradingTimeInfo = new TradingTimeInfo(seDelayMinutes, seTradingDaysList.ToArray(), null, seOpeningTimeLocal, seTradingSpan, seRelativeToUTC, seDaylightSavingTimes);
StockExchange se = new StockExchange(seID, seSuffix, seName, seCountry, seTradingTimeInfo);
string s = MyHelper.GetXmlAttributeValue(exchangeNode, "Tags");
if (s != string.Empty)
{
se.Tags.AddRange(s.Split(','));
}
lst.Add(se);
}
return(lst.ToArray());
}
protected override IDSearchResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
IDSearchResult result = null;
List <IDSearchData> lst = new List <IDSearchData>();
if (stream != null)
{
if (settings is IDInstantSearchDownloadSettings)
{
#region Instant
string resultStr = MyHelper.StreamToString(stream, ((IDInstantSearchDownloadSettings)settings).TextEncoding);
MatchCollection results = Regex.Matches(resultStr, "{\"symbol\":.*?}");
foreach (Match res in results)
{
string[] prp = res.Value.Replace("{", "").Replace("}", "").Split(',');
if (prp.Length > 0)
{
string name = string.Empty;
string id = string.Empty;
string category = string.Empty;
string exchange = string.Empty;
string type = string.Empty;
foreach (string p in prp)
{
string[] kvp = p.Replace("\"", "").Split(':');
if (kvp.Length == 2)
{
switch (kvp[0])
{
case "symbol":
id = kvp[1].Trim();
break;
case "name":
name = kvp[1].Trim();
break;
case "exch":
exchange = kvp[1].Trim();
break;
case "type":
switch (kvp[1].Trim())
{
case "S":
type = "Stock";
break;
case "I":
type = "Index";
break;
case "F":
type = "Future";
break;
case "E":
type = "ETF";
break;
case "M":
type = "Fund";
break;
}
break;
}
}
}
lst.Add(new IDSearchData(name, id, type, exchange, string.Empty, null));
}
}
Dictionary <SecurityType, int> dict = new Dictionary <SecurityType, int>();
dict.Add(SecurityType.Any, lst.Count);
return(new IDSearchResult(lst.ToArray(), 0, lst.Count, lst.Count, dict));
#endregion
}
else if (settings is IDQuerySearchDownloadSettings)
{
#region Query
IDQuerySearchDownloadSettings sett = (IDQuerySearchDownloadSettings)settings;
int pageingFrom = sett.ResultsIndex, pagingTo = sett.ResultsIndex + 20, overall = 0;
Dictionary <SecurityType, int> resultsCount = new Dictionary <SecurityType, int>();
System.Globalization.CultureInfo convCulture = new System.Globalization.CultureInfo("en-US");
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement resultNode = XPath.GetElement("//div[@id=\"yfi_sym_lookup\"]", doc);
if (resultNode != null)
{
XParseElement navigationNode = XPath.GetElement("ul[1]", resultNode);
if (navigationNode != null)
{
string s;
int t;
s = XPath.GetElement("li[1]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Any, t);
}
s = XPath.GetElement("li[2]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Stock, t);
}
s = XPath.GetElement("li[3]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Fund, t);
}
s = XPath.GetElement("li[4]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.ETF, t);
}
s = XPath.GetElement("li[5]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Index, t);
}
s = XPath.GetElement("li[6]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Future, t);
}
if (MyHelper.EnumToArray(navigationNode.Elements()).Length == 7)
{
resultsCount.Add(SecurityType.Warrant, 0);
s = XPath.GetElement("li[7]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Currency, t);
}
}
else if (MyHelper.EnumToArray(navigationNode.Elements()).Length == 8)
{
s = XPath.GetElement("li[7]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Warrant, t);
}
s = XPath.GetElement("li[8]/a/em", navigationNode).Value;
if (int.TryParse(s.Substring(s.LastIndexOf("(") + 1).Replace(")", "").Trim(), out t))
{
resultsCount.Add(SecurityType.Currency, t);
}
}
}
XParseElement contentNode = XPath.GetElement("div[1]", resultNode);
if (contentNode != null)
{
XParseElement tableNode = XPath.GetElement("/div[1]/table", contentNode);
XParseElement tableHeadNode = XPath.GetElement("/thead/tr", tableNode);
XParseElement tableBodyNode = XPath.GetElement("/tbody", tableNode);
List <string> tableColumnNames = new List <string>();
tableColumnNames.Add("symbol");
tableColumnNames.Add("name");
bool hasISIN = XPath.GetElement("/th[3]", tableHeadNode).Value.ToLower().Contains("isin");
if (hasISIN)
{
tableColumnNames.Add("isin");
}
else
{
tableColumnNames.Add("lasttrade");
}
int l = MyHelper.EnumToArray(tableHeadNode.Elements()).Length;
for (int i = 3; i < l; i++)
{
if (hasISIN)
{
switch (i)
{
case 3:
tableColumnNames.Add("lasttrade");
break;
case 4:
tableColumnNames.Add("type");
break;
case 5:
tableColumnNames.Add("exchange");
break;
}
}
else
{
string name = MyHelper.GetEnumItemAt(tableHeadNode.Elements(), i).Value.ToLower();
if (name.Contains("type"))
{
tableColumnNames.Add("type");
}
else if (name.Contains("industry"))
{
tableColumnNames.Add("industry");
}
else if (name.Contains("exchange"))
{
tableColumnNames.Add("exchange");
}
}
}
foreach (XParseElement rowNode in tableBodyNode.Elements())
{
IEnumerable <XParseElement> enm = rowNode.Elements();
if (MyHelper.EnumToArray(enm).Length >= tableColumnNames.Count)
{
string name = string.Empty, id = string.Empty, type = string.Empty, industry = string.Empty, exchange = string.Empty;
ISIN isin = null;
for (int i = 0; i < tableColumnNames.Count; i++)
{
switch (tableColumnNames[i])
{
case "symbol":
id = MyHelper.GetEnumItemAt(enm, i).Value.Trim();
break;
case "name":
name = MyHelper.GetEnumItemAt(enm, i).Value.Trim();
break;
case "isin":
if (MyHelper.GetEnumItemAt(enm, i).Value.Trim() != string.Empty)
{
try
{
isin = new ISIN(MyHelper.GetEnumItemAt(enm, i).Value.Trim());
}
catch { }
}
break;
case "lasttrade":
break;
case "type":
type = MyHelper.GetEnumItemAt(enm, i).Value.Trim();
break;
case "industry":
industry = MyHelper.GetEnumItemAt(enm, i).Value.Trim();
break;
case "exchange":
exchange = MyHelper.GetEnumItemAt(enm, i).Value.Trim();
break;
}
}
lst.Add(new IDSearchData(name, id, type, exchange, industry, isin));
}
}
overall = lst.Count;
XParseElement paginationNode = XPath.GetElement("//div[@id=\"pagination\"]", doc);
if (paginationNode != null)
{
PaginationScanner scn = new PaginationScanner();
scn.SetPagination(paginationNode.Value, out pageingFrom, out pagingTo, out overall);
}
}
}
result = new IDSearchResult(lst.ToArray(), pageingFrom, pagingTo, overall, resultsCount);
#endregion
}
else if (settings is IDAlphabeticSearchDownloadSettings)
{
#region Alphabet
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement[] resultsNodes = XPath.GetElements("//results", doc);
if (resultsNodes.Length > 0)
{
XParseElement resultNode = resultsNodes[0];
foreach (XParseElement trNode in resultNode.Elements())
{
XParseElement[] enm = MyHelper.EnumToArray(trNode.Elements());
if (trNode.Name.LocalName == "tr" && enm.Length >= 2)
{
string name = string.Empty;
foreach (XParseElement subNode in enm[0].Elements())
{
name += subNode.Value.Trim() + " ";
}
name = name.TrimEnd();
string id = enm[1].Value.Trim();
lst.Add(new IDSearchData(name, id, "stock", "", null, null));
}
}
}
result = new IDSearchResult(lst.ToArray(), -1, -1, -1, new Dictionary <SecurityType, int>());
#endregion
}
}
if (result == null)
{
result = new IDSearchResult(lst.ToArray(), -1, -1, -1, new Dictionary <SecurityType, int>());
}
return(result);
}
protected override CompanyProfileResult ConvertResult(Base.ConnectionInfo connInfo, System.IO.Stream stream, Base.SettingsBase settings)
{
CompanyProfileData res = null;
System.Globalization.CultureInfo convCulture = new System.Globalization.CultureInfo("en-US");
if (stream != null)
{
XParseDocument doc = MyHelper.ParseXmlDocument(stream);
XParseElement resultNode = XPath.GetElement("//table[@id=\"yfncsumtab\"]/tr[2]", doc);
if (resultNode != null)
{
res = new CompanyProfileData();
res.SetID(FinanceHelper.CleanIndexID(((CompanyProfileDownloadSettings)settings).ID.ToUpper()));
XParseElement nameNode = XPath.GetElement("td[1]/b[1]", resultNode);
if (nameNode != null)
{
res.CompanyName = nameNode.Value;
}
XParseElement addressNode = XPath.GetElement("td[1]", resultNode);
if (addressNode != null)
{
System.Text.StringBuilder formattedAddress = new System.Text.StringBuilder();
try
{
string addNodeStr = addressNode.ToString();
if (addNodeStr != string.Empty)
{
addNodeStr = addNodeStr.Substring(addNodeStr.IndexOf("/>") + 2);
string[] rawAddress = addNodeStr.Substring(0, addNodeStr.IndexOf("Website")).Split(new string[] { "<b>", "<br />", "</b>", "\r", "\n", " - ", "</a>" }, StringSplitOptions.RemoveEmptyEntries);
if (rawAddress.Length >= 7)
{
foreach (string line in rawAddress)
{
string l = line.Trim();
if (l != string.Empty && !l.StartsWith("<a") && l != "Map")
{
formattedAddress.AppendLine(l);
}
}
}
}
}
catch (Exception ex)
{
}
res.Address = formattedAddress.ToString().TrimEnd();
}
XParseElement indicesNode = XPath.GetElement("td[1]/table[2]/tr/td/table/tr/td[2]", resultNode);
if (indicesNode != null)
{
List <KeyValuePair <string, string> > lstIndices = new List <KeyValuePair <string, string> >();
foreach (XParseElement indexLink in indicesNode.Elements())
{
if (indexLink.Name.LocalName == "a")
{
string indexID = Uri.UnescapeDataString(MyHelper.GetXmlAttributeValue(indexLink, "href").ToUpper().Replace("HTTP://FINANCE.YAHOO.COM/Q?S=", "").Replace("&D=T", ""));
string name = string.Empty;
foreach (string p in indexLink.Value.Split(new string[] { "\r\n" }, StringSplitOptions.None))
{
name += p.Trim() + " ";
}
lstIndices.Add(new KeyValuePair <string, string>(indexID, name.TrimEnd()));
}
}
res.Details.IndexMembership = lstIndices.ToArray();
}
XParseElement sectorsNode = XPath.GetElement("td[1]/table[2]/tr/td/table/tr[2]/td[2]", resultNode);
if (sectorsNode != null)
{
foreach (XParseElement sectorLink in sectorsNode.Elements())
{
if (sectorLink.Name.LocalName == "a")
{
foreach (Sector sect in Enum.GetValues(typeof(Sector)))
{
string name = string.Empty;
foreach (string p in sectorLink.Value.Split(new string[] { "\r\n" }, StringSplitOptions.None))
{
name += p.Trim() + " ";
}
name = name.TrimEnd();
if (sect.ToString().Replace("_", " ") == name)
{
res.Details.Sector = sect;
break;
}
}
}
}
}
XParseElement industryNode = XPath.GetElement("td[1]/table[2]/tr/td/table/tr[3]/td[2]", resultNode);
if (industryNode != null)
{
foreach (XParseElement industryLink in industryNode.Elements())
{
if (industryLink.Name.LocalName == "a")
{
int indIndex = 0;
if (int.TryParse(MyHelper.GetXmlAttributeValue(industryLink, "href").Replace("http://biz.yahoo.com/ic/", "").Replace(".html", ""), out indIndex))
{
res.Details.Industry = (Industry)indIndex;
}
}
}
}
XParseElement employeesNode = XPath.GetElement("td[1]/table[2]/tr/td/table/tr[4]/td[2]", resultNode);
if (employeesNode != null)
{
int fte;
if (int.TryParse(employeesNode.Value.Trim(), System.Globalization.NumberStyles.Any, convCulture, out fte))
{
res.Details.FullTimeEmployees = fte;
}
}
XParseElement summaryNode = XPath.GetElement("td[1]/p[1]", resultNode);
if (summaryNode != null)
{
System.Text.StringBuilder summaryText = new System.Text.StringBuilder();
foreach (string line in summaryNode.Value.Split(new string[] { "\r\n" }, StringSplitOptions.None))
{
summaryText.Append(line.Trim() + " ");
}
res.BusinessSummary = summaryText.ToString().TrimEnd();
}
XParseElement websitesNodes = XPath.GetElement("td[1]/table[5]/tr/td", resultNode);
if (websitesNodes != null)
{
List <Uri> lstWebsites = new List <Uri>();
foreach (XParseElement linkNode in websitesNodes.Elements())
{
if (linkNode.Name.LocalName == "a")
{
lstWebsites.Add(new Uri(MyHelper.GetXmlAttributeValue(linkNode, "href")));
}
}
res.CompanyWebsites = lstWebsites.ToArray();
}
XParseElement governanceNode = null;
XParseElement governanceHeader = XPath.GetElement("td[3]/table[1]/tr/th/span", resultNode);
if (governanceHeader != null && governanceHeader.Value.Contains("Governance"))
{
governanceNode = XPath.GetElement("td[3]/table[2]/tr/td", resultNode);
}
if (governanceNode != null)
{
System.Text.StringBuilder governanceText = new System.Text.StringBuilder();
foreach (string line in governanceNode.Value.Split(new string[] { "\r\n" }, StringSplitOptions.None))
{
governanceText.Append(line.Trim() + " ");
}
res.CorporateGovernance = governanceText.ToString().TrimEnd();
}
XParseElement executivesNode = null;
XParseElement executivesHeader = XPath.GetElement("td[3]/table[3]/tr/th/span", resultNode);
if (executivesHeader != null && executivesHeader.Value.Contains("Executives"))
{
executivesNode = XPath.GetElement("td[3]/table[4]/tr/td/table", resultNode);
}
else
{
executivesNode = XPath.GetElement("td[3]/table[2]/tr/td/table", resultNode);
}
if (executivesNode != null)
{
List <ExecutivePersonInfo> lst = new List <ExecutivePersonInfo>();
bool isFirst = true;
foreach (XParseElement row in executivesNode.Elements())
{
if (!isFirst)
{
if (row.Name.LocalName == "tr")
{
XParseElement[] enm = MyHelper.EnumToArray(row.Elements());
if (enm.Length >= 3)
{
ExecutivePersonInfo exec = new ExecutivePersonInfo();
string name = string.Empty;
foreach (string l in MyHelper.EnumToArray(enm[0].Elements())[0].Value.Split(new string[] { "\r\n" }, StringSplitOptions.None))
{
name += l.Trim() + " ";
}
exec.Name = name.TrimEnd();
string position = string.Empty;
var enm2 = MyHelper.EnumToArray(enm[0].Elements());
foreach (string l in enm2[enm2.Length - 1].Value.Split(new string[] { "\r\n" }, StringSplitOptions.None))
{
position += l.Trim() + " ";
}
exec.Position = position.Trim();
string payStr = enm[1].Value.Replace("\r\n", "").Trim();
if (!payStr.Contains("N/A"))
{
exec.Pay = FinanceHelper.GetMillionValue(payStr) * 1000000;
}
string exercisedStr = enm[2].Value.Replace("\r\n", "").Trim();
if (!exercisedStr.Contains("N/A"))
{
double d = FinanceHelper.GetMillionValue(exercisedStr);
exec.Exercised = (int)(d * 1000000);
}
lst.Add(exec);
}
}
}
else
{
isFirst = false;
}
}
res.KeyExecutives = lst.ToArray();
}
if (res.BusinessSummary.StartsWith("There is no "))
{
res = null;
}
}
}
return(new CompanyProfileResult(res));
}
