public Stock GetStock(StockName stockName, DateTime startDate, DateTime endDate)
{
string dir = String.Format(@"..\..\StockData\Yahoo");
string filename = String.Format("{0}.stock", stockName);
var fullPath = Path.Combine(dir, filename);
List <IStockEntry> rates;
if (!File.Exists(fullPath))
{
rates = GetStockFromRemote(stockName, startDate, endDate);
Directory.CreateDirectory(dir);
using (Stream stream = File.Open(fullPath, FileMode.Create))
{
var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
bformatter.Serialize(stream, rates);
}
}
else
{
using (Stream stream = File.Open(fullPath, FileMode.Open))
{
var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
rates = (List <IStockEntry>)bformatter.Deserialize(stream);
}
}
var stock = new Stock(stockName, rates);
return(stock);
}
public Stock GetStock(StockName stockName, DateTime startDate, DateTime endDate)
{
string dir = String.Format(@"..\..\StockData\Maya");
string filename = String.Format("{0}.csv", stockName);
var fullPath = Path.Combine(dir, filename);
var rates = new List<IStockEntry>();
var parser = new TextFieldParser(fullPath) {TextFieldType = FieldType.Delimited};
parser.SetDelimiters(",");
//skips the first 3 lines
parser.ReadFields();
parser.ReadFields();
parser.ReadFields();
while (!parser.EndOfData)
{
var fields = parser.ReadFields();
if (fields != null)
{
StockEntry stockEntry = null;
rates.Add(stockEntry);
}
}
rates.Reverse();
var stock = new Stock(stockName, rates);
return stock;
}
public Stock GetStock(StockName stockName, DateTime startDate, DateTime endDate)
{
string dir = String.Format(@"..\..\StockData\Yahoo");
string filename = String.Format("{0}.stock", stockName);
var fullPath = Path.Combine(dir, filename);
List<IStockEntry> rates;
if (!File.Exists(fullPath))
{
rates = GetStockFromRemote(stockName, startDate, endDate);
Directory.CreateDirectory(dir);
using (Stream stream = File.Open(fullPath, FileMode.Create))
{
var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
bformatter.Serialize(stream, rates);
}
}
else
{
using (Stream stream = File.Open(fullPath, FileMode.Open))
{
var bformatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
rates = (List<IStockEntry>) bformatter.Deserialize(stream);
}
}
var stock = new Stock(stockName, rates);
return stock;
}
public Stock GetStock(StockName name)
{
if (_stockHolder.ContainsKey(name))
{
return(_stockHolder[name]);
}
throw new Exception(String.Format("Stock {0} does not exist", name));
}
public Stock GetStock(StockName name)
{
if (_stockHolder.ContainsKey(name))
{
return _stockHolder[name];
}
throw new Exception(String.Format("Stock {0} does not exist", name));
}
static IEnumerable <string> LoadCodeOfStocks(string codeFile)
{
var codes = File.ReadAllLines(codeFile)
.Where(s => !string.IsNullOrWhiteSpace(s))
.Select(s => StockName.NormalizeCode(s))
.OrderBy(s => s)
.ToArray();
return(codes);
}
private void listView1_SelectedIndexChanged_1(object sender, EventArgs e)
{
if (this.listView1.SelectedItems.Count == 0)
{
return;
}
StockName.Text = this.listView1.SelectedItems[0].SubItems[0].Text;
StockName.Refresh();
// listView1.Columns.Add("Column4", 100);
}
public StockBoardMetricsManager(
IEvaluationContext context,
Func <IEnumerable <ITradingObject>, IGroupRuntimeMetric> groupMetricCreator)
{
if (context == null || context.GroupMetricManager == null || groupMetricCreator == null)
{
throw new ArgumentNullException();
}
_context = context;
// create and register metric for blocks
var allTradingObjects = context.GetAllTradingObjects();
var metricPerBoard = Enumerable.Range(0, (int)StockBoard.All + 1)
.Select(board =>
{
var tradingObjects = allTradingObjects
.Where(
o =>
{
if (o.Object != null && o.Object is StockName)
{
StockName stockName = (StockName)o.Object;
return((board & (int)stockName.Board) != 0);
}
return(false);
})
.ToArray();
if (tradingObjects.Length == 0)
{
return(null);
}
else
{
return(groupMetricCreator(tradingObjects));
}
})
.ToArray();
_boardMetricIndices = metricPerBoard.Select(m => m == null ? -1 : context.GroupMetricManager.RegisterMetric(m)).ToArray();
// register observer
_context.GroupMetricManager.RegisterAfterUpdatedMetricsObserver(this);
}
public bool FetchReport(StockName stock, string outputFile, out string errorMessage)
{
if (stock == null)
{
throw new ArgumentNullException("stock");
}
if (string.IsNullOrWhiteSpace(outputFile))
{
throw new ArgumentNullException("outputFile");
}
var address = ReportServerAddressFormatter.Format(FinanceReportServerAddress, stock, ReportServerAddressFormatter.DefaultAbbrevationMarketFormatter);
return(FetchReport(address, outputFile, out errorMessage));
}
public StandardEvaluationContext(
ITradingDataProvider provider,
EquityManager equityManager,
ILogger logger,
TradingSettings settings = null,
StreamWriter dumpDataWriter = null,
StockBlockRelationshipManager relationshipManager = null)
{
if (equityManager == null || provider == null || logger == null)
{
throw new ArgumentNullException();
}
_provider = provider;
_equityManager = equityManager;
_logger = logger;
_settings = settings;
_relationshipManager = relationshipManager;
var metricManager = new StandardRuntimeMetricManager(_provider.GetAllTradingObjects().Length);
var groupMetricManager = new StandardGroupRuntimeMetricManager(metricManager);
// register the group metric manager as observer of metric manager.
metricManager.RegisterAfterUpdatedMetricsObserver(groupMetricManager);
_metricManager = metricManager;
_groupMetricManager = groupMetricManager;
_boardIndexTradingObjects = new Dictionary <string, ITradingObject>();
var boards = new StockBoard[]
{
StockBoard.GrowingBoard,
StockBoard.MainBoard,
StockBoard.SmallMiddleBoard
};
foreach (var board in boards)
{
string boardIndex = StockName.GetBoardIndex(board);
ITradingObject tradingObject = GetTradingObject(boardIndex);
_boardIndexTradingObjects.Add(boardIndex, tradingObject);
}
_dumper = dumpDataWriter == null ? null : new StreamDataDumper(dumpDataWriter, 8, 3, _settings.DumpMetrics, this, _provider);
}
static StockHistoryData LoadInputFile(string file, DateTime startDate, DateTime endDate)
{
if (string.IsNullOrEmpty(file))
{
throw new ArgumentNullException();
}
Csv inputData = Csv.Load(file, Encoding.UTF8, ",");
if (inputData.RowCount == 0)
{
return(null);
}
string code = inputData[0][0];
StockName name = new StockName(code);
// header is code,date,open,highest,lowest,close,volume,amount
List <Bar> data = new List <Bar>(inputData.RowCount);
foreach (var row in inputData.Rows)
{
DateTime date = DateTime.Parse(row[1]);
if (date < startDate || date > endDate)
{
continue;
}
Bar dailyData = new Bar();
dailyData.Time = DateTime.Parse(row[1]);
dailyData.OpenPrice = double.Parse(row[2]);
dailyData.HighestPrice = double.Parse(row[3]);
dailyData.LowestPrice = double.Parse(row[4]);
dailyData.ClosePrice = double.Parse(row[5]);
dailyData.Volume = double.Parse(row[6]);
dailyData.Amount = double.Parse(row[7]);
if (dailyData.Volume != 0.0)
{
data.Add(dailyData);
}
}
return(new StockHistoryData(name, 86400L, data));
}
private void ScrapePage(string pageName)
{
Console.WriteLine("Doing page " + pageName);
string url = @"http://www.nyse.com/about/listed/lc_ny_name_`.js".Replace("`", pageName);
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document = AgentHandler.Instance.PerformAction(session, action);
string[] segments = document.ResponseString.Split(']');
foreach (string segment in segments)
{
int startIdx = segment.IndexOf("\"") + 1;
if (startIdx == 0)
{
continue;
}
int endIdx = segment.IndexOf("\"", startIdx);
string stockTicker = segment.Substring(startIdx, endIdx - startIdx);
startIdx = segment.IndexOf("\"", endIdx + 1) + 1;
endIdx = segment.IndexOf("\"", startIdx);
string companyName = segment.Substring(startIdx, endIdx - startIdx);
startIdx = segment.IndexOf("\"", endIdx + 1) + 1;
endIdx = segment.IndexOf("\"", startIdx);
string country = segment.Substring(startIdx, endIdx - startIdx);
;
StockName name = new StockName
{
Stock = stockTicker,
Company_Name = companyName,
Country = country,
Exchange = "NYSE"
};
if (!Madness.StockNames.Any(x => x.Stock == stockTicker))
{
Console.WriteLine("Found new stock: " + stockTicker);
Madness.AddToStockNames(name);
}
}
}
private static string GetNormalizedCode(string code)
{
StockName name = new StockName(code);
string prefix = string.Empty;
if (name.Market == StockExchangeMarket.ShangHai)
{
prefix = "SH";
}
else if (name.Market == StockExchangeMarket.ShengZhen)
{
prefix = "SZ";
}
return(prefix + name.Code);
}
private static string GetNormalizedCode(string code)
{
var name = StockName.Parse(code);
var prefix = string.Empty;
if (name.ExchangeId == StockExchangeId.ShanghaiExchange)
{
prefix = "SH";
}
else if (name.ExchangeId == StockExchangeId.ShenzhenExchange)
{
prefix = "SZ";
}
return(prefix + name.Code);
}
public IGroupRuntimeMetric GetMetricForTradingObject(ITradingObject tradingObject)
{
if (tradingObject == null)
{
throw new ArgumentNullException();
}
if (tradingObject.Object != null && tradingObject.Object is StockName)
{
StockName stockName = (StockName)tradingObject.Object;
return(GetMetricForBoard(stockName.Board));
}
else
{
return(GetMetricForBoard(StockBoard.All));
}
}
public ActionResult Index()
{
var stoctVlue = _IRelativeProfitService.GetStockNameList();
List <StockName> RadioItemList = new List <StockName>();
foreach (var item in stoctVlue)
{
StockName stockName = new StockName();
stockName.StockCode = item.StockCode;
stockName.Name = item.Name;
RadioItemList.Add(stockName);
}
StockVM stoctVM = new StockVM();
stoctVM.RadioItemList = RadioItemList;
return(View(stoctVM));
}
/// <summary>
/// Determine which exchange a given security can be traded in
/// </summary>
/// <param name="code">the code of security to be checked</param>
/// <returns>true if the security can be exchanged, otherwise false</returns>
public static Exchange GetTradeableExchangeForSecurity(string code)
{
if (string.IsNullOrWhiteSpace(code))
{
throw new ArgumentNullException();
}
switch (StockName.GetExchangeId(code))
{
case StockExchangeId.ShanghaiExchange:
return(Exchange.ShanghaiExchange);
case StockExchangeId.ShenzhenExchange:
return(Exchange.ShenzhenExchange);
default:
throw new ArgumentException(string.Format("unsupported code {0}", code));
}
}
private List <IStockEntry> GetStockFromRemote(StockName stockName, DateTime startDate, DateTime endData)
{
var c = startDate.Year;
var a = startDate.Month - 1;
var b = startDate.Day;
var f = endData.Year;
var d = endData.Month - 1;
var e = endData.Day;
var entries = new List <IStockEntry>();
using (var web = new WebClient())
{
string downloadString =
string.Format("http://ichart.finance.yahoo.com/table.csv?s={0}&a={1}&b={2}&c={3}&d={4}&e={5}&f={6}",
_stockFactory.ConvertNameToSymbol(stockName), a, b, c, d, e, f);
string data = web.DownloadString(downloadString);
data = data.Replace("r", "");
string[] rows = data.Split('\n');
//First row is headers so Ignore it
for (int i = 1; i < rows.Length; i++)
{
if (rows[i].Replace("\n", "").Trim() == "")
{
continue;
}
string[] cols = rows[i].Split(',');
var se = new StockEntry(Convert.ToDateTime(cols[0]), Convert.ToDouble(cols[1]),
Convert.ToDouble(cols[2]), Convert.ToDouble(cols[3]), Convert.ToDouble(cols[4]),
Convert.ToDouble(cols[5]), Convert.ToDouble(cols[6]));
entries.Add(se);
}
entries.Reverse();
return(entries);
}
}
public ChinaStock(
int index,
StockName stockName,
int volumePerHand = 100,
int volumePerBuyingUnit = 100,
int volumePerSellingUnit = 1,
double minPriceUnit = 0.01,
double limitUpRatio = 0.1,
double limitDownRatio = 0.1)
{
Index = index;
Code = stockName.Code;
Name = stockName.Names.Last();
Object = stockName;
VolumePerHand = volumePerHand;
VolumePerBuyingUnit = volumePerBuyingUnit;
VolumePerSellingUnit = volumePerSellingUnit;
MinPriceUnit = minPriceUnit;
LimitUpRatio = limitUpRatio;
LimitDownRatio = limitDownRatio;
}
public static string Format(string format, StockName stock, Func <StockExchangeMarket, string> marketFormatter = null)
{
if (format == null)
{
throw new ArgumentNullException("format");
}
if (stock == null)
{
throw new ArgumentNullException("stock");
}
// replace %code%
format = format.Replace("%code%", stock.Code);
// replace %market%
string market = marketFormatter == null?stock.Market.ToString() : marketFormatter(stock.Market);
format = format.Replace("%market%", market);
return(format);
}
private List<IStockEntry> GetStockFromRemote(StockName stockName, DateTime startDate, DateTime endData)
{
var c = startDate.Year;
var a = startDate.Month - 1;
var b = startDate.Day;
var f = endData.Year;
var d = endData.Month - 1;
var e = endData.Day;
var entries = new List<IStockEntry>();
using (var web = new WebClient())
{
string downloadString =
string.Format("http://ichart.finance.yahoo.com/table.csv?s={0}&a={1}&b={2}&c={3}&d={4}&e={5}&f={6}",
_stockFactory.ConvertNameToSymbol(stockName), a, b, c, d, e, f);
string data = web.DownloadString(downloadString);
data = data.Replace("r", "");
string[] rows = data.Split('\n');
//First row is headers so Ignore it
for (int i = 1; i < rows.Length; i++)
{
if (rows[i].Replace("\n", "").Trim() == "") continue;
string[] cols = rows[i].Split(',');
var se = new StockEntry(Convert.ToDateTime(cols[0]), Convert.ToDouble(cols[1]),
Convert.ToDouble(cols[2]), Convert.ToDouble(cols[3]), Convert.ToDouble(cols[4]),
Convert.ToDouble(cols[5]), Convert.ToDouble(cols[6]));
entries.Add(se);
}
entries.Reverse();
return entries;
}
}
public Stock GetStock(StockName stockName, DateTime startDate, DateTime endDate)
{
string dir = String.Format(@"..\..\StockData\Maya");
string filename = String.Format("{0}.csv", stockName);
var fullPath = Path.Combine(dir, filename);
var rates = new List <IStockEntry>();
var parser = new TextFieldParser(fullPath)
{
TextFieldType = FieldType.Delimited
};
parser.SetDelimiters(",");
//skips the first 3 lines
parser.ReadFields();
parser.ReadFields();
parser.ReadFields();
while (!parser.EndOfData)
{
var fields = parser.ReadFields();
if (fields != null)
{
StockEntry stockEntry = null;
rates.Add(stockEntry);
}
}
rates.Reverse();
var stock = new Stock(stockName, rates);
return(stock);
}
public Stock(StockName name, IList <IStockEntry> rates)
{
Name = name;
Rates = rates;
}
static int Run(Options options)
{
if (string.IsNullOrEmpty(options.OutputFileFolder))
{
Console.WriteLine("output file folder is empty");
return(-2);
}
var folder = Path.GetFullPath(options.OutputFileFolder);
// try to create output file folder if it does not exist
if (!Directory.Exists(folder))
{
try
{
Directory.CreateDirectory(folder);
}
catch (Exception ex)
{
Console.WriteLine("Create output file folder {0} failed. Exception: \n{1}", folder, ex);
return(-3);
}
}
StockNameTable table;
if (!string.IsNullOrEmpty(options.InputFile))
{
// single input file
StockName name = ProcessOneFile(options.InputFile, options.StartDate, options.EndDate, folder);
table = new StockNameTable();
if (name != null)
{
table.AddStock(name);
}
}
else
{
table = ProcessListOfFiles(options.InputFileList, options.StartDate, options.EndDate, folder);
}
if (!string.IsNullOrEmpty(options.NameFile))
{
Console.WriteLine();
Console.WriteLine("Output name file: {0}", options.NameFile);
File.WriteAllLines(
options.NameFile,
table.StockNames.Select(sn => sn.ToString()).ToArray(),
Encoding.UTF8);
}
if (!string.IsNullOrEmpty(options.CodeFile))
{
Console.WriteLine();
Console.WriteLine("Output code file: {0}", options.CodeFile);
File.WriteAllLines(
options.CodeFile,
table.StockNames.Select(sn => sn.Code).ToArray(),
Encoding.UTF8);
}
Console.WriteLine("Done.");
return(0);
}
public string ConvertNameToSymbol(StockName name)
{
return(_stockNameToSymbolMap[name]);
}
private static bool FetchReport(string serverAddress, StockName stock, string outputFile, out string errorMessage)
{
errorMessage = string.Empty;
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(serverAddress);
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode != HttpStatusCode.OK)
{
errorMessage = string.Format("HTTP Status {0} for {1}", response.StatusCode, serverAddress);
return(false);
}
string body = null;
string charset = response.CharacterSet;
Encoding encoding = string.IsNullOrWhiteSpace(charset) ? Encoding.UTF8 : Encoding.GetEncoding(charset);
// save the response body to memory stream to avoid send web request twice if
// the character set specified in HTTP header is different with what specified in body.
using (MemoryStream bodyStream = new MemoryStream())
{
response.GetResponseStream().CopyTo(bodyStream);
bodyStream.Seek(0, SeekOrigin.Begin);
// do not use "using" or close the reader to avoid the memory stream being closed.
StreamReader sr = new StreamReader(bodyStream, encoding);
body = sr.ReadToEnd();
// Check real charset meta-tag in HTML
const string meta = "charset=";
int charsetStart = body.IndexOf(meta);
if (charsetStart > 0)
{
charsetStart += meta.Length;
int charsetEnd = body.IndexOfAny(new[] { ' ', '\"', ';' }, charsetStart);
string realCharset = body.Substring(charsetStart, charsetEnd - charsetStart);
// real charset meta-tag in HTML differs from supplied server header???
if (realCharset != response.CharacterSet)
{
// get correct encoding
encoding = Encoding.GetEncoding(realCharset);
// reset stream position to beginning
bodyStream.Seek(0, SeekOrigin.Begin);
// reread response stream with the correct encoding
sr = new StreamReader(bodyStream, encoding);
body = sr.ReadToEnd();
}
}
}
using (StreamWriter writer = new StreamWriter(outputFile, false, Encoding.UTF8))
{
writer.Write(body);
}
}
}
catch (UriFormatException ex)
{
errorMessage = ex.ToString();
return(false);
}
catch (WebException ex)
{
errorMessage = ex.ToString();
return(false);
}
catch (IOException ex)
{
errorMessage = ex.ToString();
return(false);
}
return(true);
}
public string ConvertNameToSymbol(StockName name)
{
return _stockNameToSymbolMap[name];
}
private static void GrabStats(string symbol)
{
Console.Write("Grabbing Key Stats for " + symbol);
string url = @"http://finance.yahoo.com/q/ks?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
var doc = document.ResponseString;
var extractor = new ProgressiveDataExtractor(doc, "Valuation Measures");
if (extractor.NotFound)
{
Console.WriteLine(" - Not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyStat stats = new CompanyStat
{
market_cap = extractor.ExtractLong("Market Cap"),
enterprise_value = extractor.ExtractLong("Enterprise Value"),
trailing_pe = extractor.ExtractDecimal("Trailing P/E"),
forward_pe = extractor.ExtractDecimal("Forward P/E"),
peg_ratio = extractor.ExtractDecimal("PEG Ratio"),
price_sales = extractor.ExtractDecimal("Price/Sales"),
price_book = extractor.ExtractDecimal("Price/Book"),
ent_value_rev = extractor.ExtractLong("Enterprise Value/Revenue"),
ent_value_ebitda = extractor.ExtractDecimal("Enterprise Value/EBITDA"),
fiscal_year_ends = extractor.ExtractString("Fiscal Year Ends"),
most_recent_quarter = extractor.ExtractString("Most Recent Quarter"),
profit_margin = extractor.ExtractDecimal("Profit Margin"),
operating_margin = extractor.ExtractDecimal("Operating Margin"),
return_on_assets = extractor.ExtractDecimal("Return on Assets"),
return_on_equity = extractor.ExtractDecimal("Return on Equity"),
revenue = extractor.ExtractLong("Revenue"),
revenue_per_share = extractor.ExtractDecimal("Revenue Per Share"),
qrt_rev_growth = extractor.ExtractDecimal("Qtrly Revenue Growth"),
gross_profit = extractor.ExtractLong("Gross Profit"),
ebitda = extractor.ExtractLong("EBITDA"),
net_income_a_c = extractor.ExtractLong("Net Income Avl to Common"),
diluted_eps = extractor.ExtractDecimal("Diluted EPS"),
qrt_earnings_growth = extractor.ExtractDecimal("Qtrly Earnings Growth"),
total_cash = extractor.ExtractLong("Total Cash"),
cash_per_share = extractor.ExtractDecimal("Total Cash Per Share"),
total_debt = extractor.ExtractLong("Total Debt"),
total_debt_equit = extractor.ExtractDecimal("Total Debt/Equity"),
current_ratio = extractor.ExtractDecimal("Current Ratio"),
book_value_p_share = extractor.ExtractDecimal("Book Value Per Share"),
operating_cash_flow = extractor.ExtractLong("Operating Cash Flow"),
levered_free_cash_flow = extractor.ExtractLong("Levered Free Cash Flow"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
madness.AddToCompanyStats(stats);
madness.SaveChanges();
}
public static void PullQuotes(DateTime start, DateTime stop, string stockName)
{
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First();
Console.WriteLine("Working on " + stock.Stock);
//http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv
string url = @"http://ichart.finance.yahoo.com/table.csv?s=" +
stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year +
"&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year +
"&g=d&ignore=.csv";
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
bool found = false;
AgentDocument document = null;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
found = true;
}
catch (Exception ex)
{
Console.WriteLine("ERROR: " + url + " - " + ex.Message);
}
if (!found)
{
return;
}
//Date,Open,High,Low,Close,Volume,Adj Close
string[] rows = document.ResponseString.Split('\n');
if (rows.Length < 2)
{
return;
}
for (int i = 1; i < rows.Length; i++)
{
string row = rows[i];
string[] fields = row.Split(',');
if (fields.Length < 7)
{
Console.WriteLine((i - 2) + " records stored");
continue;
}
quote q = new quote
{
day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]),
open = decimal.Parse(fields[1]),
high = decimal.Parse(fields[2]),
low = decimal.Parse(fields[3]),
close = decimal.Parse(fields[4]),
volume = long.Parse(fields[5]),
adjusted_close = decimal.Parse(fields[6]),
StockName = stock
};
madness.AddToquotes(q);
}
madness.SaveChanges();
}
public string BuildActualDataFilePathAndName(string code)
{
return(Path.Combine(
StockDataFileDirectory,
StockDataFileNamePattern.Replace(StockCodePattern, StockName.UnnormalizeCode(code))));
}
private static void GrabProfile(string symbol)
{
Console.Write("Grabbing Profile for " + symbol);
string url = @"http://finance.yahoo.com/q/pr?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
string doc = document.ResponseString.ToLower();
var extractor = new ProgressiveDataExtractor(doc, ">details<");
if (extractor.NotFound)
{
Console.WriteLine(" - not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
string sector = extractor.ExtractString("sector");
Sector dbSector = null;
Industry dbIndustry = null;
if (sector != null && sector != "n/a")
{
sector = sector.Replace("&", "&");
string industry = extractor.ExtractString("industry").Replace("&", "&");
var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray();
if (obj.Length == 1)
{
dbSector = obj[0];
}
else
{
dbSector = new Sector {
Sector1 = sector
};
madness.AddToSectors(dbSector);
}
var obj2 = (from q in madness.Industries
where q.Sector.Sector1 == sector && q.Iindustry == industry
select q).ToArray();
if (obj2.Length == 1)
{
dbIndustry = obj2[0];
}
else
{
dbIndustry = new Industry {
Iindustry = industry, Sector = dbSector
};
madness.AddToIndustries(dbIndustry);
}
}
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyProfile profile = new CompanyProfile
{
Sector = dbSector,
Industry = dbIndustry,
num_employees = extractor.ExtractInt("full time employees"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
profile.summary = extractor.ExtractString("business summary");
profile.cgq = extractor.ExtractDecimal("corporate governance", "<b");
extractor.ExtractString("key executives");
int totalAge = 0;
int numAges = 0;
long totalPay = 0;
int numPays = 0;
int? curAge;
long?curPay;
do
{
curAge = extractor.ExtractInt("yfnc_tabledata1", "</b");
if (curAge > 111)
{
curAge = null;
}
if (curAge != null)
{
numAges++;
totalAge += (int)curAge;
curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap");
if (curPay != null)
{
numPays++;
totalPay += (long)curPay;
}
}
} while (curAge != null);
profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges;
profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays;
madness.AddToCompanyProfiles(profile);
madness.SaveChanges();
}
public ITradingObject GetBoardIndexTradingObject(StockBoard board)
{
return(_boardIndexTradingObjects[StockName.GetBoardIndex(board)]);
}
static StockName ProcessOneFile(string file, DateTime startDate, DateTime endDate, string outputFileFolder)
{
if (string.IsNullOrEmpty(file) || string.IsNullOrEmpty(outputFileFolder))
{
throw new ArgumentNullException();
}
var lines = File.ReadAllLines(file, Encoding.GetEncoding("GB2312"));
// in general the file contains at least 3 lines, 2 lines of header and at least 1 line of data.
if (lines.Length <= 2)
{
Console.WriteLine("Input {0} contains less than 3 lines, ignore it", file);
return(null);
}
// first line contains the stock code, name and '日线'
var fields = lines[0].Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (fields.Length < 3)
{
Console.WriteLine("Invalid first line in file {0}", file);
return(null);
}
var code = fields[0];
var name = string.Concat(fields.Skip(1).Take(fields.Length - 2).ToArray());
var stockName = new StockName(code, name);
var fullDataFile = Path.Combine(outputFileFolder, code + ".day.csv");
var deltaDataFile = Path.Combine(outputFileFolder, code + ".day.delta.csv");
var generateDeltaFile = File.Exists(fullDataFile);
var outputFile = generateDeltaFile ? deltaDataFile : fullDataFile;
using (var outputter = new StreamWriter(outputFile, false, Encoding.UTF8))
{
const string header = "code,date,open,highest,lowest,close,volume,amount";
const int indexOfVolume = 6;
outputter.WriteLine(header);
fields = header.Split(new[] { ',' });
var expectedFieldCount = fields.Length - 1; // remove the first column 'code' which does not exists in input file
for (var i = 2; i < lines.Length - 1; ++i)
{
lines[i] = lines[i].Trim();
fields = lines[i].Split(new[] { ',' });
if (fields.Length == expectedFieldCount)
{
// the first field is date
DateTime date;
if (!DateTime.TryParse(fields[0], out date))
{
continue;
}
if (date < startDate || date > endDate)
{
continue;
}
int volume;
if (int.TryParse(fields[indexOfVolume], out volume))
{
if (volume == 0)
{
continue;
}
}
outputter.WriteLine("{0},{1}", stockName.Code, lines[i]);
}
}
}
if (generateDeltaFile)
{
MergeFile(fullDataFile, deltaDataFile);
}
return(stockName);
}
