static void save(StockSplitInfo[] infos, splitType filename)
{
var data = tryGet(a => true, filename);
var ex = infos.Except(data);
if (!ex.Any())
{
StockPriceManager.SetMessage(DataSource.KabuDotCom, "新規データが存在しなかったため書き込み処理をスキップしました。");
return;
}
if (!Directory.Exists(localPath))
{
Directory.CreateDirectory(localPath);
}
string tgtPath = localPath + filename.toFileName();
try {
using (FileStream f = new FileStream(tgtPath, FileMode.Create)) {
var seri = new XmlSerializer(typeof(StockSplitInfo[]));
seri.Serialize(f, data.Union(infos).ToArray());
StockPriceManager.SetMessage(DataSource.KabuDotCom, filename.toDisplayName() + "データを保存しました。");
}
} catch (Exception e) {
StockPriceManager.SetMessage(new ManagementMessage()
{
Sender = DataSource.KabuDotCom,
Signal = MessageSignal.Error,
Message = "書き込み処理に失敗しました。",
Detail = e.Message,
});
}
}
/// <summary>期間を指定して株価情報を取得する</summary>
static IEnumerable <SerializablePrices> _download(DateTime since, DateTime until, object ticker)
{
//string url = @"http://stocks.finance.yahoo.co.jp/stocks/history/?code=" + ticker.ToString() + spanFormat(since, until);
string url = spanFormat(ticker, since, until);
var data = Enumerable.Empty <SerializablePrices>();
var result = Enumerable.Empty <SerializablePrices>();
int page = 1;
do
{
data = _download(url + page.ToString());
//result = result.Union(data);
page++;
} while (isContinue(since, until, data, ref result) /*data.Max(a => a.Date) < until*/);
StockPriceManager.SetMessage(DataSource.Yahoo, since.ToShortDateString() + " -> " + until.ToShortDateString() + "ダウンロード完了");
if (ticker is int)
{
foreach (var d in result)
{
d.TickerSymbol = (int)ticker;
yield return(d);
}
}
else if (ticker is FXPair)
{
foreach (var d in result)
{
d.SymbolName = ticker.ToString();
yield return(d);
}
}
}
static void add(DateTime date, IEnumerable <SerializablePrices> table, KdbData type)
{
if (!table.Any())
{
return;
}
createDirectoryIfNotFound(date, type);
string targetPath = createFileName(date, type);
try {
var seri = new XmlSerializer(typeof(SerializablePrices[]));
using (FileStream fs = new FileStream(targetPath, FileMode.Create)) {
seri.Serialize(fs, table.ToArray());
}
StockPriceManager.SetMessage(
DataSource.Kdb,
type.ToString() + ", " + date.ToString("yyyy-MM-dd") + ", 書き込み完了"
);
} catch (Exception e) {
StockPriceManager.SetMessage(new ManagementMessage()
{
Sender = DataSource.Kdb,
Signal = MessageSignal.Error,
Message = type.ToString() + " " + date.ToString("yyyy-MM-dd") + " のデータの書き込みに失敗しました。",
Detail = e.Message,
});
}
}
/// <summary>データを保存する。</summary>
static void save(IEnumerable <SerializablePrices> src, yahooData type, object ticker)
{
string fileName = getFilePath(type, ticker);
createDirectoryIfNotFound(type);
var seri = new XmlSerializer(typeof(SerializablePrices[]));
using (FileStream fs = new FileStream(fileName, FileMode.Create)) {
seri.Serialize(fs, src.ToArray());
}
StockPriceManager.SetMessage(DataSource.Yahoo, "保存完了");
}
internal static void Update()
{
foreach (KdbData en in Enum.GetValues(typeof(KdbData)))
{
IEnumerable <Tuple <DateTime, IEnumerable <SerializablePrices> > > srcs
= Enumerable.Empty <Tuple <DateTime, IEnumerable <SerializablePrices> > >();
var today = DateTime.Today.AddDays(-1);
try {
var sdate = getExistingDataDate(en);
if (getExistingDataDate(en).Any())
{
var localMin = getExistingDataDate(en).Min();
var localMax = getExistingDataDate(en).Max();
if (sinceMin < localMin)
{
srcs = srcs.Union(download(localMin.AddDays(-1), sinceMin, en));
}
if (localMax < today)
{
srcs = srcs.Union(download(localMax.AddDays(1), today, en));
}
}
else
{
srcs = download(sinceMin, today, en);
}
foreach (var kvp in srcs)
{
add(kvp.Item1, kvp.Item2.ToArray(), en);
}
} catch (WebException we) {
StockPriceManager.SetMessage(
new ManagementMessage()
{
Sender = DataSource.Kdb,
Signal = MessageSignal.Error,
Message = en.ToString() + " 接続エラー発生",
Detail = we.Message,
});
} catch (CsvHelperException ce) {
StockPriceManager.SetMessage(
new ManagementMessage()
{
Sender = DataSource.Kdb,
Signal = MessageSignal.Error,
Message = en.ToString() + " 解析中にエラー発生",
Detail = ce.Message,
});
}
}
}
/// <summary>ローカルに保存されているデータを取得する。</summary>
static IEnumerable <SerializablePrices> getLocalData(yahooData type, object ticker)
{
string fileName = getFilePath(type, ticker);
if (!File.Exists(fileName))
{
return(new SerializablePrices[0]);
}
var deseri = new XmlSerializer(typeof(SerializablePrices[]));
using (FileStream fs = new FileStream(fileName, FileMode.Open)) {
SerializablePrices[] src;
src = deseri.Deserialize(fs) as SerializablePrices[];
StockPriceManager.SetMessage(DataSource.Yahoo, "読み込み完了");
return(src);
}
}
static IEnumerable <Tuple <DateTime, IEnumerable <SerializablePrices> > > download(DateTime start, DateTime end, KdbData type)
{
Func <DateTime, DateTime, DateTime> nxt = (c, e) => {
return((c < e) ? c.AddDays(1) : (c > e) ? c.AddDays(-1) : c);
};
DateTime current = start;
do
{
if (current.DayOfWeek != DayOfWeek.Saturday && current.DayOfWeek != DayOfWeek.Sunday)
{
Thread.Sleep(2000);
var s = _download(current, type);
StockPriceManager.SetMessage(
DataSource.Kdb,
type.ToString() + ", " + current.ToString("yyyy-MM-dd") + ", ダウンロード完了"
);
if (!string.IsNullOrEmpty(s))
{
using (var str = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(s))))
using (var csv = new CsvReader(str)) {
switch (type)
{
case KdbData.stocks:
csv.Configuration.RegisterClassMap <StockCsvMap>();
break;
case KdbData.indices:
csv.Configuration.RegisterClassMap <IndexCsvMap>();
break;
}
var rec = csv.GetRecords <SerializablePrices>().ToArray();
foreach (var r in rec)
{
r.Date = current;
}
yield return(new Tuple <DateTime, IEnumerable <SerializablePrices> >(current, rec));
}
}
Thread.Sleep(3000);
}
current = nxt(current, end);
} while (current != end);
}
static IEnumerable <StockSplitInfo> _download(splitType type)
{
string str = "";
try {
str = getSource(type.getUrl()).Result;
StockPriceManager.SetMessage(DataSource.KabuDotCom, type.toDisplayName() + "データのダウンロード完了");
} catch (Exception e) {
StockPriceManager.SetMessage(new ManagementMessage()
{
Sender = DataSource.KabuDotCom,
Signal = MessageSignal.Error,
Message = "接続エラー",
Detail = e.Message,
});
return(new StockSplitInfo[0]);
}
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(str);
try {
var dd = doc.DocumentNode
//.SelectNodes(@"table[@class=""tbl01""]");
.SelectNodes(@"//table[@class='tbl01']")
.Descendants("tr")
.Select(a => new { Prm = a.Elements("td").Select(e => e.InnerText) })
//.Select(a => ResultWithValue.Of<IEnumerable<string>, StockSplitInfo>(tryParse, a.Prm))
//.Where(a => a.Result)
//.Select(a => a.Value);
.Where(a => a.Prm.Any())
.Select(a => parse(a.Prm));
StockPriceManager.SetMessage(DataSource.KabuDotCom, type.toDisplayName() + "データの解析完了");
return(dd);
}catch (Exception e) {
StockPriceManager.SetMessage(new ManagementMessage()
{
Sender = DataSource.KabuDotCom,
Signal = MessageSignal.Error,
Message = "解析エラー",
Detail = e.Message,
});
return(new StockSplitInfo[0]);
}
}
static SerializablePrices[] tryGet(DateTime date, KdbData type)
{
if (!getExistingDataDate(type).Any(a => a == date))
{
return(new SerializablePrices[0]);
}
try {
var deseri = new XmlSerializer(typeof(SerializablePrices[]));
using (FileStream fs = new FileStream(createFileName(date, type), FileMode.Open)) {
return(deseri.Deserialize(fs) as SerializablePrices[]);
}
}catch (Exception e) {
StockPriceManager.SetMessage(new ManagementMessage()
{
Sender = DataSource.Kdb,
Signal = MessageSignal.Error,
Message = type.ToString() + " " + date.ToString("yyyy-MM-dd") + " のデータの読み込みに失敗しました。",
Detail = e.Message
});
return(new SerializablePrices[0]);
}
}
/// <summary>更新または再取得が必要だった場合はダウンロードを実行する。</summary>
/// <returns>更新または再取得した場合は true</returns>
static bool download(DateTime since, DateTime until, object ticker, ref IEnumerable <SerializablePrices> src)
{
since = since.Date;
until = until.Date;
if (!src.Any() || isWebChanged(ticker, src))
{
src = _download(since, until, ticker);
StockPriceManager.SetMessage(DataSource.Yahoo, "新規ダウンロードしました。");
return(true);
}
var srcMx = src.Max(a => a.Date);
var srcMn = src.Min(a => a.Date);
if (srcMn <= since && until <= srcMx)
{
return(false);
}
IEnumerable <SerializablePrices> dldata = Enumerable.Empty <SerializablePrices>();
if (since < srcMn)
{
dldata = dldata.Union(_download(since, srcMn.AddDays(-1), ticker));
}
if (srcMx < until)
{
dldata = dldata.Union(_download(srcMx.AddDays(1), until, ticker));
}
var ald = src.Union(dldata);
if (ald.Except(src).Any())
{
src = ald;
StockPriceManager.SetMessage(DataSource.Yahoo, "追加ダウンロードしました。");
return(true);
}
return(false);
}