public static void LoadData(data.baseDS.priceDataDataTable tbl, string timeScale, DateTime frDate, DateTime toDate, string stockCode)
{
priceDataTA.ClearBeforeFill = false;
if (timeScale == AppTypes.MainDataTimeScale.Code)
{
if (frDate == DateTime.MinValue && toDate == DateTime.MaxValue)
{
priceDataTA.FillByCode(tbl, stockCode);
}
else
{
priceDataTA.FillByDateCode(tbl, frDate, toDate, stockCode);
}
}
else
{
if (frDate == DateTime.MinValue && toDate == DateTime.MaxValue)
{
priceDataTA.FillByTypeCode(tbl, timeScale, stockCode);
}
else
{
priceDataTA.FillByTypeDateCode(tbl, timeScale, frDate, toDate, stockCode);
}
}
}
/// <summary>
/// Concatenate 2 table
/// </summary>
/// <param name="frTbl">Source table</param>
/// <param name="fromId">The index in source where the copy begins</param>
/// <param name="toTbl">Destination table</param>
public static void DataConcat(data.baseDS.priceDataDataTable frTbl, int fromIdx, data.baseDS.priceDataDataTable toTbl)
{
for (int idx = fromIdx; idx < frTbl.Count; idx++)
{
toTbl.ImportRow(frTbl[idx]);
}
}
public static bool ImportOHLCV_CSV(string csvFileName, string stockExchangeForNewCode,
data.baseDS.priceDataDataTable priceDataTbl,
libs.OnUpdatePriceData onUpdateDataFunc)
{
return(libs.ImportOHLCV_CSV(csvFileName, ',', common.dateTimeLibs.DateTimeFormats.YYMMDD,
stockExchangeForNewCode, libs.CultureInfoVN,
priceDataTbl, DoImportRow, onUpdateDataFunc, null));
}
// importPrice->priceData
private static void AddImportPrice(data.importDS.importPriceDataTable importPriceTbl,
DailyPrice dailyPrice, data.baseDS.priceDataDataTable priceDataTbl)
{
data.baseDS.priceDataSumRow dailyPriceRow;
data.baseDS.priceDataRow priceDataRow;
decimal volume = 0;
for (int idx = 0; idx < importPriceTbl.Count; idx++)
{
//Invalid price, ignore
if (importPriceTbl[idx].closePrice <= 0)
{
continue;
}
volume = importPriceTbl[idx].volume;
// If this is totail volume then minus the last volume to get the real volume in the period
if (importPriceTbl[idx].isTotalVolume)
{
dailyPriceRow = dailyPrice.GetData(importPriceTbl[idx]);
if (dailyPriceRow != null)
{
volume -= dailyPriceRow.volume;
}
if (volume <= 0)
{
continue;
}
}
priceDataRow = priceDataTbl.NewpriceDataRow();
commonClass.AppLibs.InitData(priceDataRow);
priceDataRow.onDate = importPriceTbl[idx].onDate;
priceDataRow.stockCode = importPriceTbl[idx].stockCode;
priceDataRow.openPrice = importPriceTbl[idx].openPrice;
priceDataRow.lowPrice = importPriceTbl[idx].lowPrice;
priceDataRow.highPrice = importPriceTbl[idx].highPrice;
priceDataRow.closePrice = importPriceTbl[idx].closePrice;
priceDataRow.volume = volume;
//Fix other invalid price
if (priceDataRow.highPrice <= 0)
{
priceDataRow.highPrice = priceDataRow.closePrice;
}
if (priceDataRow.lowPrice <= 0)
{
priceDataRow.highPrice = priceDataRow.lowPrice;
}
if (priceDataRow.openPrice <= 0)
{
priceDataRow.highPrice = priceDataRow.openPrice;
}
priceDataTbl.AddpriceDataRow(priceDataRow);
//Update the last row
dailyPrice.UpdateData(importPriceTbl[idx]);
}
}
public static data.baseDS.priceDataRow GetLastPriceData(string stockCode)
{
data.baseDS.priceDataDataTable priceTbl = priceDataTA.GetLastPrice(stockCode);
if (priceTbl.Count == 0)
{
return(null);
}
return(priceTbl[0]);
}
private static commonClass.DataBars GetData(data.baseDS.priceDataDataTable dataTbl, int startIdx)
{
commonClass.DataBars bars = new commonClass.DataBars();
for (int idx = startIdx; idx < dataTbl.Count; idx++)
{
bars.Add((double)dataTbl[idx].openPrice, (double)dataTbl[idx].highPrice,
(double)dataTbl[idx].lowPrice, (double)dataTbl[idx].closePrice,
(double)dataTbl[idx].volume, dataTbl[idx].onDate.ToOADate());
}
return(bars);
}
public static void ImportPrice_URL(DateTime updateTime, string url, string stockExchangeCode)
{
data.importDS.importPriceDataTable importPriceTbl = GetPriceFromURL(updateTime, url, stockExchangeCode);
libs.AddNewCode(importPriceTbl, null);
application.DbAccess.UpdateData(importPriceTbl);
data.baseDS.priceDataDataTable priceTbl = new data.baseDS.priceDataDataTable();
libs.AddImportPrice(importPriceTbl, priceTbl);
application.DbAccess.UpdateData(priceTbl);
// In VN culture : start of week is Monday (not Sunday)
libs.AggregatePriceData(priceTbl, libs.CultureInfoVN, null);
}
public static void ImportPrice_URL(DateTime updateTime, string url, string marketCode)
{
data.importDS.importPriceDataTable importPriceTbl = GetPrice(updateTime, url, marketCode);
libs.AddNewCode(importPriceTbl, null);
application.DbAccess.UpdateData(importPriceTbl);
data.baseDS.priceDataDataTable priceTbl = new data.baseDS.priceDataDataTable();
libs.AddImportPrice(importPriceTbl, priceTbl);
application.DbAccess.UpdateData(priceTbl);
// Gold use US culture
libs.AggregatePriceData(priceTbl, libs.CultureInfoUS, null);
}
public static void LoadData(data.baseDS.priceDataDataTable tbl, string timeScale, DateTime frDate, string stockCode)
{
priceDataTA.ClearBeforeFill = false;
if (timeScale == AppTypes.MainDataTimeScale.Code)
{
priceDataTA.FillByCodeFromDate(tbl, stockCode, frDate);
}
else
{
priceDataTA.FillByTypeCodeFromDate(tbl, timeScale, stockCode, frDate);
}
}
public static void AddNewCode(data.baseDS.priceDataDataTable tbl, string stockEchangeCode, OnAddData onAddstockCodeFunc)
{
data.baseDS.stockCodeDataTable stockCodeTbl = new data.baseDS.stockCodeDataTable();
for (int count = 0; count < tbl.Count; count++)
{
if (tbl[count].RowState == DataRowState.Deleted)
{
continue;
}
if (AddNewCode(tbl[count].stockCode, stockEchangeCode, stockCodeTbl) != null &&
onAddstockCodeFunc != null)
{
onAddstockCodeFunc(tbl[count].stockCode);
}
}
application.DbAccess.UpdateData(stockCodeTbl);
}
// Set [dataIsDailyPrice] = true if [priceTbl] contains all day data (not a real time data.
/// <summary>
/// Agrregate a data table to hourly,daily data...
/// </summary>
/// <param name="priceTbl">source data to be aggregated </param>
/// <param name="cultureCode"></param>
/// <param name="isDailyPrice">
/// Volume can be accumulated real-time or at the end of the day.
/// - If data is collected in realtime,
/// updateVolume table is used to culmulated the volume for each day and that will need some more resources.
/// - If data is collected at the end of the day, the voulume alredy is the total volume and updateVolume table
/// should not be used to save resources.
/// </param>
/// <param name="onAggregateDataFunc">function that was triggered after each agrregation</param>
public static void AggregatePriceData(data.baseDS.priceDataDataTable priceTbl, CultureInfo cultureInfo,
OnAggregateData onAggregateDataFunc)
{
data.baseDS.priceDataSumDataTable priceSumDataTbl = new data.baseDS.priceDataSumDataTable();
agrregateStat myAgrregateStat = new agrregateStat();
myAgrregateStat.maxCount = priceTbl.Count;
priceTbl.DefaultView.Sort = priceTbl.onDateColumn.ColumnName + "," + priceTbl.stockCodeColumn.ColumnName;
data.baseDS.priceDataRow priceDataRow;
decimal changeVolume;
int lastYear = int.MinValue;
for (int idx = 0; idx < priceTbl.DefaultView.Count; idx++)
{
priceDataRow = (data.baseDS.priceDataRow)priceTbl.DefaultView[idx].Row;
myAgrregateStat.count = idx;
if (onAggregateDataFunc != null)
{
onAggregateDataFunc(myAgrregateStat);
}
if (myAgrregateStat.cancel)
{
priceSumDataTbl.Clear();
break;
}
changeVolume = priceDataRow.volume;
foreach (AppTypes.TimeScale timeScale in AppTypes.myTimeScales)
{
if (timeScale.Type == AppTypes.TimeScaleTypes.RealTime)
{
continue;
}
AggregatePriceData(priceDataRow, changeVolume, timeScale, cultureInfo, priceSumDataTbl);
Application.DoEvents();
}
//Update and clear cache to speed up the performance
if (lastYear != priceDataRow.onDate.Year)
{
application.DbAccess.UpdateData(priceSumDataTbl);
priceSumDataTbl.Clear();
lastYear = priceDataRow.onDate.Year;
}
}
application.DbAccess.UpdateData(priceSumDataTbl);
}
private void MakeIndicatorData(common.myKeyValueItem[] indicatorCode,
data.baseDS.priceDataDataTable priceDataTbl,
data.baseDS.indicatorDataDataTable toTbl)
{
common.myKeyValueItem[] smaCodes = new common.myKeyValueItem[0];
common.myKeyValueItem[] macdCodes = new common.myKeyValueItem[0];
for (int idx = 0; idx < indicatorCode.Length; idx++)
{
if (indicatorCode[idx].Key.StartsWith("SMA"))
{
Array.Resize(ref smaCodes, smaCodes.Length + 1);
smaCodes[smaCodes.Length - 1] = new common.myKeyValueItem(indicatorCode[idx].Key, indicatorCode[idx].Value);
continue;
}
if (indicatorCode[idx].Key.StartsWith("MACD"))
{
Array.Resize(ref macdCodes, macdCodes.Length + 1);
macdCodes[macdCodes.Length - 1] = new common.myKeyValueItem(indicatorCode[idx].Key, indicatorCode[idx].Value);
continue;
}
}
}
//Updated data from the last read/update point
public static int UpdateAnalysisData(commonClass.BaseAnalysisData dataObj)
{
int lastDataIdx = dataObj.priceDataTbl.Count - 1;
DateTime lastDateTime;
if (lastDataIdx < 0)
{
lastDateTime = commonClass.Settings.sysStartDataDate;
}
else
{
lastDateTime = dataObj.priceDataTbl[lastDataIdx].onDate;
}
data.baseDS.priceDataDataTable tbl = GetData_ByTimeScale_Code_FrDate(dataObj.DataTimeScale.Code, dataObj.DataStockCode, lastDateTime);
if (tbl.Count > 0)
{
//Delete the last data because the updated data will include this one.
if (lastDataIdx >= 0)
{
dataObj.priceDataTbl[lastDataIdx].ItemArray = tbl[0].ItemArray;
commonClass.AppLibs.DataConcat(tbl, 1, dataObj.priceDataTbl);
}
else
{
commonClass.AppLibs.DataConcat(tbl, 0, dataObj.priceDataTbl);
}
}
//Update cache
AnalysisDataCache data = new AnalysisDataCache();
data.dataTbl = (data.baseDS.priceDataDataTable)dataObj.priceDataTbl.Copy();
data.firstData = dataObj.FirstDataStartAt;
AddCache(MakeAnalysisDataCacheKey(dataObj), data);
return(dataObj.priceDataTbl.Count - 1 - lastDataIdx);
}
private static commonClass.DataSeries GetData(data.baseDS.priceDataDataTable dataTbl, int startIdx, PriceDataType type)
{
commonClass.DataSeries ds = new commonClass.DataSeries();
switch (type)
{
case PriceDataType.High:
for (int i = startIdx, j = 0; i < dataTbl.Count; i++, j++)
{
if (dataTbl[i].RowState == System.Data.DataRowState.Deleted)
{
continue;
}
ds.Add((double)dataTbl[i].highPrice);
}
break;
case PriceDataType.Low:
for (int i = startIdx, j = 0; i < dataTbl.Count; i++, j++)
{
if (dataTbl[i].RowState == System.Data.DataRowState.Deleted)
{
continue;
}
ds.Add((double)dataTbl[i].lowPrice);
}
break;
case PriceDataType.Open:
for (int i = startIdx, j = 0; i < dataTbl.Count; i++, j++)
{
if (dataTbl[i].RowState == System.Data.DataRowState.Deleted)
{
continue;
}
ds.Add((double)dataTbl[i].openPrice);
}
break;
case PriceDataType.Close:
for (int i = startIdx, j = 0; i < dataTbl.Count; i++, j++)
{
if (dataTbl[i].RowState == System.Data.DataRowState.Deleted)
{
continue;
}
ds.Add((double)dataTbl[i].closePrice);
}
break;
case PriceDataType.Volume:
for (int i = startIdx, j = 0; i < dataTbl.Count; i++, j++)
{
if (dataTbl[i].RowState == System.Data.DataRowState.Deleted)
{
continue;
}
ds.Add((double)dataTbl[i].volume);
}
break;
case PriceDataType.DateTime:
for (int i = startIdx, j = 0; i < dataTbl.Count; i++, j++)
{
if (dataTbl[i].RowState == System.Data.DataRowState.Deleted)
{
continue;
}
ds.Add(dataTbl[i].onDate.ToOADate());
}
break;
default:
common.system.ThrowException("Invalid dataField in MakeDataList()"); break;
}
return(ds);
}
public static void UpdateData(data.baseDS.priceDataDataTable tbl)
{
priceDataTA.Update(tbl);
tbl.AcceptChanges();
}
public static void LoadDataOneRow(data.baseDS.priceDataDataTable tbl, DateTime frDate, DateTime toDate, string stockCode)
{
priceDataTA.ClearBeforeFill = false;
priceDataTA.FillOneByDateStockCode(tbl, frDate, toDate, stockCode);
}
//Updated data from the last read/update point
public static int UpdateAnalysisData(AnalysisData dataObj)
{
int lastDataIdx = dataObj.priceDataTbl.Count - 1;
DateTime lastDateTime;
if (lastDataIdx < 0) lastDateTime = commonClass.Settings.sysStartDataDate;
else lastDateTime = dataObj.priceDataTbl[lastDataIdx].onDate;
data.baseDS.priceDataDataTable tbl = new data.baseDS.priceDataDataTable();
DbAccess.LoadData(tbl, dataObj.DataTimeScale.Code, lastDateTime, dataObj.DataStockCode);
if (tbl.Count > 0)
{
//Delete the last data because the updated data will include this one.
if (lastDataIdx >= 0)
{
dataObj.priceDataTbl[lastDataIdx].ItemArray = tbl[0].ItemArray;
commonClass.AppLibs.DataConcat(tbl, 1, dataObj.priceDataTbl);
}
else commonClass.AppLibs.DataConcat(tbl, 0, dataObj.priceDataTbl);
}
return dataObj.priceDataTbl.Count - 1 - lastDataIdx;
}
public static void AddImportPrice(data.importDS.importPriceDataTable importPriceTbl,
data.baseDS.priceDataDataTable priceDataTbl)
{
AddImportPrice(importPriceTbl, myDailyPrice, priceDataTbl);
}
//Use the idea from http://www.codeproject.com/KB/database/CsvReader.aspx by Sebastien Lorion
public static bool ImportOHLCV_CSV(string csvFileName, char delimiter, common.dateTimeLibs.DateTimeFormats dataDateFormat,
string stockExchangeForNewCode, CultureInfo culture,
data.baseDS.priceDataDataTable priceDataTbl,
ImportRow ImportRowFunc,
OnUpdatePriceData onUpdateDataFunc,
OnEndImportPriceData onEndImportFunc)
{
libs.importStat myImportStat = new libs.importStat();
myImportStat.Reset();
myImportStat.dateFormat = dataDateFormat;
myImportStat.culture = culture;
data.baseDS.stockCodeDataTable stockCodeTbl = new data.baseDS.stockCodeDataTable();
data.baseDS.priceDataRow priceDataRow;
DataRowView[] foundRows;
application.DbAccess.LoadData(stockCodeTbl, AppTypes.CommonStatus.Enable);
DataView stockCodeView = new DataView(stockCodeTbl);
stockCodeView.Sort = stockCodeTbl.codeColumn.ColumnName;
bool fCanceled = false;
DateTime lastPriceDate = common.Consts.constNullDate;
importOHLCV data;
// open the file "data.csv" which is a CSV file with headers
using (CsvReader csv = new CsvReader(new StreamReader(csvFileName), true, delimiter))
{
// missing fields will not throw an exception,
// but will instead be treated as if there was a null value
csv.MissingFieldAction = MissingFieldAction.ReplaceByNull;
int fieldCount = csv.FieldCount;
if (fieldCount < 7)
{
return(false);
}
while (csv.ReadNextRecord())
{
Application.DoEvents();
myImportStat.dataCount++;
data = ImportRowFunc(csv, myImportStat);
if (myImportStat.cancel)
{
fCanceled = true; break;
}
if (data == null)
{
myImportStat.errorCount++;
continue;
}
//Assume that all price must be valid
if (data.Open <= 0 || data.High <= 0 || data.Low <= 0 || data.Close <= 0)
{
continue;
}
foundRows = stockCodeView.FindRows(data.code);
if (foundRows.Length == 0)
{
//Try to add new stock code
libs.AddNewCode(data.code, stockExchangeForNewCode, stockCodeTbl);
application.DbAccess.UpdateData(stockCodeTbl);
}
// Ignore all data that was in database
//if (!foundLastPriceDate)
//{
// lastPriceDate = libs.FindLastPriceDate(data.code);
// foundLastPriceDate = true;
//}
if (lastPriceDate != common.Consts.constNullDate && data.dateTime <= lastPriceDate)
{
continue;
}
if (priceDataTbl.FindBystockCodeonDate(data.code, data.dateTime) != null)
{
myImportStat.errorCount++;
continue;
}
myImportStat.updateCount++;
priceDataRow = priceDataTbl.NewpriceDataRow();
commonClass.AppLibs.InitData(priceDataRow);
priceDataRow.stockCode = data.code;
priceDataRow.onDate = data.dateTime;
//Try to fix some error in data
priceDataRow.openPrice = (decimal)data.Open;
priceDataRow.highPrice = (decimal)data.High;
priceDataRow.lowPrice = (decimal)data.Low;
priceDataRow.closePrice = (decimal)data.Close;
priceDataRow.volume = (decimal)data.Volume;
priceDataTbl.AddpriceDataRow(priceDataRow);
if (onUpdateDataFunc != null)
{
onUpdateDataFunc(priceDataRow, myImportStat);
}
}
}
if (fCanceled)
{
priceDataTbl.Clear();
return(false);
}
if (onEndImportFunc != null)
{
onEndImportFunc(priceDataTbl);
}
return(true);
}
public data.baseDS.priceDataDataTable GetData_ByTimeScale_Code_DateRange(string timeScaleCode, string stockCode, DateTime frDate, DateTime toDate)
{
data.baseDS.priceDataDataTable tbl = new data.baseDS.priceDataDataTable();
DbAccess.LoadData(tbl, timeScaleCode, frDate, toDate, stockCode);
return tbl;
}
public data.baseDS.priceDataDataTable GetData_ByTimeScale_Code_DateRange(string timeScaleCode, string stockCode, DateTime frDate, DateTime toDate)
{
data.baseDS.priceDataDataTable tbl = new data.baseDS.priceDataDataTable();
DbAccess.LoadData(tbl, timeScaleCode, frDate, toDate, stockCode);
return(tbl);
}
public static void LoadData(data.baseDS.priceDataDataTable tbl, string stockCode)
{
priceDataTA.ClearBeforeFill = false;
priceDataTA.FillByCode(tbl, stockCode);
}
public static void ImportPrice_URL(DateTime updateTime, string url, string stockExchangeCode)
{
data.importDS.importPriceDataTable importPriceTbl = GetPriceFromURL(updateTime, url, stockExchangeCode);
libs.AddNewCode(importPriceTbl, null);
application.DbAccess.UpdateData(importPriceTbl);
data.baseDS.priceDataDataTable priceTbl = new data.baseDS.priceDataDataTable();
libs.AddImportPrice(importPriceTbl, priceTbl);
application.DbAccess.UpdateData(priceTbl);
// In VN culture : start of week is Monday (not Sunday)
libs.AggregatePriceData(priceTbl, libs.CultureInfoVN, null);
}
private void DoAggregate(data.baseDS.priceDataDataTable tbl)
{
libs.AggregatePriceData(tbl, libs.CultureInfoVN, onAggregateData);
}
public static void ImportPrice_URL(DateTime updateTime, string url, string marketCode)
{
data.importDS.importPriceDataTable importPriceTbl = GetPrice(updateTime, url, marketCode);
libs.AddNewCode(importPriceTbl, null);
application.DbAccess.UpdateData(importPriceTbl);
data.baseDS.priceDataDataTable priceTbl = new data.baseDS.priceDataDataTable();
libs.AddImportPrice(importPriceTbl, priceTbl);
application.DbAccess.UpdateData(priceTbl);
// Gold use US culture
libs.AggregatePriceData(priceTbl, libs.CultureInfoUS, null);
}
