public override DataLoadResult Load()
{
DataLoadResult result = DataLoadResult.NothingLoaded;
if (amigaMusicOk || amigaSoundOk || amigaGraphicsOk)
{
var amigaResult = amiga.Load();
if (!amigaResult.HasFlag(DataLoadResult.GraphicsLoaded))
{
amigaGraphicsOk = false;
}
if (!amigaResult.HasFlag(DataLoadResult.MusicLoaded))
{
amigaMusicOk = false;
}
if (!amigaResult.HasFlag(DataLoadResult.SoundLoaded))
{
amigaSoundOk = false;
}
result |= amigaResult;
}
if (dosOk)
{
var dosResult = dos.Load();
dosOk = dosResult != DataLoadResult.NothingLoaded;
result |= dosResult;
}
return(result);
}
public async Task <DataLoadResult> PerformDataLoad()
{
try
{
// Step 0: Prepare data load response object
DataLoadResult result = new DataLoadResult();
int accountsProcessed = 0;
int positionsProcessed = 0;
// Step 1: Fetch all accounts in SQL Server Accounts table
var accounts = _context.Accounts.ToList();
foreach (Accounts account in accounts)
{
// Step 2: Iterate through each account and grab positions
var positionsByAccount = _context.Positions.Where(x => x.AccountId == account.Id).ToList();
// Step 3: Generate performance report
var reportObject =
await _performanceReportCalculator.GeneratePortfolioPerformanceReport(account,
positionsByAccount, DataRequestType.RequestType.HistoricalMarketDataRequest);
var performanceReport = reportObject;
// Step 4: Insert performance data into Mongo DB
var performanceReportAsBson = performanceReport.ToBsonDocument();
await _mongoCollection.InsertOneAsync(performanceReportAsBson);
accountsProcessed++;
}
// Step 5: Return the data load results
result.AccountsProcessed = accountsProcessed;
if (accountsProcessed > 0)
{
result.Status = "OK";
}
else
{
result.Status = "PARTIAL LOAD";
}
return(result);
}
catch (Exception e)
{
DataLoadResult result = new DataLoadResult()
{
AccountsProcessed = 0,
Status = "FAILED"
};
return(result);
}
}
private static void ImportData(MarketSeriesBase series, DataLoadResult result)
{
//UpdateProgress(0.97, "Loading data into memory");
if (series.TimeFrame == "t1")
{
var tickSeries = (MarketTickSeries)series;
tickSeries.Add(result.Ticks, result.StartDate, result.EndDate);
}
else
{
var barSeries = (MarketSeries)series;
barSeries.Add(result.Bars, result.StartDate, result.EndDate);
}
series.RaiseLoadHistoricalDataCompleted(result.StartDate, result.EndDate);
//Debug.WriteLine($"[{series} - data imported] {result} with {result.Count} items: now has {series.Count} items total");
}
public static DataLoadResult LoadData()
{
DataLoadResult result = new DataLoadResult();
string dataFilePath = Path.Combine(Application.persistentDataPath, "playerSave.dat");
if (File.Exists(dataFilePath))
{
BinaryFormatter formatter = new BinaryFormatter();
using (FileStream stream = File.Open(dataFilePath, FileMode.Open))
{
result.Data = (Data)formatter.Deserialize(stream);
result.IsDataLoaded = true;
}
}
return(result);
}
public static async Task <IEnumerable <DataLoadResult> > GetData(this IHistoricalDataProvider provider, MarketSeriesBase series, DateTime?startDate, DateTime endDate, bool cacheOnly = false, bool writeCache = true, TimeSpan?maxOutOfDate = null, int totalDesiredBars = 0, bool forceRetrieve = false, bool forceReretrieveEmptyData = false, CancellationToken?cancellationToken = null)
{
logger.LogTrace($"[GetData] {series} {startDate} - {endDate}");
await series.DataLock.WaitAsync().ConfigureAwait(false);
try
{
if (!startDate.HasValue)
{
startDate = endDate;
}
if (!maxOutOfDate.HasValue)
{
maxOutOfDate = HistoricalDataUtils.GetDefaultMaxOutOfDate(series);
}
List <DataLoadResult> results = new List <DataLoadResult>();
DateTime nextChunkDate;
if (series.HasAccount && series.Account.IsBacktesting)
{
nextChunkDate = DateTime.FromBinary(Math.Min(series.Account.BacktestEndDate.ToBinary(), series.OpenTime.First().ToBinary()));
}
else
{
if (cacheOnly)
{
nextChunkDate = endDate;
if (nextChunkDate == default(DateTime))
{
nextChunkDate = startDate.Value;
}
}
else
{
nextChunkDate = DateTime.UtcNow + TimeSpan.FromMinutes(1);
}
}
DataLoadResult dataLoadResult = null;
int giveUpAfterNoData = GetGiveUpAfterNoData(series.TimeFrame);
do
{
if (nextChunkDate == default(DateTime))
{
break;
}
DateTime chunkStart;
DateTime chunkEnd;
HistoricalDataCacheFile.GetChunkRange(series.TimeFrame, nextChunkDate, out chunkStart, out chunkEnd);
dataLoadResult = await provider.GetDataForChunk(series, chunkStart, chunkEnd, cacheOnly, writeCache, forceReretrieveEmptyData : forceReretrieveEmptyData, cancellationToken : cancellationToken).ConfigureAwait(false);
results.Add(dataLoadResult);
nextChunkDate = chunkStart - TimeSpan.FromMinutes(1);
if (!dataLoadResult.IsAvailable)
{
giveUpAfterNoData--;
}
else
{
giveUpAfterNoData = GetGiveUpAfterNoData(series.TimeFrame);
}
//Debug.WriteLine($"[data {series}] {dataLoadResult}" + (cacheOnly?" (CACHE ONLY)":""));
} while (giveUpAfterNoData > 0 &&
((nextChunkDate != default(DateTime) && nextChunkDate > startDate.Value) ||
series.Count < totalDesiredBars)
);
return(results);
}
finally
{
series.DataLock.Release();
}
}
/// <summary>
/// Load data from cache if available, or else retrieve from source.
/// </summary>
/// <param name="provider"></param>
/// <param name="series"></param>
/// <param name="chunkStart"></param>
/// <param name="chunkEnd"></param>
/// <param name="cacheOnly"></param>
/// <param name="writeCache">If true, write any retrieved data to cache</param>
/// <param name="maxOutOfDate"></param>
/// <param name="forceRetrieve"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <DataLoadResult> GetDataForChunk(this IHistoricalDataProvider provider, MarketSeriesBase series, DateTime chunkStart, DateTime chunkEnd, bool cacheOnly = false, bool writeCache = true, TimeSpan?maxOutOfDate = null, bool forceRetrieve = false, bool forceReretrieveEmptyData = false, CancellationToken?cancellationToken = null)
{
logger.LogTrace($"[GetDataForChunk] {series} {chunkStart}");
if (!maxOutOfDate.HasValue)
{
maxOutOfDate = HistoricalDataUtils.GetDefaultMaxOutOfDate(series);
}
if (chunkStart >= series.DataStartDate && chunkEnd <= series.DataEndDate)
{
var chunkEndPastLast = chunkEnd - series.OpenTime.LastValue;
if (chunkEndPastLast <= TimeSpan.Zero)
{
//logger.LogTrace($"[{series}] Already contains data for {chunkStart} - {chunkEnd}");
return(DataLoadResult.AlreadyLoaded);
}
var nowPastLast = DateTime.UtcNow - series.OpenTime.LastValue;
if (!forceRetrieve && nowPastLast < maxOutOfDate)
{
//logger.LogTrace($"[{series}] Already contains data for {chunkStart} - {chunkEnd} (Only slightly out of date: {nowPastLast})");
return(DataLoadResult.AlreadyLoaded);
}
}
var cacheFile = await HistoricalDataCacheFile.GetCacheFile(series, chunkStart).ConfigureAwait(false);
DataLoadResult result = cacheFile.DataLoadResult;
bool isUpToDateAfterClose = series.CloseTime.HasValue && cacheFile.QueryDate > series.CloseTime.Value;
if (!forceRetrieve && (!forceReretrieveEmptyData || !cacheFile.EmptyData) && cacheFile.IsPersisted && (isUpToDateAfterClose || cacheFile.OutOfDateTimeSpan < maxOutOfDate.Value)
//&& (chunkEnd > DateTime.UtcNow || !result.IsPartial)
)
{
//Debug.WriteLine($"[cache available {series}] Chunk {chunkStart}");
if (!cacheOnly)
{
if (result.Bars == null)
{
result.Bars = new List <TimedBar>();
}
ImportData(series, result);
//if (series is MarketSeries) // TOC#7 OLD
//{
// var s = series as MarketSeries;
// if (s != null) { s.Add(cacheFile.Bars, result.StartDate, result.EndDate); }
//}
//else if (series is MarketSeries) // TOC#7
//{
// var s = series as MarketTickSeries;
// if (s != null) { s.Add(cacheFile.Ticks, result.StartDate, result.EndDate); }
//}
}
}
else
{
var str = forceRetrieve ? "FORCE RETRIEVE" : (forceReretrieveEmptyData && cacheFile.EmptyData) ? "RERETRIEVE EMPTY DATA" : "CACHE MISS";
logger.LogInformation($"[{series} {str}] Retrieving chunk {chunkStart} ");
#if DEBUG
logger.LogTrace($"<<<cache debug>>> cacheFile.IsPersisted {cacheFile.IsPersisted}, isUpToDateAfterClose {isUpToDateAfterClose}, cacheFile.OutOfDateTimeSpan < maxOutOfDate.Value {cacheFile.OutOfDateTimeSpan < maxOutOfDate.Value} ");
#endif
var loader = new object();
try
{
series.OnLoadingStarted(loader);
result = await provider.RetrieveDataForChunk(series, chunkStart, cacheOnly, writeCache, cancellationToken : cancellationToken).ConfigureAwait(false);
if (result.Faulted)
{
throw new Exception("Retrieve data for chunk failed");
}
//if (LinkWithAccountData && Account != null && ResultBars.Count > 0)
if (!cacheOnly)
{
ImportData(series, result);
}
}
finally
{
series.OnLoadingFinished(loader);
}
if (writeCache)
{
if (!result.Faulted)
{
//UpdateProgress(0.98, "Writing data into cache"); // FUTURE: Async?
TaskManager.OnFireAndForget(HistoricalDataCacheFile.SaveCacheFile(result), name: $"Save cache for {series.SymbolCode} from {chunkStart} - {chunkEnd}");
}
}
else
{
logger.LogTrace("WRITECACHE OFF");
}
}
return(result);
}
public async Task <DataLoadResult> RetrieveDataForChunk(MarketSeriesBase series, DateTime date, bool cacheOnly = false, bool writeCache = true, TimeSpan?maxOutOfDate = null, CancellationToken?token = null)
{
var result = new DataLoadResult(series);
DateTime chunkStart;
DateTime chunkEnd;
HistoricalDataCacheFile.GetChunkRange(series.TimeFrame, date, out chunkStart, out chunkEnd);
result.QueryDate = DateTime.UtcNow;
var key = $"{series.ToString()};{date.ToString()};{cacheOnly};{writeCache};{maxOutOfDate}";
bool created = false;
tryagain:
var job = jobs.GetOrAdd(key, k =>
{
created = true;
return(new SpotwareDataJob(series)
{
StartTime = chunkStart,
EndTime = chunkEnd,
WriteCache = writeCache,
LinkWithAccountData = !cacheOnly,
CancellationToken = token,
});
});
if (!created)
{
if (DateTime.UtcNow - job.CreateDate > TimeSpan.FromMinutes(1))
{
if (jobs.TryRemove(key, out job))
{
goto tryagain;
}
}
else
{
await job.Wait().ConfigureAwait(false);
}
}
await job.Start().ConfigureAwait(false);
await job.Wait().ConfigureAwait(false);
if (job.Faulted)
{
result.Faulted = true;
return(result);
}
if (job.ResultCount > 0)
{
result.IsAvailable = true;
}
result.Bars = job.ResultBars;
result.Ticks = job.ResultTicks;
result.StartDate = chunkStart;
result.EndDate = chunkEnd;
if (result.QueryDate < chunkEnd)
{
result.IsPartial = true;
}
return(result);
}
