/// <summary>
/// Method in charge of making all the steps to save the data. It makes the request, parses the data and saves it.
/// This method takes into account the size limitation of the responses, slicing big request into smaller ones.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="resolution">The resolution.</param>
/// <param name="startUtc">The start UTC.</param>
/// <param name="endUtc">The end UTC.</param>
public void Run(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc)
{
var data = new List <BaseData>();
var requestDayInterval = 0;
var writer = new FxcmVolumeWriter(resolution, symbol, _dataDirectory);
var intermediateStartDate = startUtc;
var intermediateEndDate = endUtc;
// As the responses has a Limit of 10000 lines, hourly data the minute data request should be sliced.
if (resolution == Resolution.Minute && (endUtc - startUtc).TotalMinutes > 10000)
{
// Six days are 8640 minute observations, 7 days are 10080.
requestDayInterval = 6;
}
else if (resolution == Resolution.Hour && (endUtc - startUtc).TotalHours > 10000)
{
// 410 days x 24 hr = 9840 hr.
requestDayInterval = 410;
}
var counter = 0;
do
{
if (requestDayInterval != 0)
{
if (counter++ != 0)
{
intermediateStartDate = intermediateEndDate.AddDays(value: 1);
}
intermediateEndDate = intermediateStartDate.AddDays(requestDayInterval);
if (intermediateEndDate > endUtc)
{
intermediateEndDate = endUtc;
}
}
data.AddRange(Get(symbol, resolution, intermediateStartDate, intermediateEndDate));
// For every 300k observations in memory, write it.
if (resolution == Resolution.Minute && counter % 30 == 0)
{
writer.Write(data);
data.Clear();
}
} while (intermediateEndDate != endUtc);
// Seems the data has som duplicate values! this makes the writer throw an error.
var duplicates = data.GroupBy(x => x.Time)
.Where(g => g.Count() > 1)
.Select(g => g.Key)
.ToList();
if (duplicates.Count != 0)
{
foreach (var duplicate in duplicates)
{
data.RemoveAll(o => o.Time == duplicate);
}
}
writer.Write(data);
}
/// <summary>
/// Method in charge of making all the steps to save the data. It makes the request, parses the data and saves it.
/// This method takes into account the size limitation of the responses, slicing big request into smaller ones.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="resolution">The resolution.</param>
/// <param name="startUtc">The start UTC.</param>
/// <param name="endUtc">The end UTC.</param>
/// <param name="update">Flag to </param>
public void Run(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc, bool update = false)
{
var data = new List <BaseData>();
var requestDayInterval = 0;
var writer = new FxcmVolumeWriter(resolution, symbol, _dataDirectory);
var intermediateStartDate = startUtc;
var intermediateEndDate = endUtc;
if (update)
{
var updatedStartDate = FxcmVolumeAuxiliaryMethods.GetLastAvailableDateOfData(symbol, resolution, writer.FolderPath);
if (updatedStartDate == null)
{
return;
}
intermediateStartDate = ((DateTime)updatedStartDate).AddDays(value: -1);
intermediateEndDate = DateTime.Today;
}
// As the responses has a Limit of 10000 lines, hourly data the minute data request should be sliced.
if (resolution == Resolution.Minute && (endUtc - startUtc).TotalMinutes > 10000)
{
// Six days are 8640 minute observations, 7 days are 10080.
requestDayInterval = 6;
}
else if (resolution == Resolution.Hour && (endUtc - startUtc).TotalHours > 10000)
{
// 410 days x 24 hr = 9840 hr.
requestDayInterval = 410;
}
var counter = 0;
do
{
if (requestDayInterval != 0)
{
if (counter++ != 0)
{
intermediateStartDate = intermediateEndDate.AddDays(value: 1);
}
intermediateEndDate = intermediateStartDate.AddDays(requestDayInterval);
if (intermediateEndDate > endUtc)
{
intermediateEndDate = endUtc;
}
}
data.AddRange(Get(symbol, resolution, intermediateStartDate, intermediateEndDate));
// For every 300k observations in memory, write it.
if (resolution == Resolution.Minute && counter % 30 == 0)
{
writer.Write(data);
data.Clear();
}
} while (intermediateEndDate != endUtc);
writer.Write(data, update);
}
