/// <summary> /// Method in charge of making all the steps to save the data. It makes the request, parses the data and saves it. /// This method takes into account the size limitation of the responses, slicing big request into smaller ones. /// </summary> /// <param name="symbol">The symbol.</param> /// <param name="resolution">The resolution.</param> /// <param name="startUtc">The start UTC.</param> /// <param name="endUtc">The end UTC.</param> /// <param name="update">Flag to </param> public void Run(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc, bool update = false) { var data = new List <BaseData>(); var requestDayInterval = 0; var writer = new FxcmVolumeWriter(resolution, symbol, _dataDirectory); var intermediateStartDate = startUtc; var intermediateEndDate = endUtc; if (update) { var updatedStartDate = FxcmVolumeAuxiliaryMethods.GetLastAvailableDateOfData(symbol, resolution, writer.FolderPath); if (updatedStartDate == null) { return; } intermediateStartDate = ((DateTime)updatedStartDate).AddDays(value: -1); intermediateEndDate = DateTime.Today; } // As the responses has a Limit of 10000 lines, hourly data the minute data request should be sliced. if (resolution == Resolution.Minute && (endUtc - startUtc).TotalMinutes > 10000) { // Six days are 8640 minute observations, 7 days are 10080. requestDayInterval = 6; } else if (resolution == Resolution.Hour && (endUtc - startUtc).TotalHours > 10000) { // 410 days x 24 hr = 9840 hr. requestDayInterval = 410; } var counter = 0; do { if (requestDayInterval != 0) { if (counter++ != 0) { intermediateStartDate = intermediateEndDate.AddDays(value: 1); } intermediateEndDate = intermediateStartDate.AddDays(requestDayInterval); if (intermediateEndDate > endUtc) { intermediateEndDate = endUtc; } } data.AddRange(Get(symbol, resolution, intermediateStartDate, intermediateEndDate)); // For every 300k observations in memory, write it. if (resolution == Resolution.Minute && counter % 30 == 0) { writer.Write(data); data.Clear(); } } while (intermediateEndDate != endUtc); writer.Write(data, update); }
private IEnumerable <BaseData> ReadActualDataAndAppendNewData(IEnumerable <BaseData> data) { // Read the actual data var zipFilePath = Path.Combine(FolderPath, _symbol.Value.ToLowerInvariant() + "_volume.zip"); var actualData = FxcmVolumeAuxiliaryMethods.GetFxcmVolumeFromZip(zipFilePath); return(actualData.Concat(data)); }