/// <summary> /// Method in charge of making all the steps to save the data. It makes the request, parses the data and saves it. /// This method takes into account the size limitation of the responses, slicing big request into smaller ones. /// </summary> /// <param name="symbol">The symbol.</param> /// <param name="resolution">The resolution.</param> /// <param name="startUtc">The start UTC.</param> /// <param name="endUtc">The end UTC.</param> public void Run(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc) { var data = new List <BaseData>(); var requestDayInterval = 0; var writer = new FxcmVolumeWriter(resolution, symbol, _dataDirectory); var intermediateStartDate = startUtc; var intermediateEndDate = endUtc; // As the responses has a Limit of 10000 lines, hourly data the minute data request should be sliced. if (resolution == Resolution.Minute && (endUtc - startUtc).TotalMinutes > 10000) { // Six days are 8640 minute observations, 7 days are 10080. requestDayInterval = 6; } else if (resolution == Resolution.Hour && (endUtc - startUtc).TotalHours > 10000) { // 410 days x 24 hr = 9840 hr. requestDayInterval = 410; } var counter = 0; do { if (requestDayInterval != 0) { if (counter++ != 0) { intermediateStartDate = intermediateEndDate.AddDays(value: 1); } intermediateEndDate = intermediateStartDate.AddDays(requestDayInterval); if (intermediateEndDate > endUtc) { intermediateEndDate = endUtc; } } data.AddRange(Get(symbol, resolution, intermediateStartDate, intermediateEndDate)); // For every 300k observations in memory, write it. if (resolution == Resolution.Minute && counter % 30 == 0) { writer.Write(data); data.Clear(); } } while (intermediateEndDate != endUtc); // Seems the data has som duplicate values! this makes the writer throw an error. var duplicates = data.GroupBy(x => x.Time) .Where(g => g.Count() > 1) .Select(g => g.Key) .ToList(); if (duplicates.Count != 0) { foreach (var duplicate in duplicates) { data.RemoveAll(o => o.Time == duplicate); } } writer.Write(data); }
/// <summary> /// Method in charge of making all the steps to save the data. It makes the request, parses the data and saves it. /// This method takes into account the size limitation of the responses, slicing big request into smaller ones. /// </summary> /// <param name="symbol">The symbol.</param> /// <param name="resolution">The resolution.</param> /// <param name="startUtc">The start UTC.</param> /// <param name="endUtc">The end UTC.</param> /// <param name="update">Flag to </param> public void Run(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc, bool update = false) { var data = new List <BaseData>(); var requestDayInterval = 0; var writer = new FxcmVolumeWriter(resolution, symbol, _dataDirectory); var intermediateStartDate = startUtc; var intermediateEndDate = endUtc; if (update) { var updatedStartDate = FxcmVolumeAuxiliaryMethods.GetLastAvailableDateOfData(symbol, resolution, writer.FolderPath); if (updatedStartDate == null) { return; } intermediateStartDate = ((DateTime)updatedStartDate).AddDays(value: -1); intermediateEndDate = DateTime.Today; } // As the responses has a Limit of 10000 lines, hourly data the minute data request should be sliced. if (resolution == Resolution.Minute && (endUtc - startUtc).TotalMinutes > 10000) { // Six days are 8640 minute observations, 7 days are 10080. requestDayInterval = 6; } else if (resolution == Resolution.Hour && (endUtc - startUtc).TotalHours > 10000) { // 410 days x 24 hr = 9840 hr. requestDayInterval = 410; } var counter = 0; do { if (requestDayInterval != 0) { if (counter++ != 0) { intermediateStartDate = intermediateEndDate.AddDays(value: 1); } intermediateEndDate = intermediateStartDate.AddDays(requestDayInterval); if (intermediateEndDate > endUtc) { intermediateEndDate = endUtc; } } data.AddRange(Get(symbol, resolution, intermediateStartDate, intermediateEndDate)); // For every 300k observations in memory, write it. if (resolution == Resolution.Minute && counter % 30 == 0) { writer.Write(data); data.Clear(); } } while (intermediateEndDate != endUtc); writer.Write(data, update); }