/// <summary>
/// Parse order book information for Kaiko data files
/// </summary>
/// <param name="symbol">The symbol being converted</param>
/// <param name="unzippedFile">The path to the unzipped file</param>
/// <returns>Lean quote ticks representing the Kaiko data</returns>
private static IEnumerable <Tick> ParseKaikoQuoteFile(Symbol symbol, string unzippedFile)
{
using (var sr = new StreamReader(unzippedFile))
{
var headerLine = sr.ReadLine();
var headerCsv = headerLine.ToCsv();
var typeColumn = headerCsv.FindIndex(x => x == "type");
var dateColumn = headerCsv.FindIndex(x => x == "date");
var priceColumn = headerCsv.FindIndex(x => x == "price");
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
long currentEpoch = 0;
var currentEpochTicks = new List <KaikoTick>();
while (sr.Peek() >= 0)
{
var line = sr.ReadLine();
if (line == null)
{
continue;
}
var lineParts = line.Split(',');
var tickEpoch = Convert.ToInt64(lineParts[dateColumn]);
var currentTick = new KaikoTick
{
TickType = TickType.Quote,
Time = Time.UnixMillisecondTimeStampToDateTime(tickEpoch),
Quantity = ParseQuantity(lineParts, quantityColumn),
Value = Convert.ToDecimal(lineParts[priceColumn]),
OrderDirection = lineParts[typeColumn]
};
if (currentEpoch != tickEpoch)
{
var quoteTick = CreateQuoteTick(symbol, Time.UnixMillisecondTimeStampToDateTime(currentEpoch), currentEpochTicks);
if (quoteTick != null)
{
yield return(quoteTick);
}
currentEpochTicks.Clear();
currentEpoch = tickEpoch;
}
currentEpochTicks.Add(currentTick);
}
}
}
/// <summary>
/// Parse order book information for Kaiko data files
/// </summary>
/// <param name="symbol">The symbol being converted</param>
/// <param name="unzippedFile">The path to the unzipped file</param>
/// <returns>Lean quote ticks representing the Kaiko data</returns>
private static IEnumerable <Tick> ParseKaikoQuoteFile(Symbol symbol, string unzippedFile)
{
using (var sr = new StreamReader(unzippedFile))
{
var headerLine = sr.ReadLine();
var headerCsv = headerLine.ToCsv();
var typeColumn = headerCsv.FindIndex(x => x == "type");
var dateColumn = headerCsv.FindIndex(x => x == "date");
var priceColumn = headerCsv.FindIndex(x => x == "price");
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
long currentEpoch = 0;
var currentEpochTicks = new List <KaikoTick>();
while (sr.Peek() >= 0)
{
var line = sr.ReadLine();
if (line == null)
{
continue;
}
var lineParts = line.Split(',');
var tickEpoch = Convert.ToInt64(lineParts[dateColumn]);
decimal quantity;
decimal price;
try
{
quantity = ParseScientificNotationToDecimal(lineParts, quantityColumn);
price = ParseScientificNotationToDecimal(lineParts, priceColumn);
}
catch (Exception ex)
{
Log.Error($"KaikoDataConverter.ParseKaikoQuoteFile(): Data corrupted in file {unzippedFile}. Line {string.Join(" ", lineParts)}, Exception {ex}");
continue;
}
var currentTick = new KaikoTick
{
TickType = TickType.Quote,
Time = Time.UnixMillisecondTimeStampToDateTime(tickEpoch),
Quantity = quantity,
Value = price,
OrderDirection = lineParts[typeColumn]
};
if (currentEpoch != tickEpoch)
{
var quoteTick = CreateQuoteTick(symbol, Time.UnixMillisecondTimeStampToDateTime(currentEpoch), currentEpochTicks);
if (quoteTick != null)
{
yield return(quoteTick);
}
currentEpochTicks.Clear();
currentEpoch = tickEpoch;
}
currentEpochTicks.Add(currentTick);
}
}
}
/// <summary>
/// Parse order book information for Kaiko data files
/// </summary>
/// <param name="rawDataLines">The raw data lines.</param>
/// <returns>
/// IEnumerable of ticks representing the Kaiko data
/// </returns>
private IEnumerable <Tick> ParseKaikoQuoteFile(IEnumerable <string> rawDataLines)
{
var headerLine = rawDataLines.First();
var headerCsv = headerLine.ToCsv();
var typeColumn = headerCsv.FindIndex(x => x == "type");
var dateColumn = headerCsv.FindIndex(x => x == "date");
var priceColumn = headerCsv.FindIndex(x => x == "price");
var quantityColumn = headerCsv.FindIndex(x => x == "amount");
long currentEpoch = 0;
var currentEpochTicks = new List <KaikoTick>();
foreach (var line in rawDataLines.Skip(1))
{
if (line == null || line == string.Empty)
{
continue;
}
var lineParts = line.Split(',');
var tickEpoch = Parse.Long(lineParts[dateColumn]);
decimal quantity;
decimal price;
try
{
quantity = ParseScientificNotationToDecimal(lineParts, quantityColumn);
price = ParseScientificNotationToDecimal(lineParts, priceColumn);
}
catch (Exception ex)
{
Log.Error($"KaikoDataConverter.ParseKaikoQuoteFile(): Raw data corrupted. Line {string.Join(" ", lineParts)}, Exception {ex}");
continue;
}
var currentTick = new KaikoTick
{
TickType = TickType.Quote,
Time = Time.UnixMillisecondTimeStampToDateTime(tickEpoch),
Quantity = quantity,
Value = price,
OrderDirection = lineParts[typeColumn]
};
if (currentEpoch != tickEpoch)
{
var quoteTick = CreateQuoteTick(Time.UnixMillisecondTimeStampToDateTime(currentEpoch), currentEpochTicks);
if (quoteTick != null)
{
yield return(quoteTick);
}
currentEpochTicks.Clear();
currentEpoch = tickEpoch;
}
currentEpochTicks.Add(currentTick);
}
}