/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var stopwatch = Stopwatch.StartNew();
var symbolMapper = new CoinApiSymbolMapper();
var success = true;
var fileToProcess = _rawDataFolder.EnumerateFiles("*.gz")
.DistinctBy(
x =>
{
var parts = x.Name.Split('-').Take(2);
return(string.Join("-", parts));
}
);
Parallel.ForEach(fileToProcess, (file, loopState) =>
{
Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}...");
try
{
ProcessEntry(new CoinApiDataReader(symbolMapper), file);
}
catch (Exception e)
{
Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}");
success = false;
loopState.Break();
}
}
);
Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}");
return(success);
}
public void ReturnsCorrectBrokerageSymbol(string leanTicker, string market, string coinApiSymbolId)
{
var mapper = new CoinApiSymbolMapper();
var symbol = Symbol.Create(leanTicker, SecurityType.Crypto, market);
var symbolId = mapper.GetBrokerageSymbol(symbol);
Assert.AreEqual(coinApiSymbolId, symbolId);
}
public void ReturnsCorrectLeanSymbol(string coinApiSymbolId, string leanTicker, string market)
{
var mapper = new CoinApiSymbolMapper();
var symbol = mapper.GetLeanSymbol(coinApiSymbolId, SecurityType.Crypto, string.Empty);
Assert.AreEqual(leanTicker, symbol.Value);
Assert.AreEqual(SecurityType.Crypto, symbol.ID.SecurityType);
Assert.AreEqual(market, symbol.ID.Market);
}
public void ReturnsCorrectBrokerageSymbol()
{
var symbol = Symbol.Create("QTMUSD", SecurityType.Crypto, Market.Bitfinex);
var mapper = new CoinApiSymbolMapper();
var symbolId = mapper.GetBrokerageSymbol(symbol);
Assert.AreEqual("BITFINEX_SPOT_QTUM_USD", symbolId);
}
public void ReturnsCorrectLeanSymbol()
{
const string symbolId = "BITFINEX_SPOT_ANIO_USD";
var mapper = new CoinApiSymbolMapper();
var symbol = mapper.GetLeanSymbol(symbolId, SecurityType.Crypto, string.Empty);
Assert.AreEqual("NIOUSD", symbol.Value);
Assert.AreEqual(SecurityType.Crypto, symbol.ID.SecurityType);
Assert.AreEqual(Market.Bitfinex, symbol.ID.Market);
}
/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var stopwatch = Stopwatch.StartNew();
var symbolMapper = new CoinApiSymbolMapper();
var success = true;
// There were cases of files with with an extra suffix, following pattern:
// <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz
// Those cases should be ignored for SPOT prices.
var tradesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"trades",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));
var quotesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"quotes",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));
// Distinct by tick type and first two parts of the raw file name, separated by '-'.
// This prevents us from double processing the same ticker twice, in case we're given
// two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262
var apiDataReader = new CoinApiDataReader(symbolMapper);
var fileToProcess = tradesFolder.EnumerateFiles("*.gz")
.Concat(quotesFolder.EnumerateFiles("*.gz"))
.Where(f => f.Name.Contains("SPOT"))
.Where(f => f.Name.Split('_').Length == 4)
.DistinctBy(x => x.Directory.Parent.Name + apiDataReader.GetCoinApiEntryData(x, _processingDate).Symbol.ID);
Parallel.ForEach(fileToProcess, (file, loopState) =>
{
Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}...");
try
{
ProcessEntry(apiDataReader, file);
}
catch (Exception e)
{
Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}");
success = false;
loopState.Break();
}
}
);
Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}");
return(success);
}
/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var stopwatch = Stopwatch.StartNew();
var symbolMapper = new CoinApiSymbolMapper();
var success = true;
// There were cases of files with with an extra suffix, following pattern:
// <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz
// Those cases should be ignored for SPOT prices.
var tradesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"trades",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));
var quotesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"quotes",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));
var fileToProcess = tradesFolder.EnumerateFiles("*.gz")
.Concat(quotesFolder.EnumerateFiles("*.gz"))
.Where(f => f.Name.Contains("SPOT"))
.Where(f => f.Name.Split('_').Length == 4)
.DistinctBy(x => x.Directory.Parent.Name + x.Name.Split('-')[0]);
Parallel.ForEach(fileToProcess, (file, loopState) =>
{
Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}...");
try
{
ProcessEntry(new CoinApiDataReader(symbolMapper), file);
}
catch (Exception e)
{
Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}");
success = false;
loopState.Break();
}
}
);
Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}");
return(success);
}
/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var stopwatch = Stopwatch.StartNew();
var symbolMapper = new CoinApiSymbolMapper();
var success = true;
// There were cases of files with with an extra suffix, following pattern:
// <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz
// Those cases should be ignored for SPOT prices.
var fileToProcess = _rawDataFolder.EnumerateFiles("*.gz")
.Where(f => f.Name.Contains("SPOT"))
.Where(f => f.Name.Split('_').Length == 4)
.DistinctBy(
x =>
{
var parts = x.Name.Split('-').Take(2);
return(string.Join("-", parts));
}
);
Parallel.ForEach(fileToProcess, (file, loopState) =>
{
Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}...");
try
{
ProcessEntry(new CoinApiDataReader(symbolMapper), file);
}
catch (Exception e)
{
Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}");
success = false;
loopState.Break();
}
}
);
Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}");
return(success);
}
/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var stopwatch = Stopwatch.StartNew();
var symbolMapper = new CoinApiSymbolMapper();
var success = true;
// There were cases of files with with an extra suffix, following pattern:
// <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz
// Those cases should be ignored for SPOT prices.
var tradesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"trades",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));
var quotesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"quotes",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));
// Distinct by tick type and first two parts of the raw file name, separated by '-'.
// This prevents us from double processing the same ticker twice, in case we're given
// two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262
var apiDataReader = new CoinApiDataReader(symbolMapper);
var filesToProcessCandidates = tradesFolder.EnumerateFiles("*.gz")
.Concat(quotesFolder.EnumerateFiles("*.gz"))
.Where(f => f.Name.Contains("SPOT"))
.Where(f => f.Name.Split('_').Length == 4)
.ToList();
var filesToProcessKeys = new HashSet <string>();
var filesToProcess = new List <FileInfo>();
foreach (var candidate in filesToProcessCandidates)
{
try
{
var key = candidate.Directory.Parent.Name + apiDataReader.GetCoinApiEntryData(candidate, _processingDate).Symbol.ID;
if (filesToProcessKeys.Add(key))
{
// Separate list from HashSet to preserve ordering of viable candidates
filesToProcess.Add(candidate);
}
}
catch (Exception err)
{
// Most likely the exchange isn't supported. Log exception message to avoid excessive stack trace spamming in console output
Log.Error(err.Message);
}
}
Parallel.ForEach(filesToProcess, (file, loopState) =>
{
Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}...");
try
{
ProcessEntry(apiDataReader, file);
}
catch (Exception e)
{
Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}");
success = false;
loopState.Break();
}
}
);
Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}");
return(success);
}
