public static List <MarketTick> GetMarketTicks(string marketName, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List <MarketTick> result = new List <MarketTick>();
try
{
Int64 startTime = (Int64)Math.Ceiling(DateTime.UtcNow.AddHours(-systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours).Subtract(Constants.Epoch).TotalSeconds);
string baseUrl = "https://poloniex.com/public?command=returnChartData&period=300&start=" + startTime.ToString() + "&end=9999999999¤cyPair=" + marketName;
log.DoLogDebug("Poloniex - Getting ticks for '" + marketName + "'...");
List <dynamic> jsonObject = GetSimpleJsonListFromURL(baseUrl, log);
if (jsonObject.Count > 0)
{
log.DoLogDebug("Poloniex - " + jsonObject.Count.ToString() + " ticks received.");
foreach (var marketTick in jsonObject)
{
MarketTick tick = new MarketTick();
tick.Price = (double)marketTick["close"];
tick.Time = Constants.Epoch.AddSeconds((int)marketTick["date"]);
result.Add(tick);
}
}
}
catch (Exception ex)
{
log.DoLogCritical(ex.Message, ex);
}
return(result);
}
/// <summary>
/// Convert tick data to Level 1 Quote
/// </summary>
private static Quote ToQuote(MarketTick tick)
{
var volume = tick.AskSize + tick.BidSize;
return(new Quote
{
AskPrice = tick.AskPrice,
BidPrice = tick.BidPrice,
AskSize = tick.AskSize,
BidSize = tick.BidSize,
Volume = volume,
Level2 = new List <MarketTick>(),
Time = tick.Time
});
}
public static List <MarketTick> GetMarketTicks(string marketName, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List <MarketTick> result = new List <MarketTick>();
try
{
string baseUrl = "https://bittrex.com/Api/v2.0/pub/market/GetTicks?tickInterval=oneMin&marketName=" + marketName;
log.DoLogDebug("Bittrex - Getting ticks for '" + marketName + "'...");
Dictionary <string, dynamic> jsonObject = GetJsonFromURL(baseUrl, log, null);
if (jsonObject.Count > 0)
{
if (jsonObject["success"])
{
if (jsonObject["result"] != null)
{
log.DoLogDebug("Bittrex - " + jsonObject["result"].Count.ToString() + " ticks received.");
foreach (var marketTick in jsonObject["result"])
{
MarketTick tick = new MarketTick();
tick.Price = (double)marketTick["C"];
tick.Time = SystemHelper.TextToDateTime(marketTick["T"].ToString(), Constants.confMinDate);
result.Add(tick);
}
}
else
{
log.DoLogDebug("Bittrex - No ticks received.");
}
}
}
}
catch (Exception ex)
{
log.DoLogCritical(ex.Message, ex);
}
return(result);
}
private void _listener_OnTick(object sender, MarketTick e)
{
_publisher.SendTick(e);
Console.WriteLine(e.ToString());
}
public static void CheckForMarketDataRecreation(string mainMarket, Dictionary <string, Market> markets, PTMagicConfiguration systemConfiguration, LogHelper log)
{
string bittrexDataDirectoryPath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar;
if (!Directory.Exists(bittrexDataDirectoryPath))
{
Directory.CreateDirectory(bittrexDataDirectoryPath);
}
DirectoryInfo dataDirectory = new DirectoryInfo(bittrexDataDirectoryPath);
// Check for existing market files
DateTime latestMarketDataFileDateTime = Constants.confMinDate;
List <FileInfo> marketFiles = dataDirectory.EnumerateFiles("MarketData*").ToList();
FileInfo latestMarketDataFile = null;
if (marketFiles.Count > 0)
{
latestMarketDataFile = marketFiles.OrderByDescending(mdf => mdf.LastWriteTimeUtc).First();
latestMarketDataFileDateTime = latestMarketDataFile.LastWriteTimeUtc;
}
if (latestMarketDataFileDateTime < DateTime.Now.ToUniversalTime().AddMinutes(-20))
{
int lastMarketDataAgeInSeconds = (int)Math.Ceiling(DateTime.Now.ToUniversalTime().Subtract(latestMarketDataFileDateTime).TotalSeconds);
// Go back in time and create market data
DateTime startDateTime = DateTime.Now.ToUniversalTime();
DateTime endDateTime = DateTime.Now.ToUniversalTime().AddHours(-systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours);
if (latestMarketDataFileDateTime != Constants.confMinDate && latestMarketDataFileDateTime > endDateTime)
{
// Existing market files too old => Recreate market data for configured timeframe
log.DoLogInfo("Bittrex - Recreating market data for " + markets.Count + " markets over " + SystemHelper.GetProperDurationTime(lastMarketDataAgeInSeconds) + ". This may take a while...");
endDateTime = latestMarketDataFileDateTime;
}
else
{
// No existing market files found => Recreate market data for configured timeframe
log.DoLogInfo("Bittrex - Recreating market data for " + markets.Count + " markets over " + systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours + " hours. This may take a while...");
}
// Get Ticks for main market
List <MarketTick> mainMarketTicks = new List <MarketTick>();
if (!mainMarket.Equals("USDT", StringComparison.InvariantCultureIgnoreCase))
{
mainMarketTicks = Bittrex.GetMarketTicks("USDT-" + mainMarket, systemConfiguration, log);
}
// Get Ticks for all markets
log.DoLogDebug("Bittrex - Getting ticks for '" + markets.Count + "' markets");
Dictionary <string, List <MarketTick> > marketTicks = new Dictionary <string, List <MarketTick> >();
foreach (string key in markets.Keys)
{
marketTicks.Add(key, Bittrex.GetMarketTicks(key, systemConfiguration, log));
if ((marketTicks.Count % 10) == 0)
{
log.DoLogInfo("Bittrex - No worries, I am still alive... " + marketTicks.Count + "/" + markets.Count + " markets done...");
}
}
log.DoLogInfo("Bittrex - Ticks completed.");
log.DoLogInfo("Bittrex - Creating initial market data ticks. This may take another while...");
int totalTicks = (int)Math.Ceiling(startDateTime.Subtract(endDateTime).TotalMinutes);
int completedTicks = 0;
if (marketTicks.Count > 0)
{
for (DateTime tickTime = startDateTime.ToUniversalTime(); tickTime >= endDateTime.ToUniversalTime(); tickTime = tickTime.AddMinutes(-1))
{
completedTicks++;
double mainCurrencyPrice = 1;
if (mainMarketTicks.Count > 0)
{
List <MarketTick> mainCurrencyTickRange = mainMarketTicks.FindAll(t => t.Time <= tickTime);
if (mainCurrencyTickRange.Count > 0)
{
MarketTick mainCurrencyTick = mainCurrencyTickRange.OrderByDescending(t => t.Time).First();
mainCurrencyPrice = mainCurrencyTick.Price;
}
}
Dictionary <string, Market> tickMarkets = new Dictionary <string, Market>();
foreach (string key in markets.Keys)
{
List <MarketTick> tickRange = marketTicks[key].FindAll(t => t.Time <= tickTime);
if (tickRange.Count > 0)
{
MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First();
Market market = new Market();
market.Position = markets.Count + 1;
market.Name = key;
market.Symbol = key;
market.Price = marketTick.Price;
//market.Volume24h = marketTick.Volume24h;
market.MainCurrencyPriceUSD = mainCurrencyPrice;
tickMarkets.Add(market.Name, market);
}
}
DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime();
FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(tickMarkets), fileDateTime, fileDateTime);
log.DoLogDebug("Bittrex - Market data saved for tick " + fileDateTime.ToString() + " - MainCurrencyPrice=" + mainCurrencyPrice.ToString("#,#0.00") + " USD.");
if ((completedTicks % 100) == 0)
{
log.DoLogInfo("Bittrex - Our magicbots are still at work, hang on... " + completedTicks + "/" + totalTicks + " ticks done...");
}
}
}
log.DoLogInfo("Bittrex - Initial market data created. Ready to go!");
}
}
public static void CheckForMarketDataRecreation(string mainMarket, Dictionary <string, Market> markets, PTMagicConfiguration systemConfiguration, LogHelper log)
{
string binanceUSDataDirectoryPath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar;
if (!Directory.Exists(binanceUSDataDirectoryPath))
{
Directory.CreateDirectory(binanceUSDataDirectoryPath);
}
DirectoryInfo dataDirectory = new DirectoryInfo(binanceUSDataDirectoryPath);
// Check for existing market files
DateTime latestMarketDataFileDateTime = Constants.confMinDate;
List <FileInfo> marketFiles = dataDirectory.EnumerateFiles("MarketData*").ToList();
FileInfo latestMarketDataFile = null;
if (marketFiles.Count > 0)
{
latestMarketDataFile = marketFiles.OrderByDescending(mdf => mdf.LastWriteTimeUtc).First();
latestMarketDataFileDateTime = latestMarketDataFile.LastWriteTimeUtc;
}
if (latestMarketDataFileDateTime < DateTime.UtcNow.AddMinutes(-20))
{
int lastMarketDataAgeInSeconds = (int)Math.Ceiling(DateTime.UtcNow.Subtract(latestMarketDataFileDateTime).TotalSeconds);
// Go back in time and create market data
DateTime startDateTime = DateTime.UtcNow;
DateTime endDateTime = DateTime.UtcNow.AddHours(-systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours);
if (latestMarketDataFileDateTime != Constants.confMinDate && latestMarketDataFileDateTime > endDateTime)
{
// Existing market files too old => Recreate market data for configured timeframe
log.DoLogInfo("BinanceUS - Recreating market data for " + markets.Count + " markets over " + SystemHelper.GetProperDurationTime(lastMarketDataAgeInSeconds) + ". This may take a while...");
endDateTime = latestMarketDataFileDateTime;
}
else
{
// No existing market files found => Recreate market data for configured timeframe
log.DoLogInfo("BinanceUS - Recreating market data for " + markets.Count + " markets over " + systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours + " hours. This may take a while...");
}
int totalTicks = (int)Math.Ceiling(startDateTime.Subtract(endDateTime).TotalMinutes);
// Get Ticks for main market
List <MarketTick> mainMarketTicks = new List <MarketTick>();
if (!mainMarket.Equals("USDT", StringComparison.InvariantCultureIgnoreCase))
{
mainMarketTicks = BinanceUS.GetMarketTicks(mainMarket + "USDT", totalTicks, systemConfiguration, log);
}
// Get Ticks for all markets
log.DoLogDebug("BinanceUS - Getting ticks for '" + markets.Count + "' markets");
ConcurrentDictionary <string, List <MarketTick> > marketTicks = new ConcurrentDictionary <string, List <MarketTick> >();
int ParallelThrottle = 4;
if (systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours > 200)
{
ParallelThrottle = 2;
log.DoLogInfo("----------------------------------------------------------------------------");
log.DoLogInfo("StoreDataMaxHours is greater than 200. Historical data requests will be");
log.DoLogInfo("throttled to avoid exceeding exchange data request limits. This initial ");
log.DoLogInfo("run could take more than 30 minutes. Please go outside for a walk...");
log.DoLogInfo("----------------------------------------------------------------------------");
}
Parallel.ForEach(markets.Keys,
new ParallelOptions {
MaxDegreeOfParallelism = ParallelThrottle
},
(key) =>
{
if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log)))
{
// Failed to add ticks to dictionary
throw new Exception("Failed to add ticks for " + key + " to the memory dictionary, results may be incorrectly calculated!");
}
if ((marketTicks.Count % 10) == 0)
{
log.DoLogInfo("BinanceUS - No worries, I am still alive... " + marketTicks.Count + "/" + markets.Count + " markets done...");
}
});
log.DoLogInfo("BinanceUS - Ticks completed.");
log.DoLogInfo("BinanceUS - Creating initial market data ticks. This may take another while...");
// Go back in time and create market data
int completedTicks = 0;
if (marketTicks.Count > 0)
{
for (DateTime tickTime = startDateTime; tickTime >= endDateTime; tickTime = tickTime.AddMinutes(-1))
{
completedTicks++;
double mainCurrencyPrice = 1;
if (mainMarketTicks.Count > 0)
{
List <MarketTick> mainCurrencyTickRange = mainMarketTicks.FindAll(t => t.Time <= tickTime);
if (mainCurrencyTickRange.Count > 0)
{
MarketTick mainCurrencyTick = mainCurrencyTickRange.OrderByDescending(t => t.Time).First();
mainCurrencyPrice = mainCurrencyTick.Price;
}
}
Dictionary <string, Market> tickMarkets = new Dictionary <string, Market>();
foreach (string key in markets.Keys)
{
List <MarketTick> tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List <MarketTick>();
if (tickRange.Count > 0)
{
MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First();
Market market = new Market();
market.Position = markets.Count + 1;
market.Name = key;
market.Symbol = key;
market.Price = marketTick.Price;
//market.Volume24h = marketTick.Volume24h;
market.MainCurrencyPriceUSD = mainCurrencyPrice;
tickMarkets.Add(market.Name, market);
}
}
DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime();
FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(tickMarkets), fileDateTime, fileDateTime);
log.DoLogDebug("BinanceUS - Market data saved for tick " + fileDateTime.ToString() + " - MainCurrencyPrice=" + mainCurrencyPrice.ToString("#,#0.00") + " USD.");
if ((completedTicks % 100) == 0)
{
log.DoLogInfo("BinanceUS - Our magicbots are still at work, hang on... " + completedTicks + "/" + totalTicks + " ticks done...");
}
}
}
log.DoLogInfo("BinanceUS - Initial market data created. Ready to go!");
}
}
public static List <MarketTick> GetMarketTicks(string marketName, int ticksNeeded, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List <MarketTick> result = new List <MarketTick>();
try
{
Int64 endTime = (Int64)Math.Ceiling(DateTime.UtcNow.Subtract(Constants.Epoch).TotalMilliseconds);
int ticksLimit = 500;
string baseUrl = "";
int ticksFetched = 0;
if (ticksNeeded < ticksLimit)
{
ticksLimit = ticksNeeded;
}
bool go = true;
if (!(marketName == "USDUSDT"))
{
while (ticksFetched < ticksNeeded && go)
{
baseUrl = "https://api.binance.us/api/v1/klines?interval=1m&symbol=" + marketName + "&endTime=" + endTime.ToString() + "&limit=" + ticksLimit.ToString();
log.DoLogDebug("BinanceUS - Getting " + ticksLimit.ToString() + " ticks for '" + marketName + "'...");
Newtonsoft.Json.Linq.JArray jsonArray = GetSimpleJsonArrayFromURL(baseUrl, log);
if (jsonArray.Count > 0)
{
log.DoLogDebug("BinanceUS - " + jsonArray.Count.ToString() + " ticks received.");
foreach (Newtonsoft.Json.Linq.JArray marketTick in jsonArray)
{
MarketTick tick = new MarketTick();
tick.Price = (double)marketTick[4];
tick.Volume24h = (double)marketTick[7];
tick.Time = Constants.Epoch.AddMilliseconds((Int64)marketTick[0]);
result.Add(tick);
}
ticksFetched = ticksFetched + jsonArray.Count;
endTime = endTime - ticksLimit * 60 * 1000;
if (ticksNeeded - ticksFetched < ticksLimit)
{
ticksLimit = ticksNeeded - ticksFetched;
}
}
else
{
log.DoLogDebug("BinanceUS - No ticks received.");
go = false;
}
}
}
}
catch (WebException ex)
{
if (ex.Response != null)
{
using (HttpWebResponse errorResponse = (HttpWebResponse)ex.Response)
{
using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream()))
{
Dictionary <string, string> errorData = JsonConvert.DeserializeObject <Dictionary <string, string> >(reader.ReadToEnd());
if (errorData != null)
{
string errorMessage = "Unable to get data from BinanceUS with URL '" + errorResponse.ResponseUri + "'!";
if (errorData.ContainsKey("code"))
{
errorMessage += " - Code: " + errorData["code"];
}
if (errorData.ContainsKey("msg"))
{
errorMessage += " - Message: " + errorData["msg"];
}
log.DoLogError(errorMessage);
}
}
}
}
result = null;
}
catch (Exception ex)
{
log.DoLogCritical(ex.Message, ex);
}
return(result);
}
public void SendTick(MarketTick tick)
{
_ticks.PublishAsync(ObjectToByteArray(tick));
}
