public KrakenClient()
{
_url = ConfigurationManager.AppSettings["KrakenBaseAddress"];
_version = int.Parse(ConfigurationManager.AppSettings["KrakenApiVersion"]);
_key = ConfigurationManager.AppSettings["KrakenKey"];
_secret = ConfigurationManager.AppSettings["KrakenSecret"];
_rateGate = new RateGate(1, TimeSpan.FromSeconds(5));
}
public TradingEconomicsEarningsDownloader(string destinationFolder)
{
_fromDate = new DateTime(1998, 1, 1);
_toDate = DateTime.Now;
_destinationFolder = Path.Combine(destinationFolder, "earnings");
_requestGate = new RateGate(1, TimeSpan.FromSeconds(1));
_mapFileResolver = MapFileResolver.Create(Globals.DataFolder, Market.USA);
Directory.CreateDirectory(_destinationFolder);
}
public ExchangeGdaxAPI(ExchangeSettings exchangeSettings)
{
Logger = LogManager.GetCurrentClassLogger();
RequestContentType = "application/json";
_exchangeSettings = exchangeSettings;
LoadSettings();
RateLimit = new RateGate(2, TimeSpan.FromSeconds(1));
}
public TinkoffClient(string token)
{
_token = token;
Task worker = new Task(WorkerPlace);
worker.Start();
_rateGate = new RateGate(1, TimeSpan.FromMilliseconds(500));
}
internal ExecuteRateLimitedOperations(Queue <T> documentIds, Func <T, TransactionOperationsMerger.MergedTransactionCommand> commandToExecute, RateGate rateGate,
OperationCancelToken token, int?batchSize = null)
{
_documentIds = documentIds;
_commandToExecute = commandToExecute;
_rateGate = rateGate;
_token = token;
_batchSize = batchSize;
_cancellationToken = token.Token;
}
public Client()
{
var dao = new AccountDao();
var account = dao.Select().Find(a => a.Default > 0 && a.Platform.ID == Platform.ID);
_url = Platform.Url;
_version = account.ApiVersion;
_key = account.Key;
_secret = account.Secret;
_rateGate = new RateGate(1, TimeSpan.FromSeconds(5));
}
/// <summary>
/// Creates an instance of the class
/// </summary>
/// <param name="destinationDirectory">
/// Directory to write files to. This will be a top-level folder that will have folders
/// following the pattern: {destinationDirectory}/YYYY/MM/dd/entry.json
///
/// This should normally be `/raw_data_folder/alternative/benzinga`
/// </param>
/// <param name="apiKey">Key to access Benzinga's API</param>
public BenzingaNewsDataDownloader(DirectoryInfo destinationDirectory, string apiKey)
{
_destinationDirectory = destinationDirectory;
_apiKey = string.IsNullOrWhiteSpace(apiKey) ? Config.Get("benzinga-news-api-key") : apiKey;
// Limit ourselves to 2 requests per second to Benzinga's API.
// There are 237391 articles starting from 2017-09-12 and ending on 2019-09-13.
// This means that at 200 articles/second, it would take us approximately
// 1186.955 seconds (19 minutes 45 seconds) + some latency to download.
_rateGate = new RateGate(1, TimeSpan.FromSeconds(0.5));
}
public TradingEconomicsCalendarDownloader(string destinationFolder)
{
_fromDate = new DateTime(2000, 10, 01);
_toDate = DateTime.Now;
_destinationFolder = Path.Combine(destinationFolder, "calendar");
// Rate limits on Trading Economics is one request per second
_requestGate = new RateGate(1, TimeSpan.FromSeconds(1));
// Create the destination directory so that we don't error out in case there's no data
Directory.CreateDirectory(_destinationFolder);
}
public ExmoClient(string pubKey, string secKey)
{
_key = pubKey;
_secret = secKey;
Thread worker = new Thread(WorkerPlace);
worker.IsBackground = true;
worker.Start();
_rateGate = new RateGate(1, TimeSpan.FromMilliseconds(500));
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (_rateGate != null)
{
_rateGate.Dispose();
}
}
_rateGate = null;
}
public void RateGate_ShouldSkipBecauseOfTimeout()
{
var gate = new RateGate(1, TimeSpan.FromSeconds(20));
var timer = Stopwatch.StartNew();
Assert.IsTrue(gate.WaitToProceed(-1));
Assert.IsFalse(gate.WaitToProceed(0));
timer.Stop();
Assert.LessOrEqual(timer.Elapsed, TimeSpan.FromSeconds(10));
}
private bool disposedValue = false; // To detect redundant calls
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
if (_rateGate != null)
{
_rateGate.Dispose();
}
}
_rateGate = null;
disposedValue = true;
}
}
internal ExecuteRateLimitedOperations(Queue <T> documentIds, Func <T, TransactionOperationsMerger.MergedTransactionCommand> commandToExecute, RateGate rateGate,
OperationCancelToken token,
int?maxTransactionSize,
int?batchSize)
{
_documentIds = documentIds;
_commandToExecute = commandToExecute;
_rateGate = rateGate;
_token = token;
if (maxTransactionSize != null)
{
_maxTransactionSizeInPages = Math.Max(1, maxTransactionSize.Value / Constants.Storage.PageSize);
}
_batchSize = batchSize;
_cancellationToken = token.Token;
}
/// <summary>
/// Uses Selenium to retrieve the HTML source of each page of Nova Ragnarok's Item Index
/// </summary>
/// <param name="driver">A Selenium WebDriver</param>
/// <returns>HTML source of Nova RO's Item Index</returns>
private static IEnumerable <string> GetItemsHtml(IWebDriver driver)
{
var htmlPages = new List <string>();
using (var rateGate = new RateGate(5, TimeSpan.FromSeconds(2)))
{
for (int i = 1; i <= 1120; i++) // TODO: hard-coded value
{
rateGate.WaitToProceed();
driver.Navigate().GoToUrl($"https://www.novaragnarok.com/?module=item&action=index&p={i}");
htmlPages.Add(driver.PageSource);
}
}
return(htmlPages);
}
public void CheckForUpdates()
{
this._logger.LogDebug(message: "Checking tv maze updates..");
var api = new TvMazeApi(settings: this._settings);
// in each iteration begin with getting the timestamps of all tvshows in the db.
// https://www.tvmaze.com/api#updates
var result = api.GetUpdates();
// Then consider rate limiting
// https://www.tvmaze.com/api#rate-limiting
// Instead of rate limiting directly the execute function of TvMazeApi class, we use rate limiting here to avoid redis timeout failure because of the Parallel foreach.
// redis repository could be implemented to work with bench operations, just didn't want to postpone the actual operation.
// Rate limit is currently 20 calls in 10 seconds, the RateGate component is chosen because it uses SemaphoreSlim to optimize the call rate with the execution time.
using (var rateGate = new RateGate(occurrences: this._settings.RateLimitOccurrences,
timeUnit: TimeSpan.FromSeconds(value: this._settings.RateLimitSeconds)))
{
// Run a loop for every update in the tv maze database
// we dont need more thread to start, then wait in the semaphore queue. So we use MaxDegreeOfParallelism property.
Parallel.ForEach(source: result, parallelOptions: new ParallelOptions
{
MaxDegreeOfParallelism = this._settings.RateLimitOccurrences
},
body: currentElement =>
{
// Wait for the rate limit...
rateGate.WaitToProceed();
// check for the timestamp recently obtained matches the value in the redis cache.
var isValid = this._repository.IsConcurrencyValueValid(value: currentElement);
if (!isValid)
{
// if not, get the show details from api,
// https://www.tvmaze.com/api#shows
var updated = api.GetShowDetail(showId: currentElement.Key);
// Create Integration Event to be published through the Event Bus
var showUpdatedEvent = new ShowUpdatedIntegrationEvent(payload: updated);
// Publish the event so the other projects may know about this change.
this._eventBus.Publish(@event: showUpdatedEvent);
}
});
}
}
private static void TestRateGate()
{
try
{
int timesPerPeriod = 1;
int ms = 500;
int loops = 10;
double msMax = (double)ms * 1.1;
double msMin = (double)ms * 0.9;
RateGate gate = new RateGate(timesPerPeriod, TimeSpan.FromMilliseconds(ms));
if (!gate.WaitToProceed(0))
{
throw new APIException("Rate gate should have allowed immediate access to first attempt");
}
for (int i = 0; i < loops; i++)
{
Stopwatch timer = Stopwatch.StartNew();
gate.WaitToProceed();
timer.Stop();
if (i > 0)
{
// check for too much elapsed time with a little fudge
if (timer.Elapsed.TotalMilliseconds > msMax)
{
throw new APIException("Rate gate took too long to wait in between calls: " + timer.Elapsed.TotalMilliseconds + "ms");
}
// check for too little elapsed time with a little fudge
else if (timer.Elapsed.TotalMilliseconds < msMin)
{
throw new APIException("Rate gate took too little to wait in between calls: " + timer.Elapsed.TotalMilliseconds + "ms");
}
}
}
Console.WriteLine("TestRateGate OK");
}
catch (Exception ex)
{
Console.WriteLine("TestRateGate fail: {0}", ex);
}
}
async Task <IEnumerable <Listing> > QueryAllPages(string type, string query)
{
List <Listing> listings = new List <Listing>();
int page = 1;
AanbodDto responseModel;
using (var rateGate = new RateGate(100, TimeSpan.FromMinutes(1)))
{
do
{
rateGate.WaitToProceed();
responseModel = await QuerySinglePage(type, query, page);
listings.AddRange(responseModel.Objects);
page++;
} while (page <= responseModel.Paging.AantalPaginas);
}
return(listings);
}
public async Task RateGate()
{
const int timesPerPeriod = 1;
const int ms = 100;
const int loops = 5;
const double msMax = (double)ms * 1.5;
const double msMin = (double)ms * (1.0 / 1.5);
var gate = new RateGate(timesPerPeriod, TimeSpan.FromMilliseconds(ms));
var entered = await gate.WaitToProceedAsync(0);
if (!entered)
{
throw new APIException("Rate gate should have allowed immediate access to first attempt");
}
for (var i = 0; i < loops; i++)
{
var timer = Stopwatch.StartNew();
await gate.WaitToProceedAsync();
timer.Stop();
if (i <= 0)
{
continue;
}
// check for too much elapsed time with a little fudge
Assert.IsTrue(
timer.Elapsed.TotalMilliseconds <= msMax,
"Rate gate took too long to wait in between calls: " + timer.Elapsed.TotalMilliseconds + "ms"
);
Assert.IsTrue(
timer.Elapsed.TotalMilliseconds >= msMin,
"Rate gate took too little to wait in between calls: " + timer.Elapsed.TotalMilliseconds + "ms"
);
}
}
public async void IndexCharactersFromTradeApi()
{
using (var rateGate = new RateGate(1, TimeSpan.FromSeconds(2.5)))
{
// Run loop one time per x seconds declared above.
while (true)
{
if (_rateLimited) // Wait for one minute if we are rate limited or if the API is down.
{
_log.LogError("Ratelimited or other error, waiting 30 seconds.");
var task = Task.Delay(30000);
task.Wait();
_rateLimited = false;
}
var tradeUrl = $"{TradeUrl}/?id={_nextChangeId}";
//Don't make this one async, we want to send multiple requests and handle the response when we get it.
GetAndIndexTradeRequest(tradeUrl);
await rateGate.WaitToProceed();
}
}
}
public RateLimitBehavior(int occurences, TimeSpan duration)
{
Gate = new RateGate(occurences, duration);
}
/// <summary>
/// Sets the time interval between calls.
/// For more information, please refer to: https://intrinio.com/documentation/api#limits
/// </summary>
/// <param name="timeSpan">Time interval between to consecutive calls.</param>
/// <remarks>
/// Paid subscription has limits of 1 call per second.
/// Free subscription has limits of 1 call per minute.
/// </remarks>
public static void SetTimeIntervalBetweenCalls(TimeSpan timeSpan)
{
RateGate = new RateGate(1, timeSpan);
}
public async Task FetchOrders(List <int> regions)
{
OrdersBag = new ConcurrentBag <List <MarketOrder> >(); //new List<MarketOrder>(3000000);
var handler = new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
var pageRequests = new ConcurrentQueue <string>();
foreach (var regionId in regions)
{
pageRequests.Enqueue($"https://crest-tq.eveonline.com/market/{regionId}/orders/all/?page=1");
}
// EVE crest allows 150 requests per second
// ignoring the burst allowance of 400 for now
await Task.Run(() =>
{
using (var client = new HttpClient(handler))
using (var rateGate = new RateGate(150, TimeSpan.FromSeconds(1)))
{
ParallelUtils.ParallelWhileNotEmpty(pageRequests, (item, adder) =>
{
rateGate.WaitToProceed();
var result = client.GetAsync(item).Result;
var json = result.Content.ReadAsStringAsync().Result;
var response = JsonConvert.DeserializeObject <MarketOrderCollection>(json);
if (response != null)
{
if (response.next != null)
{
adder(response.next.href);
}
OrdersBag.Add(response.Orders);
//Orders.AddRange(response.Orders);
//Trace.WriteLine("Download: " + item);
}
});
}
}).ContinueWith((prevTask) =>
{
var orders = new List <MarketOrder>(3000000);
foreach (var list in OrdersBag)
{
orders.AddRange(list);
}
var groupedByStation = orders.ToLookup(x => x.StationId);
foreach (var stationGroup in groupedByStation)
{
var station = Db.Instance.Stations.FirstOrDefault(s => s.stationID == stationGroup.Key);
if (station == null)
{
continue;
}
foreach (var order in stationGroup)
{
order.SolarSystemId = station.solarSystemID;
order.RegionId = station.regionID;
order.StationSecurity = station.security; //SecurityUtils.RoundSecurity(station.security);
}
}
orders.RemoveAll(x => x.SolarSystemId == 0);
if (Settings.Default.IgnoreNullSecStations)
{
orders.RemoveAll(x => x.StationSecurity <= 0.0);
}
if (Settings.Default.IgnoreContraband)
{
var contraband = Db.Instance.ContrabandTypes.GroupBy(x => x.typeID).Select(x => x.Key);
orders.RemoveAll(x => contraband.Contains(x.TypeId));
}
OrdersByType = orders.ToLookup(o => o.TypeId); // create subsets for each item type
});
/*await Task.Run(() =>
* {
* var groupedByStation = Orders.ToLookup(x => x.StationID);
*
* foreach (var stationGroup in groupedByStation)
* {
* var station = DB.Instance.Stations.FirstOrDefault(s => s.stationID == stationGroup.Key);
* if (station == null) continue;
*
* foreach (var order in stationGroup)
* {
* order.SolarSystemID = station.solarSystemID;
* order.RegionID = station.regionID;
* order.StationSecurity = station.security;
* }
* }
*
* if(Settings.Default.IgnoreNullSecStations)
* Orders.RemoveAll(x => x.StationSecurity <= 0.0);
*
* if (Settings.Default.IgnoreContraband)
* {
* var contraband = DB.Instance.ContrabandTypes.GroupBy(x => x.typeID).Select(x => x.Key);
* Orders.RemoveAll(x => contraband.Contains(x.TypeID));
* }
* });*/
//OrdersByType = orders.ToLookup(o => o.TypeID); // create subsets for each item type
//Orders.Clear();
}
private async void UpdateLadder(string league)
{
LadderStore.SetLadderRunning(league);
var oldLadder = LadderStore.GetLadder(league);
var newLadder = new List <LadderPlayerModel>();
var pages = Enumerable.Range(0, 25);
using (var rateGate = new RateGate(2, TimeSpan.FromSeconds(1)))
{
foreach (int page in pages)
{
await rateGate.WaitToProceed();
LadderApiResponse result = await FetchLadderApiPage(league, page);
if (result != null)
{
var LadderPlayerList = result.Entries.Select(t => new LadderPlayerModel()
{
Name = t.Character.Name,
Level = t.Character.Level,
Online = t.Online,
Dead = t.Dead,
Account = t.Account.Name,
Experience = t.Character.Experience,
ExperiencePerHour = 0,
Rank = new LadderPlayerRankModel()
{
Overall = t.Rank
},
Depth = new LadderPlayerDepthModel()
{
Solo = t.Character.Depth != null ? t.Character.Depth.Solo : 0,
Group = t.Character.Depth != null ? t.Character.Depth.@default : 0
},
Twitch = t.Account.Twitch?.Name,
Class = t.Character.Class,
Updated = DateTime.Now
}).ToList();
// Convert result to LadderPlayer model here
newLadder.AddRange(LadderPlayerList);
if (newLadder.Count == result.Total || result.Entries.Count == 0)
{
break;
}
}
else
{
LadderStore.RemoveLadderStatus(league);
break;
}
}
}
if (newLadder.Count > 0)
{
newLadder = CalculateStatistics(oldLadder, newLadder);
LadderStore.SetLadder(league, newLadder);
LadderStore.SetLadderFinished(league);
}
}
/// <summary>
/// Get historical data enumerable for a trading pair, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="symbol">Symbol for the data we're looking for.</param>
/// <param name="resolution">Resolution of the data request</param>
/// <param name="startUtc">Start time of the data in UTC</param>
/// <param name="endUtc">End time of the data in UTC</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc)
{
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
if (resolution != Resolution.Tick)
{
throw new NotSupportedException("Only Tick Resolution is supported.");
}
var startUnixTime = Convert.ToInt64(Time.DateTimeToUnixTimeStamp(startUtc) * 1000000000); // Multiply by 10^9 per Kraken API
var endUnixTime = Convert.ToInt64(Time.DateTimeToUnixTimeStamp(endUtc) * 1000000000);
var url = string.Format(UrlPrototype, symbol.Value, startUnixTime);
List <List <string> > data;
using (var client = new WebClient())
{
var rateGate = new RateGate(10, TimeSpan.FromMinutes(1)); // 10 calls per minute for Kraken API
rateGate.WaitToProceed();
var response = client.DownloadString(url);
dynamic result = JsonConvert.DeserializeObject <dynamic>(response);
if (result.error.Count != 0)
{
throw new Exception("Error in Kraken API: " + result.error[0]);
}
data = result.result[symbol.Value].ToObject <List <List <string> > >();
foreach (var i in data)
{
var time = Time.UnixTimeStampToDateTime(Convert.ToDouble(i[2].Split('.')[0]));
if (time > endUtc)
{
break;
}
var value = Decimal.Parse(i[0]);
var volume = Decimal.Parse(i[1]);
yield return(new Tick
{
Value = value,
Time = time,
DataType = MarketDataType.Tick,
Symbol = symbol,
TickType = TickType.Trade,
Quantity = volume,
Exchange = "kraken"
});
}
var last = Convert.ToInt64(result.result.last);
while (last < endUnixTime)
{
url = string.Format(UrlPrototype, symbol.Value, last);
rateGate.WaitToProceed();
response = client.DownloadString(url);
result = JsonConvert.DeserializeObject <dynamic>(response);
var errorCount = 0;
while (result.error.Count != 0 && errorCount < 10)
{
errorCount++;
rateGate.WaitToProceed();
response = client.DownloadString(url);
result = JsonConvert.DeserializeObject <dynamic>(response);
}
if (result.error.Count != 0 && errorCount >= 10)
{
throw new Exception("Error in Kraken API: " + result.error[0]);
}
data = result.result[symbol.Value].ToObject <List <List <string> > >();
foreach (var i in data)
{
var time = Time.UnixTimeStampToDateTime(Convert.ToDouble(i[2].Split('.')[0]));
if (time > endUtc)
{
break;
}
var value = Decimal.Parse(i[0]);
var volume = Decimal.Parse(i[1]);
yield return(new Tick
{
Value = value,
Time = time,
DataType = MarketDataType.Tick,
Symbol = symbol,
TickType = TickType.Trade,
Quantity = volume,
Exchange = "kraken"
});
}
last = Convert.ToInt64(result.result.last);
}
}
}
/// <summary>
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable <BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
if (tickType != TickType.Trade)
{
yield break;
}
if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
}
if (resolution != Resolution.Tick)
{
throw new NotSupportedException("Only Tick Resolution is supported.");
}
var startUnixTime = Convert.ToInt64(Time.DateTimeToUnixTimeStamp(startUtc) * 1000000000); // Multiply by 10^9 per Kraken API
var endUnixTime = Convert.ToInt64(Time.DateTimeToUnixTimeStamp(endUtc) * 1000000000);
var url = string.Format(CultureInfo.InvariantCulture, UrlPrototype, symbol.Value, startUnixTime);
List <List <string> > data;
using (var client = new WebClient())
{
var rateGate = new RateGate(10, TimeSpan.FromMinutes(1)); // 10 calls per minute for Kraken API
rateGate.WaitToProceed();
var response = client.DownloadString(url);
dynamic result = JsonConvert.DeserializeObject <dynamic>(response);
if (result.error.Count != 0)
{
throw new Exception("Error in Kraken API: " + result.error[0]);
}
if (result.result.ContainsKey(symbol.Value))
{
data = result.result[symbol.Value].ToObject <List <List <string> > >();
}
else
{
throw new NotSupportedException("Asset pair was not found in the response. Make sure you use the correct model (XBTUSD -> XXBTZUSD).");
}
foreach (var i in data)
{
var time = Time.UnixTimeStampToDateTime(Parse.Double(i[2].Split('.')[0]));
if (time > endUtc)
{
break;
}
var value = Parse.Decimal(i[0]);
var volume = Parse.Decimal(i[1]);
yield return(new Tick
{
Value = value,
Time = time,
DataType = MarketDataType.Tick,
Symbol = symbol,
TickType = TickType.Trade,
Quantity = volume,
Exchange = "kraken"
});
}
var last = Convert.ToInt64(result.result.last);
while (last < endUnixTime)
{
url = string.Format(UrlPrototype, symbol.Value, last);
rateGate.WaitToProceed();
response = client.DownloadString(url);
result = JsonConvert.DeserializeObject <dynamic>(response);
var errorCount = 0;
while (result.error.Count != 0 && errorCount < 10)
{
errorCount++;
rateGate.WaitToProceed();
response = client.DownloadString(url);
result = JsonConvert.DeserializeObject <dynamic>(response);
}
if (result.error.Count != 0 && errorCount >= 10)
{
throw new Exception("Error in Kraken API: " + result.error[0]);
}
data = result.result[symbol.Value].ToObject <List <List <string> > >();
foreach (var i in data)
{
var time = Time.UnixTimeStampToDateTime(Parse.Double(i[2].Split('.')[0]));
if (time > endUtc)
{
break;
}
var value = Parse.Decimal(i[0]);
var volume = Parse.Decimal(i[1]);
yield return(new Tick
{
Value = value,
Time = time,
DataType = MarketDataType.Tick,
Symbol = symbol,
TickType = TickType.Trade,
Quantity = volume,
Exchange = "kraken"
});
}
last = Convert.ToInt64(result.result.last);
}
}
}
public ApiClient(ApiContext context, RateGate rateGate, IResponseConverter converter)
{
_apiRequest = new ApiRequest(context, rateGate);
_converter = converter;
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (_rateGate != null)
_rateGate.Dispose();
}
_rateGate = null;
}
protected internal Method()
{
_rateGate = new RateGate(1, TimeSpan.FromSeconds(5));
}
public ApiRequest(ApiContext context, RateGate rateGate)
{
_apiContext = context;
_rateGate = rateGate;
}
private ThroughputControlChannelModule(IChannel channel, int throughput)
{
_channel = channel;
_rateGate = new RateGate(throughput, TimeSpan.FromSeconds(1));
}
