/// <summary>
/// Triggers the on day completed event.
/// </summary>
/// <param name="action">The action.</param>
/// <param name="reward">The reward.</param>
private void DayCompleted(ActionType action, double reward)
{
using (var DbContext = new DeepQStockContext())
{
var deepRLagentParameters = DbContext.DeepRLAgentParameters.Single(a => a.Id == Agent.Parameters.Id);
var dayCompleted = new OnDayComplete()
{
Agent = deepRLagentParameters,
AgentId = deepRLagentParameters.Id,
DayNumber = DaysSimulated,
Date = CurrentState.Today.Date,
SelectedAction = action,
Reward = reward,
AccumulatedProfit = Profits,
AnnualProfits = AnnualProfits,
AnnualRent = AnnualRent,
TotalOfYears = TotalOfYears,
Period = CurrentState.Today,
VolumeOperated = VolumeOperated
};
DbContext.OnDaysCompletes.Add(dayCompleted);
DbContext.SaveChanges();
RedisManager.Publish(RedisPubSubChannels.OnDayComplete, JsonConvert.SerializeObject(dayCompleted));
}
}
/// <summary>
/// Removes the specified identifier.
/// </summary>
/// <param name="id">The identifier.</param>
public void Remove(long id)
{
using (var ctx = new DeepQStockContext())
{
string jobId = null;
ActiveAgents.TryRemove(id, out jobId);
var agent = ctx.DeepRLAgentParameters.Single(a => a.Id == id);
// Here we have two posible situations, one is if the agent is running, in this case we cannot remove the agent immediately,
// we need mark the agent as removed and stop the agent's job. The remove process will be handle in the shutdown process.
// And the other situation is when the agent is not running, in that case, we can remove immediately.
if (agent.Status == AgentStatus.Running)
{
agent.Status = AgentStatus.Removed;
if (!string.IsNullOrEmpty(jobId))
{
BackgroundJob.Delete(jobId);
}
}
else
{
ctx.RemoveAgent(agent);
}
ctx.SaveChanges();
}
}
/// <summary>
/// Get all instance of agents
/// </summary>
/// <returns></returns>
public DeepRLAgentParameters GetById(long id)
{
DeepRLAgentParameters agent = null;
using (var ctx = new DeepQStockContext())
{
ctx.Configuration.LazyLoadingEnabled = false;
ctx.Configuration.ProxyCreationEnabled = false;
agent = ctx.DeepRLAgentParameters.SingleOrDefault(a => a.Id == id);
ctx.Entry(agent).Reference(a => a.QNetwork).Load();
ctx.Entry(agent).Reference(a => a.StockExchange).Load();
ctx.Entry(agent).Collection(a => a.Decisions).Load();
var lastDecision = agent.Decisions.LastOrDefault();
if (lastDecision != null)
{
ctx.Entry(lastDecision).Reference(d => d.Period).Load();
}
}
return(agent);
}
/// <summary>
/// Initialize and execute the simulation
/// </summary>
/// <param name="token"></param>
/// <param name="agentId"></param>
public void Run(IJobCancellationToken token, long?agentId = null)
{
try
{
Initialize(agentId);
if (Agent == null)
{
return;
}
Simulate(token);
using (var DbContext = new DeepQStockContext())
{
var agentParameters = DbContext.DeepRLAgentParameters.Single(a => a.Id == agentId.Value);
agentParameters.Status = AgentStatus.Completed;
// We need this assignation here because the simulation complete when the current state is null, and
// now we need to store some of the values of the last State.
CurrentState = PreviousState;
var result = new SimulationResult()
{
AgentId = Agent.Parameters.Id,
AnnualProfits = AnnualProfits,
AnnualRent = AnnualRent,
NetCapital = NetCapital,
Profits = Profits,
Earnings = Earnings,
Symbol = Agent.Parameters.StockExchange.Symbol,
CreatedOn = DateTime.Now,
TransactionCost = TransactionCost
};
DbContext.SimulationResults.Add(result);
DbContext.SaveChanges();
RedisManager.Publish(RedisPubSubChannels.OnSimulationComplete, JsonConvert.SerializeObject(new OnSimulationComplete()
{
AgentId = agentParameters.Id
}));
}
}
catch (JobAbortedException)
{
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
Shutdown();
}
}
/// <summary>
/// Shutdowns this instance.
/// </summary>
public void Shutdown()
{
using (var DbContext = new DeepQStockContext())
{
var agent = DbContext.DeepRLAgentParameters.Single(a => a.Id == Agent.Parameters.Id);
if (agent.Status == AgentStatus.Removed)
{
DbContext.RemoveAgent(agent);
}
else if (agent.Status == AgentStatus.Paused || agent.Status == AgentStatus.Completed)
{
Agent.SaveQNetwork();
CurrentState.StockExchangeId = Parameters.Id;
foreach (var p in CurrentState.InternalPeriods)
{
if (p.Id > 0 && !DbContext.IsAttached(p))
{
DbContext.Entry(p).State = EntityState.Unchanged;
}
else
{
DbContext.Periods.AddOrUpdate(p);
}
}
if (CurrentState.Id > 0)
{
var currentStateInDb = DbContext.States.Single(s => s.Id == CurrentState.Id);
DbContext.Entry(currentStateInDb).CurrentValues.SetValues(CurrentState);
DbContext.Entry(currentStateInDb).State = EntityState.Modified;
}
else
{
var oldState = DbContext.States.SingleOrDefault(s => s.StockExchangeId == Parameters.Id);
if (oldState != null)
{
DbContext.States.Remove(oldState);
}
DbContext.States.Add(CurrentState);
}
foreach (var indicator in DailyIndicators.Concat(WeeklyIndicators).Concat(MonthlyIndicators))
{
indicator.Save(DbContext);
}
}
DbContext.SaveChanges();
}
}
/// <summary>
/// Get all simulation results
/// </summary>
/// <returns></returns>
public IEnumerable <SimulationResult> GetAllResults()
{
var results = new List <SimulationResult>();
using (var ctx = new DeepQStockContext())
{
results = ctx.SimulationResults.ToList();
}
return(results);
}
/// <summary>
/// Get all instance of agents
/// </summary>
/// <returns></returns>
public IEnumerable <DeepRLAgentParameters> GetAll()
{
var agents = new List <DeepRLAgentParameters>();
using (var ctx = new DeepQStockContext())
{
agents = ctx.DeepRLAgentParameters
.Include(a => a.StockExchange)
.Include(a => a.QNetwork)
.ToList();
}
return(agents);
}
/// <summary>
/// Save the indicator
/// </summary>
/// <param name="ctx"></param>
public virtual void Save(DeepQStockContext ctx)
{
var set = ctx.Set(System.Type.GetType(ClassType));
var dbObj = set.Find(Id);
if (dbObj == null)
{
set.Add(this);
}
else
{
ctx.Entry(dbObj).CurrentValues.SetValues(this);
ctx.Entry(dbObj).State = EntityState.Modified;
}
}
/// <summary>
/// Save an agent
/// </summary>
/// <param name="agent"></param>
/// <returns></returns>
public long Save(DeepRLAgentParameters agent)
{
using (var ctx = new DeepQStockContext())
{
agent.StockExchange.CsvDataFilePath = Path.Combine(Settings.CsvDataDirectory, string.Format("{0}.csv", agent.StockExchange.Symbol));
ctx.DeepRLAgentParameters.AddOrUpdate(agent);
ctx.StockExchangeParameters.AddOrUpdate(agent.StockExchange);
ctx.QNetworkParameters.AddOrUpdate(agent.QNetwork);
ctx.SaveChanges();
}
Clients.All.onCreatedAgent(agent);
return(agent.Id);
}
/// <summary>
/// Initialize the agent and the stock exchange
/// </summary>
/// <param name="agentId"></param>
protected void Initialize(long?agentId)
{
using (var DbContext = new DeepQStockContext())
{
if (agentId.HasValue)
{
var agentParameters = DbContext.DeepRLAgentParameters
.Include(a => a.QNetwork)
.Include(a => a.StockExchange)
.Single(a => a.Id == agentId.Value);
if (agentParameters.Status == AgentStatus.Completed)
{
DbContext.ClearAgent(agentParameters);
DbContext.SaveChanges();
}
Parameters = agentParameters.StockExchange;
RewardCalculator = RewardCalculator.Use(RewardCalculatorType.WinningsOverLoosings);
DailyIndicators = InitializeIndicators(DbContext, PeriodType.Day);
WeeklyIndicators = InitializeIndicators(DbContext, PeriodType.Week);
MonthlyIndicators = InitializeIndicators(DbContext, PeriodType.Month);
Agent = new DeepRLAgent(agentParameters);
Agent.OnTrainingEpochComplete += (e, args) => RedisManager.Publish(RedisPubSubChannels.OnTrainingEpochComplete, JsonConvert.SerializeObject(args));
var experiences = DbContext.Experiences.Where(e => e.AgentId == agentParameters.Id).ToList();
Agent.SetExperiences(experiences);
DataProvider = new CsvDataProvider(Parameters.CsvDataFilePath, Parameters.EpisodeLength);
if (agentParameters.Status == AgentStatus.Paused)
{
CurrentState = DbContext.States.Include(s => s.InternalPeriods).Single(s => s.StockExchangeId == Parameters.Id);
DataProvider.Seek(CurrentState.Today.Date.AddDays(1));
}
if (agentParameters.Status == AgentStatus.Completed)
{
DbContext.ClearAgent(agentParameters);
}
agentParameters.Status = AgentStatus.Running;
DbContext.SaveChanges();
}
}
}
/// <summary>
/// Save the indicator
/// </summary>
/// <param name="ctx"></param>
public override void Save(DeepQStockContext ctx)
{
var dbObj = ctx.RSIs.Find(Id);
if (dbObj == null)
{
ctx.RSIs.Add(this);
}
else
{
ctx.Entry(dbObj).CurrentValues.SetValues(this);
ctx.Entry(dbObj).State = EntityState.Modified;
dbObj.PreviousPeriod = PreviousPeriod;
}
}
/// <summary>
/// Configure app
/// </summary>
/// <param name="app"></param>
public void Configuration(IAppBuilder app)
{
// Configure Web API for self-host.
HttpConfiguration config = new HttpConfiguration();
config.Routes.MapHttpRoute(
name: "DeepQStockApi",
routeTemplate: "api/{controller}/{id}",
defaults: new { id = RouteParameter.Optional }
);
var appXmlType = config.Formatters.XmlFormatter.SupportedMediaTypes.FirstOrDefault(t => t.MediaType == "application/xml");
config.Formatters.XmlFormatter.SupportedMediaTypes.Remove(appXmlType);
app.Use <GlobalExceptionMiddleware>();
app.UseWebApi(config);
app.UseCors(CorsOptions.AllowAll);
var hubConfiguration = new HubConfiguration();
hubConfiguration.EnableDetailedErrors = true;
app.MapSignalR(hubConfiguration);
var connString = new System.Data.Entity.Infrastructure.SqlConnectionFactory();
using (var ctx = new DeepQStockContext())
{
ctx.DeepRLAgentParameters.FirstOrDefault();
}
GlobalConfiguration.Configuration.UseSqlServerStorage(@"Server=.\sqlexpress; Database=DeepQStockDB; Integrated Security=SSPI;");
GlobalConfiguration.Configuration.UseActivator(new DependencyResolverJobActivator(GlobalHost.DependencyResolver));
app.UseHangfireDashboard();
app.UseHangfireServer();
ConfigureService();
int minWorker, minIOC;
ThreadPool.GetMinThreads(out minWorker, out minIOC);
ThreadPool.SetMinThreads(8, 8);
}
/// <summary>
/// Remove an state if exists
/// </summary>
/// <param name="state"></param>
public void RemoveState(State state)
{
if (state != null)
{
if (state.Id > 0)
{
using (var ctx = new DeepQStockContext())
{
if (!ctx.IsAttached(state))
{
state.InternalPeriods = null;
ctx.States.Attach(state);
}
ctx.Entry(state).State = EntityState.Deleted;
ctx.SaveChanges();
}
}
}
}
/// <summary>
/// Start the simulation of the agent agentId
/// </summary>
/// <param name="id"></param>
public void Pause(long id)
{
using (var ctx = new DeepQStockContext())
{
var agent = ctx.DeepRLAgentParameters.Single(a => a.Id == id);
if (agent.Status != AgentStatus.Running)
{
return;
}
agent.Status = AgentStatus.Paused;
ctx.SaveChanges();
string jobId = null;
ActiveAgents.TryRemove(id, out jobId);
if (!string.IsNullOrEmpty(jobId))
{
BackgroundJob.Delete(jobId);
}
}
}
/// <summary>
/// Generates the a mini batch of randoms sample for train and update Q-network wegihts.
/// </summary>
/// <returns></returns>
private IList <Experience> GenerateMiniBatch()
{
var experiences = new List <Experience>();
if (MemoryReplay.Count <= Parameters.MiniBatchSize)
{
experiences = MemoryReplay.ToList();
}
else
{
var indexes = Enumerable.Range(0, MemoryReplay.Count - 1).OrderBy(x => RandomGenerator.Next());
experiences = MemoryReplay.Where((e, i) => indexes.Contains(i)).ToList();
}
using (var ctx = new DeepQStockContext())
{
foreach (var experience in experiences)
{
if (experience.Id > 0 && !ctx.IsAttached(experience))
{
ctx.Experiences.Attach(experience);
}
if (experience.From == null)
{
ctx.Entry(experience).Reference(e => e.From).Load();
}
if (experience.To == null)
{
ctx.Entry(experience).Reference(e => e.To).Load();
}
}
}
return(experiences.ToList());
}
/// <summary>
/// Initializes a new instance of the <see cref="StorageTests"/> class.
/// </summary>
public StorageTests()
{
Database = new DeepQStockContext();
}
/// <summary>
/// Intialize all the stock indicators
/// </summary>
private IList <TechnicalIndicatorBase> InitializeIndicators(DeepQStockContext DbContext, PeriodType type)
{
var atr = DbContext.AverageTrueRanges.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type);
if (atr == null)
{
atr = new AverageTrueRange(type, Parameters.Id);
DbContext.AverageTrueRanges.Add(atr);
}
var ma8 = DbContext.SimpleMovingAverages.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type && a.Size == 8);
if (ma8 == null)
{
ma8 = new SimpleMovingAverage(type, Parameters.Id, 8);
DbContext.SimpleMovingAverages.Add(ma8);
}
var ema20 = DbContext.ExponentialMovingAverages.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type && a.Size == 20);
if (ema20 == null)
{
ema20 = new ExponentialMovingAverage(type, Parameters.Id, 20);
DbContext.ExponentialMovingAverages.Add(ema20);
}
var ema50 = DbContext.ExponentialMovingAverages.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type && a.Size == 50);
if (ema50 == null)
{
ema50 = new ExponentialMovingAverage(type, Parameters.Id, 50);
DbContext.ExponentialMovingAverages.Add(ema50);
}
var ema200 = DbContext.ExponentialMovingAverages.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type && a.Size == 200);
if (ema200 == null)
{
ema200 = new ExponentialMovingAverage(type, Parameters.Id, 200);
DbContext.ExponentialMovingAverages.Add(ema200);
}
var rsi = DbContext.RSIs.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type);
if (rsi == null)
{
rsi = new RSI(type, Parameters.Id);
DbContext.RSIs.Add(rsi);
}
var dmi = DbContext.DMIs.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type);
if (dmi == null)
{
dmi = new DMI(type, Parameters.Id, atr: atr);
DbContext.DMIs.Add(dmi);
}
var macd = DbContext.MACDs.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type);
if (macd == null)
{
macd = new MACD(type, Parameters.Id);
DbContext.MACDs.Add(macd);
}
var bollingerB = DbContext.BollingerBandsPercentBs.SingleOrDefault(a => a.StockExchangeId == Parameters.Id && a.Type == type);
if (bollingerB == null)
{
bollingerB = new BollingerBandsPercentB(type, Parameters.Id);
DbContext.BollingerBandsPercentBs.Add(bollingerB);
}
return(new List <TechnicalIndicatorBase>()
{
ma8, ema20, ema50, ema200, atr, rsi, dmi, macd, bollingerB
});
}
