public void Update(int p_epochs)
{
for (int i = 0; i < SolverConfig.GetInstance().async_learners; i++)
{
if (_learners[i].Started)
{
_learners[i].Update();
}
else
{
FG++;
_learners[i].Reset();
_learners[i].UpdateParams(FG, p_epochs);
if (_actorMonitor.ContainsKey(i))
{
_actorMonitor[i]++;
}
else
{
_actorMonitor[i] = 1;
}
foreach (int v in _actorMonitor.Values)
{
Console.Write(v + " ");
}
Console.WriteLine();
Console.WriteLine("Epoch " + FG);
}
_agent.UpdateQt();
}
}
override public void Init(NeuralNetwork p_network = null)
{
NeuralNetwork network = null;
Optimizer optimizer = null;
if (p_network == null)
{
network = new NeuralNetwork();
network.AddLayer("input", new InputLayer(GetParam(STATE_DIM)), BaseLayer.TYPE.INPUT);
network.AddLayer("hidden0", new CoreLayer(SolverConfig.GetInstance().hidden_layer, ACTIVATION.RELU, BaseLayer.TYPE.HIDDEN), BaseLayer.TYPE.HIDDEN);
network.AddLayer("output", new CoreLayer(GetParam(ACTION_DIM), ACTIVATION.TANH, BaseLayer.TYPE.OUTPUT), BaseLayer.TYPE.OUTPUT);
// feed-forward connections
network.AddConnection("input", "hidden0", Connection.INIT.GLOROT_UNIFORM);
network.AddConnection("hidden0", "output", Connection.INIT.GLOROT_UNIFORM);
}
else
{
network = p_network;
}
optimizer = new ADAM(network);
//optimizer = new RMSProp(network);
//optimizer = new BackProp(network, 1e-5f, 0.99f, true);
_critic = new DeepQLearning(optimizer, network, 0.99f, SolverConfig.GetInstance().memory_size, SolverConfig.GetInstance().batch_size, SolverConfig.GetInstance().qtupdate_size);
_critic.SetAlpha(SolverConfig.GetInstance().learning_rate);
}
public FeasibilityStatus Check(
Instance instance,
StartTimes startTimes,
SolverConfig solverConfig,
int?objective = null)
{
this.instance = ExtendedInstance.GetExtendedInstance(instance);
this.instance.ComputeOptimalSwitchingCosts(); // For TEC computation.
this.startTimes = startTimes;
this.solverConfig = solverConfig;
this.Job = null;
this.NextJob = null;
this.Machine = null;
this.Status = FeasibilityStatus.Unknown;
var feasible =
this.EveryJobHasStartTime() &&
this.JobsWithinHorizon() &&
this.JobsNotOverlapping() &&
this.TransitionsExist() &&
this.ObjectiveCorrespondsWithStartTimes(objective);
if (feasible)
{
this.Status = FeasibilityStatus.Feasible;
}
return(this.Status);
}
public SolverManager()
{
FG = 0;
_timestamp = DateTime.Now.ToFileTime();
_logger = new Logger();
#if !DEBUG
_logger.Init(".\\app_" + _timestamp + ".log");
#endif
_logger.Log(JsonConvert.SerializeObject(SolverConfig.GetInstance()));
int asyncLearners = SolverConfig.GetInstance().async_learners;
_actorMonitor = new Dictionary <int, int>();
_inputInterface = new InputInterface();
_encoder = new StateEncoder(_inputInterface);
_agent = new AsyncQN(asyncLearners);
_learners = new AsyncSolver[asyncLearners];
for (int i = 0; i < asyncLearners; i++)
{
IExploration exp = null;
exp = new BoltzmannExploration(SolverConfig.GetInstance().epsilon, 0.13f);
exp.Init(0.13f);
//exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
//exp.Init(0.05f, 0f);
_learners[i] = new AsyncSolver(_inputInterface, _encoder, _agent, exp, _logger);
}
}
public void Run()
{
int EPOCHS = SolverConfig.GetInstance().epochs;
Console.WriteLine("Epochs: " + EPOCHS);
MazeExample tester = new MazeExample();
//BasePool.StrongControl = true;
IExploration exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
exp.Init(0.02f, 0f);
//IExploration exp = new BoltzmannExploration(0.12f, 0.06f);
tester.Start(exp);
for (int e = 0; e < EPOCHS; e++)
{
while (tester.Started)
{
tester.Update();
}
tester.Reset();
tester.UpdateParams(e, EPOCHS);
//BasePool.Instance.Check();
}
BasePool.Instance.Check();
tester.Quit();
}
public SolverConfig ToSolverConfig(JObject specializedSolverConfig)
{
var solverConfig = new SolverConfig();
if (this.RandomSeed.HasValue)
{
solverConfig.Random = new Random(this.RandomSeed.Value);
}
if (this.TimeLimit.HasValue)
{
solverConfig.TimeLimit = this.TimeLimit;
}
if (this.NumWorkers.HasValue)
{
solverConfig.NumWorkers = this.NumWorkers.Value;
}
if (this.PresolveLevel.HasValue)
{
solverConfig.PresolveLevel = this.PresolveLevel.Value;
}
if (specializedSolverConfig != null)
{
solverConfig.SpecializedSolverConfig = specializedSolverConfig.ToObject <Dictionary <string, object> >();
}
return(solverConfig);
}
private void __Run()
{
int EPOCHS = SolverConfig.GetInstance().epochs;
Console.WriteLine("Level: " + SolverConfig.GetInstance().level);
Console.WriteLine("Epochs: " + EPOCHS);
IExploration exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
exp.Init(0f, 0f);
//tester.Start(new BoltzmannExploration(2, 0, -4));
_solver.Start(_tester);
//BasePool.StrongControl = true;
for (int e = 0; e < EPOCHS; e++)
{
while (_solver.Started)
{
_solver.Update();
}
_solver.Reset();
if (!_tester)
{
_solver.UpdateParams(e, EPOCHS);
}
Console.WriteLine(exp.ToString());
//BasePool.Instance.Check();
}
BasePool.Instance.Check();
Quit();
}
private static SolverResult Solve(Config config, SolverConfig solverConfig, Instance instance)
{
var solver = new SolverFactory().Create(config.SolverName);
Console.WriteLine($"Starting solver: {solver.GetType().Name}");
var solverResult = solver.Solve(solverConfig, instance);
Console.WriteLine($"Solver finished.");
if (solverResult.Status == Status.Heuristic || solverResult.Status == Status.Optimal)
{
Console.WriteLine($"Solution found, checking its feasibility.");
var feasibilityChecker = new FeasibilityChecker();
var feasibilityCheckStatus = feasibilityChecker.Check(
instance,
solverResult.StartTimes,
solverConfig,
solverResult.Objective);
if (feasibilityCheckStatus != FeasibilityChecker.FeasibilityStatus.Feasible)
{
throw new Exception($"Solution not feasible: {feasibilityCheckStatus}");
}
Console.WriteLine($"Solution feasible.");
}
else
{
Console.WriteLine($"No solution found.");
}
return(solverResult);
}
override public void Init(NeuralNetwork p_network = null)
{
_qtUpdateIndex = 0;
_qtUpdateSize = SolverConfig.GetInstance().qtupdate_size;
_networkQ = null;
if (p_network == null)
{
_networkQ = new NeuralNetwork();
_networkQ.AddLayer("input", new InputLayer(GetParam(STATE_DIM)), BaseLayer.TYPE.INPUT);
_networkQ.AddLayer("hidden0", new CoreLayer(SolverConfig.GetInstance().hidden_layer, ACTIVATION.RELU, BaseLayer.TYPE.HIDDEN), BaseLayer.TYPE.HIDDEN);
_networkQ.AddLayer("output", new CoreLayer(GetParam(ACTION_DIM), ACTIVATION.LINEAR, BaseLayer.TYPE.OUTPUT), BaseLayer.TYPE.OUTPUT);
// feed-forward connections
_networkQ.AddConnection("input", "hidden0", Connection.INIT.GLOROT_UNIFORM);
_networkQ.AddConnection("hidden0", "output", Connection.INIT.GLOROT_UNIFORM);
}
else
{
_networkQ = p_network;
}
CreateNetworkQt();
for (int i = 0; i < _learners.Capacity; i++)
{
AsyncDoubleQLearning worker = new AsyncDoubleQLearning(new ADAM(_networkQ), _networkQ, _networkQt, 0.99f, SolverConfig.GetInstance().async_update);
//worker.SetAlpha(SolverConfig.GetInstance().learning_rate);
worker.Optimizer.InitAlpha(SolverConfig.GetInstance().learning_rate, SolverConfig.GetInstance().learning_rate / 10);
_learners.Add(worker);
}
}
public void Start()
{
for (int i = 0; i < SolverConfig.GetInstance().async_learners; i++)
{
_learners[i].Start(i);
}
}
public void Start(bool p_test = false)
{
_test = p_test;
_wins = _loses = 0;
_DEBUG_logger = new Logger();
_DEBUG_limit = SolverConfig.GetInstance().batch_size * 15;
Reset();
}
public static SolverConfig GetInstance()
{
if (_instance == null)
{
_instance = new SolverConfig();
}
return(_instance);
}
public void Init(Action p_initialized)
{
_initialized = p_initialized;
int seed = SolverConfig.GetInstance().seed;
/*
* if (seed == 0)
* {
* seed = RandomGenerator.getInstance().rand(1, 5);
* }
*/
(new StartGameRequest(_inputInterface)).Send(SolverConfig.GetInstance().level, seed, OnStartGameCompleted);
}
private void __Run()
{
int EPOCHS = SolverConfig.GetInstance().epochs;
Console.WriteLine("Parallelism: " + SolverConfig.GetInstance().async_learners);
Console.WriteLine("Level: " + SolverConfig.GetInstance().level);
Console.WriteLine("Epochs: " + EPOCHS);
Console.WriteLine("Epoch " + _solver.FG);
_solver.Start();
while (_solver.FG < EPOCHS)
{
_solver.Update(EPOCHS);
}
BasePool.Instance.Check();
_solver.Quit();
}
public void Reset()
{
_accReward = 0;
waitingForInterface = true; // musi byt este pred send
_level = SolverConfig.GetInstance().level;
_seed = SolverConfig.GetInstance().seed;
if (_seed == 0)
{
_seed = RandomGenerator.getInstance().Rand(1, 5);
}
(new StartGameRequest(_inputInterface)).Send(SolverConfig.GetInstance().level, _seed, OnStartGameCompleted);
_DEBUG_clicks = new List <Vector>();
started = true;
}
public Experiment1()
{
SolverConfig.GetInstance().Load(@".\config.json");
_timestamp = DateTime.Now.ToFileTime();
_logger = new Logger();
#if !DEBUG
_logger.Init(".\\app_" + _timestamp + ".log");
#endif
_logger.Log(JsonConvert.SerializeObject(SolverConfig.GetInstance()));
int asyncLearners = SolverConfig.GetInstance().async_learners;
_inputInterface = new InputInterface();
_encoder = new StateEncoder(_inputInterface);
_agent = new DDQN();
IExploration exp = new EGreedyExploration(SolverConfig.GetInstance().epsilon, 0f);
exp.Init(0.1f, 1f);
_solver = new Solver(_inputInterface, _encoder, _agent, exp, _logger);
}
private void OnStartGameCompleted(StartGameRequest.ResultData result)
{
_encoder.PrepareEncoderProjection(result.levelConfig);
_agent.SetParam(BaseAgent.ACTION_DIM, _encoder.GetActionDim());
_agent.SetParam(BaseAgent.BOOST_INDEX, _encoder.GetActionDim() - 1);
_agent.SetParam(BaseAgent.STATE_DIM, _encoder.GetStateDim());
Console.WriteLine("Max. moves: " + result.levelConfig.moves);
Console.WriteLine("Action dim: " + _agent.GetParam(BaseAgent.ACTION_DIM));
Console.WriteLine("State dim: " + _agent.GetParam(BaseAgent.STATE_DIM));
if (SolverConfig.GetInstance().filename != string.Empty)
{
string filename = ".\\" + SolverConfig.GetInstance().filename;
Console.WriteLine("Loading file " + filename);
_agent.Load(filename);
Console.WriteLine(filename + " loaded.");
}
else
{
_agent.Init();
}
_initialized.Invoke();
}
//TODO: make this more pretty
public static void Main(string[] args)
{
var cliApp = new CommandLineApplication();
var xivPathOpt = cliApp.Option("-p |--game-path <pathToFFXIV>",
"Path to the FFXIV game install (folder containing boot and game)", CommandOptionType.SingleValue);
var configOpt = cliApp.Option("-c |--config-path <pathToYaml>",
"Path to configuration YAML file, default to config.yaml", CommandOptionType.SingleValue);
var excludedOpt = cliApp.Option("-X |--exclude <itemId>",
"Item ids of gear or food to exclude from solving; repeat for non-unique items", CommandOptionType.MultipleValue);
var requiredOpt = cliApp.Option("-R |--require <itemId>",
"Item ids of items required when solving", CommandOptionType.MultipleValue);
var minIlvlOpt = cliApp.Option("-m |--min-itemlevel <ilvl>",
"Minimum item level of items to consider. Uses max-20 if not passed.", CommandOptionType.SingleValue);
var maxIlvlOpt = cliApp.Option("-M |--max-itemlevel <ilvl>",
"Maximum item level of items to consider", CommandOptionType.SingleValue);
var maxOvermeldTierOpt = cliApp.Option("-T |--max-overmeld-tier <tier>",
"The max tier of materia allowed for overmelds", CommandOptionType.SingleValue);
var noMaximizeUnweightedOpt = cliApp.Option("--no-maximize-unweighted",
"Choose to disable maximizing unweighted stats (usually accuracy). Shouldn't be needed.",
CommandOptionType.NoValue);
var noFoodOpt = cliApp.Option("--no-food", "Disable food", CommandOptionType.NoValue);
var noMateriaOpt = cliApp.Option("--no-materia", "Disable materia", CommandOptionType.NoValue);
var noRelicOpt = cliApp.Option("--no-relic", "Disable relic", CommandOptionType.NoValue);
var tiersOpt = cliApp.Option("--use-tiers", "Enable SS tiers. Warning: slow unless using a commercial solver", CommandOptionType.NoValue);
var outputOpt = cliApp.Option("-o |--output <file>", "Write output to <file>", CommandOptionType.SingleValue);
var solverOpt = cliApp.Option("-s |--solver <solver>", "Solver to use (default: GLPK)",
CommandOptionType.SingleValue);
var noSolveOpt = cliApp.Option("--no-solve", "Don't solve the model; only works in conjunction with --debug", CommandOptionType.NoValue);
var debugOpt = cliApp.Option("-d |--debug", "Print the used models in the current directory as model.lp",
CommandOptionType.NoValue);
var jobArg = cliApp.Argument("<job>", "Enter the job abbreviation to solve for");
cliApp.HelpOption("-h |--help");
cliApp.OnExecute(() =>
{
if (jobArg.Value == null)
{
Console.Error.WriteLine("You must provide a job to solve for.");
return(1);
}
if (!xivPathOpt.HasValue())
{
Console.Error.WriteLine("You must provide a path to FFXIV!");
return(1);
}
var realm = new ARealmReversed(xivPathOpt.Value(), Language.English);
var xivColl = realm.GameData;
//TODO: can combine those converters
var deserializer = new DeserializerBuilder()
.WithTypeConverter(new BaseParamConverter(xivColl))
.WithTypeConverter(new ClassJobConverter(xivColl))
.WithTypeConverter(new EquipSlotConverter(xivColl))
.WithTypeConverter(new ItemConverter(xivColl))
.WithTypeConverter(new PiecewiseLinearConverter())
.WithNamingConvention(new CamelCaseNamingConvention())
.Build();
SolverConfig solverConfig = null;
using (var s = new FileStream(configOpt.HasValue() ? configOpt.Value() : "config.yaml", FileMode.Open))
{
solverConfig = deserializer.Deserialize <SolverConfig>(new StreamReader(s));
}
solverConfig.MaximizeUnweightedValues = !noMaximizeUnweightedOpt.HasValue();
solverConfig.UseTiers = tiersOpt.HasValue();
var classJob = xivColl.GetSheet <ClassJob>().Single(x => string.Equals(x.Abbreviation, jobArg.Value, StringComparison.InvariantCultureIgnoreCase));
var items = xivColl.GetSheet <Item>().ToList();
if (excludedOpt.HasValue())
{
var excludedIds = new List <int>();
foreach (var excluded in excludedOpt.Values)
{
try
{
var id = int.Parse(excluded);
var item = xivColl.Items[id];
excludedIds.Add(id);
}
catch (KeyNotFoundException)
{
Console.Error.WriteLine($"Unknown id {excluded}, ignoring.");
}
catch (FormatException)
{
Console.Error.WriteLine($"Not an integer: {excluded}");
}
catch (OverflowException)
{
Console.Error.WriteLine($"Too large: {excluded}");
}
}
items = items.Where(k => !excludedIds.Contains(k.Key)).ToList();
}
//TODO: duplicated code
if (requiredOpt.HasValue())
{
solverConfig.RequiredItems = new List <int>();
requiredOpt.Values.Select(int.Parse).ForEach(solverConfig.RequiredItems.Add);
}
var equip = items.OfType <Equipment>().Where(e => e.ClassJobCategory.ClassJobs.Contains(classJob));
var maxIlvl = equip.Max(x => x.ItemLevel.Key);
if (maxIlvlOpt.HasValue())
{
maxIlvl = int.Parse(maxIlvlOpt.Value());
}
var minIlvl = maxIlvl - 20;
if (minIlvlOpt.HasValue())
{
minIlvl = int.Parse(minIlvlOpt.Value());
}
equip = equip.Where(e => e.ItemLevel.Key >= minIlvl && e.ItemLevel.Key <= maxIlvl || solverConfig.RequiredItems != null && solverConfig.RequiredItems.Contains(e.Key)).ToList();
var food = noFoodOpt.HasValue() ? new List <FoodItem>() : items.Where(FoodItem.IsFoodItem).Select(t => new FoodItem(t));
var maxTier = items.OfType <MateriaItem>().Max(i => i.Tier);
var materia = noMateriaOpt.HasValue() ? new Dictionary <MateriaItem, bool>() : items.OfType <MateriaItem>()
.Where(i => i.Tier == maxTier || (maxOvermeldTierOpt.HasValue() && i.Tier == int.Parse(maxOvermeldTierOpt.Value()) - 1))
.ToDictionary(i => i, i => !maxOvermeldTierOpt.HasValue() || i.Tier < int.Parse(maxOvermeldTierOpt.Value()));
if (noRelicOpt.HasValue())
{
solverConfig.RelicConfigs = new Dictionary <int, RelicConfig>();
}
//TODO: improve solver handling
SolverBase solver = CreateGLPKSolver();
if (solverOpt.HasValue())
{
switch (solverOpt.Value())
{
case "Gurobi":
solver = new GurobiSolver();
solverConfig.SolverSupportsSOS = true;
break;
}
}
var debug = debugOpt.HasValue();
var settings = new OptimizationConfigSection();
//settings.ModelElement.EnableFullNames = debug;
using (var scope = new ModelScope(settings))
{
var model = new BisModel(solverConfig, classJob,
equip, food, materia);
if (debug)
{
using (var f = new FileStream("model.lp", FileMode.Create))
{
model.Model.Write(f, FileType.LP);
}
using (var f = new FileStream("model.mps", FileMode.Create))
{
model.Model.Write(f, FileType.MPS);
}
if (noSolveOpt.HasValue())
{
Console.WriteLine("Printed model, exiting...");
return(0);
}
}
var solution = solver.Solve(model.Model);
model.ApplySolution(solution);
if (outputOpt.HasValue())
{
using (var fs = new FileStream(outputOpt.Value(), FileMode.Create))
{
var sw = new StreamWriter(fs);
OutputModel(model, sw);
sw.Close();
}
}
else
{
OutputModel(model, Console.Out);
}
Console.WriteLine(solverConfig.UseTiers ? "SS tiers have been taken into account" : "SS tiers have been ignored; pass --use-tiers to enable (slow)");
}
return(0);
});
cliApp.Execute(args);
}
public Experiment2()
{
SolverConfig.GetInstance().Load(@".\config.json");
_solver = new SolverManager();
}
public Example1()
{
SolverConfig.GetInstance().Load(@".\config.maze.json");
}
public void Load(string p_filename)
{
string data = System.IO.File.ReadAllText(p_filename);
_instance = JsonConvert.DeserializeObject <SolverConfig>(data);
}
