/// <summary>
/// Creates the final <see cref="AdapterDataRecord{GurobiResult}"/>.
/// </summary>
/// <param name="timeStamp">The time stamp.</param>
/// <param name="model">The <see cref="GRBModel"/>.</param>
/// <param name="result">The <see cref="GurobiResult"/>.</param>
/// <returns>The final <see cref="AdapterDataRecord{GurobiResult}"/>.</returns>
private AdapterDataRecord <GurobiResult> CreateFinalAdapterDataRecord(DateTime timeStamp, GRBModel model, GurobiResult result)
{
if (this._instanceFeatures == null)
{
throw new ArgumentNullException(nameof(this._instanceFeatures));
}
var finalRuntimeFeatures = this.GetFinalGurobiRuntimeFeatures(model, result);
var lastRuntimeFeatures = this._lastRuntimeFeatures ?? finalRuntimeFeatures;
var adapterFeatures = GurobiUtils.ComposeAdapterFeatures(finalRuntimeFeatures, lastRuntimeFeatures, this._instanceFeatures);
var adapterFeaturesHeader = GurobiUtils.ComposeAdapterFeaturesHeader(finalRuntimeFeatures, lastRuntimeFeatures, this._instanceFeatures);
this._lastRuntimeFeatures = finalRuntimeFeatures.Copy();
return(new AdapterDataRecord <GurobiResult>(
"Gurobi901",
result.TargetAlgorithmStatus,
// The cpu time is not recordable, because all parallel Gurobi runs are started in the same process in this implementation.
TimeSpan.FromSeconds(0),
this.GetCurrentRuntime(timeStamp),
timeStamp,
adapterFeaturesHeader,
adapterFeatures,
result));
}
/// <summary>
/// Updates the <see cref="CurrentRuntimeFeatures"/> in MIP phase.
/// </summary>
private void UpdateCurrentGurobiFeaturesInMipPhase()
{
this.CurrentRuntimeFeatures.BestObjective = this.GetCallbackDouble(GRB.Callback.MIP_OBJBST, this.CurrentRuntimeFeatures.BestObjective);
this.CurrentRuntimeFeatures.BestObjectiveBound = this.GetCallbackDouble(
GRB.Callback.MIP_OBJBND,
this.CurrentRuntimeFeatures.BestObjectiveBound);
this.CurrentRuntimeFeatures.FeasibleSolutionsCount = this.GetCallbackInt(
GRB.Callback.MIP_SOLCNT,
this.CurrentRuntimeFeatures.FeasibleSolutionsCount);
this.CurrentRuntimeFeatures.ExploredNodeCount = this.GetCallbackDouble(
GRB.Callback.MIP_NODCNT,
this.CurrentRuntimeFeatures.ExploredNodeCount);
this.CurrentRuntimeFeatures.UnexploredNodeCount = this.GetCallbackDouble(
GRB.Callback.MIP_NODLFT,
this.CurrentRuntimeFeatures.UnexploredNodeCount);
this.CurrentRuntimeFeatures.SimplexIterationsCount = this.GetCallbackDouble(
GRB.Callback.MIP_ITRCNT,
this.CurrentRuntimeFeatures.SimplexIterationsCount);
this.CurrentRuntimeFeatures.CuttingPlanesCount = this.GetCallbackInt(
GRB.Callback.MIP_CUTCNT,
this.CurrentRuntimeFeatures.CuttingPlanesCount);
this.CurrentRuntimeFeatures.MipGap = GurobiUtils.GetMipGap(
this.CurrentRuntimeFeatures.BestObjective,
this.CurrentRuntimeFeatures.BestObjectiveBound);
}
/// <summary>
/// Creates the current <see cref="AdapterDataRecord{GurobiResult}"/>.
/// </summary>
/// <param name="timeStamp">The time stamp.</param>
/// <returns>The current <see cref="AdapterDataRecord{GurobiResult}"/>.</returns>
private AdapterDataRecord <GurobiResult> CreateCurrentAdapterDataRecord(DateTime timeStamp)
{
if (this._instanceFeatures == null)
{
throw new ArgumentNullException(nameof(this._instanceFeatures));
}
var currentRuntimeFeatures = this.GetCurrentGurobiRuntimeFeatures();
var lastRuntimeFeatures = this._lastRuntimeFeatures ?? currentRuntimeFeatures;
var adapterFeatures = GurobiUtils.ComposeAdapterFeatures(currentRuntimeFeatures, lastRuntimeFeatures, this._instanceFeatures);
var adapterFeaturesHeader = GurobiUtils.ComposeAdapterFeaturesHeader(currentRuntimeFeatures, lastRuntimeFeatures, this._instanceFeatures);
this._lastRuntimeFeatures = currentRuntimeFeatures.Copy();
return(new AdapterDataRecord <GurobiResult>(
"Gurobi901",
TargetAlgorithmStatus.Running,
// The cpu time is not recordable, because all parallel Gurobi runs are started in the same process in this implementation.
TimeSpan.FromSeconds(0),
this.GetCurrentRuntime(timeStamp),
timeStamp,
adapterFeaturesHeader,
adapterFeatures,
new GurobiResult(
currentRuntimeFeatures.MipGap,
this._runnerConfiguration.CpuTimeout,
TargetAlgorithmStatus.CancelledByGrayBox,
currentRuntimeFeatures.FeasibleSolutionsCount > 0)));
}
/// <summary>
/// Creates the list of instances to train on using .mps files in the given directory.
/// </summary>
/// <param name="pathToInstanceFolder">Path to the folder containing the instance .mps files.</param>
/// <param name="numSeedsToUse">The number of seeds.</param>
/// <param name="rngSeed">The random number generator seed.</param>
/// <returns>
/// The created list.
/// </returns>
public static List <InstanceSeedFile> CreateInstances(string pathToInstanceFolder, int numSeedsToUse, int rngSeed)
{
try
{
// Find all .mps files in directory and set them as instances.
var instanceDirectory = new DirectoryInfo(pathToInstanceFolder);
var instanceSeedCombinations = new List <string>();
var instanceSeedFiles = new List <InstanceSeedFile>();
foreach (var instanceFilePath in instanceDirectory.EnumerateFiles()
.Where(file => file.Extension.ToLower() == ".mps"))
{
var fileAndSeedCsv = instanceFilePath.FullName;
foreach (var seed in GurobiUtils.SeedsToUse(numSeedsToUse, rngSeed))
{
instanceSeedFiles.Add(new InstanceSeedFile(instanceFilePath.FullName, seed));
fileAndSeedCsv += $";{seed}";
}
instanceSeedCombinations.Add(fileAndSeedCsv);
}
GurobiUtils.DumpFileSeedCombinations(instanceDirectory, instanceSeedCombinations);
return(instanceSeedFiles);
}
catch (Exception e)
{
Console.Out.WriteLine(e.Message);
Console.Out.WriteLine("Cannot open folder.");
throw;
}
}
/// <summary>
/// Builds an instance of the <see cref="AlgorithmTuner{TTargetAlorithm,TInstance,TResult}" /> class for tuning Gurobi.
/// </summary>
/// <param name="configuration">The <see cref="AlgorithmTunerConfiguration" /> to use.</param>
/// <param name="pathToTrainingInstanceFolder">The path to the folder containing training instances.</param>
/// <param name="pathToTestInstanceFolder">The path to test instance folder.</param>
/// <param name="gurobiConfigBuilder">The gurobi configuration builder.</param>
/// <returns>
/// The built instance.
/// </returns>
public static AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult> BuildGurobiRunner(
AlgorithmTunerConfiguration configuration,
string pathToTrainingInstanceFolder,
string pathToTestInstanceFolder,
GurobiRunnerConfiguration.GurobiRunnerConfigBuilder gurobiConfigBuilder)
{
var gurobiConfig = gurobiConfigBuilder.Build(configuration.CpuTimeout);
var tuner = new AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult>(
targetAlgorithmFactory: new GurobiRunnerFactory(gurobiConfig),
runEvaluator: new GurobiRunEvaluator(),
trainingInstances: GurobiUtils.CreateInstances(pathToTrainingInstanceFolder, gurobiConfig.NumberOfSeeds, gurobiConfig.RngSeed),
parameterTree: GurobiUtils.CreateParameterTree(),
configuration: configuration);
try
{
if (!string.IsNullOrWhiteSpace(pathToTestInstanceFolder))
{
var testInstances = GurobiUtils.CreateInstances(pathToTestInstanceFolder, gurobiConfig.NumberOfSeeds, gurobiConfig.RngSeed);
tuner.SetTestInstances(testInstances);
}
}
catch
{
}
return(tuner);
}
/// <summary>
/// Creates a <see cref="ParameterTree" /> of tuneable Gurobi parameters for MIP solving.
/// The parameters and their connections are described in parameterTree.xml.
/// </summary>
/// <returns>The <see cref="ParameterTree" />.</returns>
public static ParameterTree CreateParameterTree()
{
var parameterTree = ParameterTreeConverter.ConvertToParameterTree(
Path.Combine(AppDomain.CurrentDomain.BaseDirectory ?? throw new InvalidOperationException(), @"parameterTree.xml"));
GurobiUtils.AddAllIndicatorParameterWrappers(parameterTree);
return(parameterTree);
}
/// <summary>
/// Updates the <see cref="CurrentRuntimeFeatures"/> in MIP_SOL phase.
/// </summary>
private void UpdateCurrentGurobiFeaturesInMipSolPhase()
{
this.CurrentRuntimeFeatures.BestObjective = this.GetCallbackDouble(GRB.Callback.MIPSOL_OBJBST, this.CurrentRuntimeFeatures.BestObjective);
this.CurrentRuntimeFeatures.BestObjectiveBound = this.GetCallbackDouble(
GRB.Callback.MIPSOL_OBJBND,
this.CurrentRuntimeFeatures.BestObjectiveBound);
this.CurrentRuntimeFeatures.FeasibleSolutionsCount = this.GetCallbackInt(
GRB.Callback.MIPSOL_SOLCNT,
this.CurrentRuntimeFeatures.FeasibleSolutionsCount);
this.CurrentRuntimeFeatures.ExploredNodeCount = this.GetCallbackDouble(
GRB.Callback.MIPSOL_NODCNT,
this.CurrentRuntimeFeatures.ExploredNodeCount);
this.CurrentRuntimeFeatures.MipGap = GurobiUtils.GetMipGap(
this.CurrentRuntimeFeatures.BestObjective,
this.CurrentRuntimeFeatures.BestObjectiveBound);
}
/// <summary>
/// Builds an instance of the <see cref="AlgorithmTuner{TTargetAlorithm,TInstance,TResult}" /> class for tuning Gurobi.
/// </summary>
/// <param name="tunerConfig">The <see cref="AlgorithmTunerConfiguration" /> to use.</param>
/// <param name="pathToTrainingInstanceFolder">The path to the folder containing training instances.</param>
/// <param name="pathToTestInstanceFolder">The path to test instance folder.</param>
/// <param name="gurobiConfigBuilder">The gurobi configuration builder.</param>
/// <returns>
/// The built instance.
/// </returns>
public static AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult> BuildGurobiRunner(
AlgorithmTunerConfiguration tunerConfig,
string pathToTrainingInstanceFolder,
string pathToTestInstanceFolder,
GurobiRunnerConfiguration.GurobiRunnerConfigBuilder gurobiConfigBuilder)
{
var gurobiConfig = Program.BuildGurobiConfigAndCheckThreadCount(gurobiConfigBuilder, tunerConfig);
var tuner = new AlgorithmTuner <GurobiRunner, InstanceSeedFile, GurobiResult>(
targetAlgorithmFactory: new GurobiRunnerFactory(gurobiConfig, tunerConfig),
runEvaluator: new GurobiRunEvaluator(tunerConfig.CpuTimeout),
trainingInstances: InstanceSeedFile.CreateInstanceSeedFilesFromDirectory(
pathToTrainingInstanceFolder,
GurobiUtils.ListOfValidFileExtensions,
gurobiConfig.NumberOfSeeds,
gurobiConfig.RngSeed),
parameterTree: GurobiUtils.CreateParameterTree(),
configuration: tunerConfig,
customGrayBoxMethods: new GurobiGrayBoxMethods());
try
{
if (!string.IsNullOrWhiteSpace(pathToTestInstanceFolder))
{
var testInstances = InstanceSeedFile.CreateInstanceSeedFilesFromDirectory(
pathToTestInstanceFolder,
GurobiUtils.ListOfValidFileExtensions,
gurobiConfig.NumberOfSeeds,
gurobiConfig.RngSeed);
tuner.SetTestInstances(testInstances);
}
}
catch
{
}
return(tuner);
}
/// <summary>
/// Creates a task that runs Gurobi on the given instance.
/// </summary>
/// <param name="instance">
/// Instance to run on.
/// </param>
/// <param name="cancellationToken">
/// The cancellation token given to the task. NOTE: In this implementation, we ignore this cancellation token, since Gurobi uses its own cancellation token in the callback.
/// </param>
/// <returns>
/// A task that returns the run's runtime, gap, feasibility and completion status onto return.
/// </returns>
public Task <GurobiResult> Run(InstanceSeedFile instance, CancellationToken cancellationToken)
{
// Check if the runner has already been disposed.
if (this._hasAlreadyBeenDisposed)
{
throw new ObjectDisposedException("GurobiRunner", "Called Run on a disposed GurobiRunner.");
}
// Continue if it hasn't.
// ReSharper disable once MethodSupportsCancellation
var solveTask = Task.Run(
() =>
{
this._recordTimer = null;
try
{
// Prepare Gurobi model: Use configured _environment and the given instance,
// then add a cancellation token source and the Gurobi callback for cancellation.
var instanceFile = new FileInfo(instance.Path);
if (!File.Exists(instanceFile.FullName))
{
throw new FileNotFoundException($"Instance {instanceFile.FullName} not found!");
}
LoggingHelper.WriteLine(VerbosityLevel.Debug, $"Setting MIPGap to {this._runnerConfiguration.TerminationMipGap}");
this._environment.MIPGap = this._runnerConfiguration.TerminationMipGap;
LoggingHelper.WriteLine(VerbosityLevel.Debug, $"Current Seed: {instance.Seed}");
this._environment.Seed = instance.Seed;
this._environment.TimeLimit = this._runnerConfiguration.CpuTimeout.TotalSeconds;
if (!Directory.Exists("GurobiLog"))
{
Directory.CreateDirectory("GurobiLog");
}
var instanceFileNameWithoutExtension = GurobiUtils.GetFileNameWithoutGurobiExtension(instanceFile);
this._environment.LogFile = "GurobiLog" + Path.DirectorySeparatorChar
+ $"GurobiRunner_{DateTime.Now:yy-MM-dd_HH-mm-ss-ffff}_"
+ instanceFileNameWithoutExtension + $"_{Guid.NewGuid()}.log";
var model = new GRBModel(this._environment, instanceFile.FullName)
{
ModelName = instanceFileNameWithoutExtension
};
if (GurobiRunner.TryToGetMstFile(instanceFile.DirectoryName, instanceFileNameWithoutExtension, out var mstFileFullName))
{
model.Read(mstFileFullName);
}
this._startTimeStamp = DateTime.Now;
this._targetAlgorithmStatus = TargetAlgorithmStatus.Running;
this._innerCancellationTokenSource = new CancellationTokenSource(this._runnerConfiguration.CpuTimeout);
this._grbCallback = new GurobiCallback(
this._innerCancellationTokenSource.Token,
this._tunerConfiguration.EnableDataRecording,
this._startTimeStamp);
model.SetCallback(this._grbCallback);
if (this._tunerConfiguration.EnableDataRecording)
{
this._lastRuntimeFeatures = null;
this.SetGurobiInstanceFeatures(model);
// Start record timer.
var autoResetEvent = new AutoResetEvent(false);
this._recordTimer = new Timer(
(timerCallback) =>
{
var currentTimeStamp = DateTime.Now;
if (currentTimeStamp - this._startTimeStamp < this._runnerConfiguration.CpuTimeout)
{
// Write current line to data log.
var currentAdapterData = this.CreateCurrentAdapterDataRecord(currentTimeStamp);
this.OnNewDataRecord?.Invoke(this, currentAdapterData);
}
},
autoResetEvent,
TimeSpan.FromSeconds(0),
this._tunerConfiguration.DataRecordUpdateInterval);
}
// Optimize. This step may be aborted in the callback.
model.Optimize();
if (this._targetAlgorithmStatus != TargetAlgorithmStatus.CancelledByGrayBox)
{
this._recordTimer?.Dispose();
this._targetAlgorithmStatus = this.GetIsRunInterrupted(model)
? TargetAlgorithmStatus.CancelledByTimeout
: TargetAlgorithmStatus.Finished;
}
var finalTimeStamp = DateTime.Now;
var result = this.CreateGurobiResult(finalTimeStamp, model);
if (this._tunerConfiguration.EnableDataRecording)
{
// Write last line to data log.
var lastAdapterData = this.CreateFinalAdapterDataRecord(finalTimeStamp, model, result);
this.OnNewDataRecord?.Invoke(this, lastAdapterData);
}
// Before returning, dispose of Gurobi model.
model.Dispose();
return(result);
}
finally
{
this._recordTimer?.Dispose();
}
});
return(solveTask);
}
/// <summary>
/// Entry point to the program.
/// </summary>
/// <param name="args">If 'master' is included as argument, a
/// <see cref="Master{TTargetAlgorithm,TInstance,TResult}"/> is starting using the provided arguments.
/// Otherwise, a <see cref="Worker"/> is started with the provided arguments.</param>
public static void Main(string[] args)
{
ProcessUtils.SetDefaultCultureInfo(CultureInfo.InvariantCulture);
LoggingHelper.Configure($"parserLog_{ProcessUtils.GetCurrentProcessId()}.log");
// Parse gurobi configuration.
var gurobiParser = new GurobiRunnerConfigurationParser();
if (!ArgumentParserUtils.ParseArguments(gurobiParser, args))
{
return;
}
if (gurobiParser.IsPostTuningRunner)
{
LoggingHelper.Configure($"consoleOutput_PostTuningRun_{ProcessUtils.GetCurrentProcessId()}.log");
// Parse and build tuner configuration.
var masterArgumentParser = new MasterArgumentParser();
if (!ArgumentParserUtils.ParseArguments(masterArgumentParser, gurobiParser.AdditionalArguments.ToArray()))
{
return;
}
var tunerConfig = masterArgumentParser.ConfigurationBuilder.Build();
LoggingHelper.ChangeConsoleLoggingLevel(tunerConfig.Verbosity);
// Build gurobi configuration.
var gurobiConfig = Program.BuildGurobiConfigAndCheckThreadCount(gurobiParser.ConfigurationBuilder, tunerConfig);
var gurobiRunnerFactory = new GurobiRunnerFactory(gurobiConfig, tunerConfig);
var parameterTree = GurobiUtils.CreateParameterTree();
// Start post tuning runner.
var parallelPostTuningRunner =
new ParallelPostTuningRunner <GurobiRunner, InstanceSeedFile, GurobiResult>(
tunerConfig,
gurobiParser.PostTuningConfiguration,
gurobiRunnerFactory,
parameterTree);
parallelPostTuningRunner.ExecutePostTuningRunsInParallel();
return;
}
if (gurobiParser.IsMaster)
{
Master <GurobiRunner, InstanceSeedFile, GurobiResult, StandardRandomForestLearner <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>, ReuseOldTreesStrategy> .Run(
args : gurobiParser.AdditionalArguments.ToArray(),
algorithmTunerBuilder : (algorithmTunerConfig, pathToInstanceFolder, pathToTestInstanceFolder) =>
Program.BuildGurobiRunner(
algorithmTunerConfig,
pathToInstanceFolder,
pathToTestInstanceFolder,
gurobiParser.ConfigurationBuilder));
}
else
{
Worker.Run(gurobiParser.AdditionalArguments.ToArray());
}
}