private async void BuildCore(string projectFilePath, string customArguments)
{
var progress = new BuildProgress();
progress.ProgressText = $"Building {projectFilePath}...";
SetContent(progress);
var buildHost = new HostedBuild(projectFilePath, customArguments);
Build result = await buildHost.BuildAndGetResult(progress);
progress.ProgressText = "Analyzing build...";
await System.Threading.Tasks.Task.Run(() => BuildAnalyzer.AnalyzeBuild(result));
DisplayBuild(result);
}
public Task <Build> BuildAndGetResult(BuildProgress progress)
{
var msbuildExe = SettingsService.GetMSBuildExe();
var postfixArguments = GetPostfixArguments();
// the command line we pass to Process.Start doesn't need msbuild.exe
var commandLine = $"{QuoteIfNeeded(projectFilePath)} {customArguments} {postfixArguments}";
// the command line we display to the user should contain the full path to msbuild.exe
progress.MSBuildCommandLine = $"{QuoteIfNeeded(msbuildExe)} {commandLine}";
return(System.Threading.Tasks.Task.Run(() =>
{
try
{
var arguments = commandLine;
var processStartInfo = new ProcessStartInfo(msbuildExe, arguments);
processStartInfo.WorkingDirectory = Path.GetDirectoryName(projectFilePath);
var process = Process.Start(processStartInfo);
process.WaitForExit();
var build = Serialization.Read(logFilePath);
File.Delete(logFilePath);
var projectImportsZip = Path.ChangeExtension(logFilePath, ".ProjectImports.zip");
if (File.Exists(projectImportsZip))
{
File.Delete(projectImportsZip);
}
return build;
}
catch (Exception ex)
{
ex = ExceptionHandler.Unwrap(ex);
var build = new Build();
build.Succeeded = false;
build.AddChild(new Message()
{
Text = "Exception occurred during build:"
});
build.AddChild(new Error()
{
Text = ex.ToString()
});
return build;
}
}));
}
private void OnOTAGBuildProgressChanged(OneTakesAllGroup.BuildProgress groupBuildProgress)
{
//Prepare readout layer version
BuildProgress buildProgress = new BuildProgress(Math.Min(_buildReadoutUnitIdx + 1, ReadoutLayerCfg.ReadoutUnitsCfg.ReadoutUnitCfgCollection.Count),
ReadoutLayerCfg.ReadoutUnitsCfg.ReadoutUnitCfgCollection.Count,
null,
Math.Min(_buildOTAGroupIdx + 1, _oneTakesAllGroupCollection.Length),
_oneTakesAllGroupCollection.Length,
groupBuildProgress
);
//Raise event
RLBuildProgressChanged?.Invoke(buildProgress);
return;
}
private async void OpenLogFile(string filePath)
{
if (!File.Exists(filePath))
{
return;
}
DisplayBuild(null);
this.logFilePath = filePath;
SettingsService.AddRecentLogFile(filePath);
UpdateRecentItemsMenu();
Title = DefaultTitle + " - " + filePath;
var progress = new BuildProgress();
progress.ProgressText = "Opening " + filePath + "...";
SetContent(progress);
bool shouldAnalyze = true;
Build build = await System.Threading.Tasks.Task.Run(() =>
{
try
{
return(Serialization.Read(filePath));
}
catch (Exception ex)
{
ex = ExceptionHandler.Unwrap(ex);
shouldAnalyze = false;
return(GetErrorBuild(filePath, ex.ToString()));
}
});
if (build == null)
{
build = GetErrorBuild(filePath, "");
shouldAnalyze = false;
}
if (shouldAnalyze)
{
progress.ProgressText = "Analyzing " + filePath + "...";
await System.Threading.Tasks.Task.Run(() => BuildAnalyzer.AnalyzeBuild(build));
}
DisplayBuild(build);
}
//Methods
/// <summary>
/// The default implementation of the build process controler.
/// </summary>
protected BuildInstr DefaultNetworkBuildController(BuildProgress buildProgress)
{
BuildInstr instructions = new BuildInstr
{
CurrentIsBetter = IsBetter(buildProgress.CurrNetwork,
buildProgress.BestNetwork
),
StopProcess = (buildProgress.CurrNetwork.HasBinErrorStats &&
buildProgress.BestNetwork.TrainingBinErrorStat.TotalErrStat.Sum == 0 &&
buildProgress.BestNetwork.TestingBinErrorStat.TotalErrStat.Sum == 0 &&
buildProgress.CurrNetwork.CombinedPrecisionError > buildProgress.BestNetwork.CombinedPrecisionError
)
};
return(instructions);
}
/// <summary>
/// Run the <see cref="BuildPipeline"/> of this <see cref="BuildConfiguration"/> to build the target.
/// </summary>
/// <returns>The result of the <see cref="BuildPipeline"/> build.</returns>
public BuildPipelineResult Build()
{
var pipeline = GetBuildPipeline();
if (!CanBuild(out var reason))
{
return(BuildPipelineResult.Failure(pipeline, this, reason));
}
var what = !string.IsNullOrEmpty(name) ? $" {name}" : string.Empty;
using (var progress = new BuildProgress($"Building{what}", "Please wait..."))
{
return(pipeline.Build(this, progress));
}
}
private async void BuildCore(string projectFilePath, string customArguments)
{
var progress = new BuildProgress()
{
IsIndeterminate = true
};
progress.ProgressText = $"Building {projectFilePath}...";
SetContent(progress);
var buildHost = new HostedBuild(projectFilePath, customArguments);
Build result = await buildHost.BuildAndGetResult(progress);
progress.ProgressText = "Analyzing build...";
await QueueAnalyzeBuild(result);
DisplayBuild(result);
}
internal void GenerateSolution()
{
using (var progress = new BuildProgress(k_WindowTitle, "Please wait..."))
{
var settingsDirectory = BeeRootDirectory.Combine("settings").ToString();
if (Directory.Exists(settingsDirectory))
{
Directory.Delete(settingsDirectory, true);
}
foreach (var item in GetRows().OfType <RootAssemblyViewItem>())
{
foreach (var configItem in item.children.OfType <ConfigViewItem>().Where(child => child.IncludeInSolution))
{
progress.Title = $"Generating '{configItem.BuildTargetName}'";
BuildConfiguration buildConfig = null;
if (configItem is BuildConfigViewItem buildConfigItem)
{
buildConfig = buildConfigItem.BuildConfiguration;
}
#if DOTS_TEST_RUNNER
else if (configItem is RuntimeTestTargetConfigViewItem runtimeTestTargetConfigViewItem)
{
buildConfig =
DotsTestRunner.GenerateBuildConfiguration(runtimeTestTargetConfigViewItem
.RuntimeTestTarget);
DotsTestRunner.HackPreRunBuildConfigFixup(buildConfig);
}
#endif
else
{
throw new Exception("Invalid row type in generate solution window view!");
}
GenerateDotsSolutionWindow.GenerateSolution(buildConfig, progress, configItem.BuildTargetName);
}
}
RunBeeProjectFiles(progress);
}
#if DOTS_TEST_RUNNER
DotsTestRunner.ClearTestBuildConfigs();
#endif
}
private void OnNetworkBuildProgressChanged(TNRNetBuilder.BuildProgress netBuildProgress)
{
//Prepare cluster version
BuildProgress buildProgress = new BuildProgress(_clusterName,
Math.Min(_repetitionIdx + 1, _crossvalidationCfg.Repetitions),
_crossvalidationCfg.Repetitions,
Math.Min(_testingFoldIdx + 1, _numOfFoldsPerRepetition),
_numOfFoldsPerRepetition,
Math.Min(_netCfgIdx + 1, _clusterCfg.ClusterNetConfigurations.Count),
_clusterCfg.ClusterNetConfigurations.Count,
netBuildProgress
);
//Raise event
ClusterBuildProgressChanged?.Invoke(buildProgress);
return;
}
public Task <Build> BuildAndGetResult(BuildProgress progress)
{
var msbuildExe = GetMSBuildExe();
var prefixArguments = GetPrefixArguments(projectFilePath);
var postfixArguments = GetPostfixArguments();
// the command line we display to the user should contain the full path to msbuild.exe
var commandLine = $@"{prefixArguments} {customArguments} {postfixArguments}";
progress.MSBuildCommandLine = commandLine;
// the command line we pass to Process.Start doesn't need msbuild.exe
commandLine = $@"""{projectFilePath}"" {customArguments} {postfixArguments}";
return(System.Threading.Tasks.Task.Run(() =>
{
try
{
var arguments = commandLine;
var processStartInfo = new ProcessStartInfo(msbuildExe, arguments);
processStartInfo.WorkingDirectory = Path.GetDirectoryName(projectFilePath);
var process = Process.Start(processStartInfo);
process.WaitForExit();
var build = XmlLogReader.ReadFromXml(xmlLogFile);
File.Delete(xmlLogFile);
return build;
}
catch (Exception ex)
{
var build = new Build();
build.Succeeded = false;
build.AddChild(new Message()
{
Text = "Exception occurred during build:"
});
build.AddChild(new Error()
{
Text = ex.ToString()
});
return build;
}
}));
}
/// <summary>
/// Monitors the progress of a job.
/// </summary>
public async Task <BuildProgress> MonitorProgressAsync(
string classroomName,
string projectName,
int userId)
{
var project = await LoadProjectAsync(classroomName, projectName);
var asyncEnumerable = GetCommitsDescending(project, userId).ToAsyncEnumerable();
var asyncEnumerator = asyncEnumerable.GetEnumerator();
while (await asyncEnumerator.MoveNext())
{
var commit = asyncEnumerator.Current;
if (commit.Build != null)
{
return(BuildProgress.CompletedBuild());
}
if (commit.BuildJobId != null)
{
var jobStatus = await _jobQueueClient
.GetJobStatusAsync(commit.BuildJobId);
if (jobStatus.State == JobState.NotStarted)
{
return(BuildProgress.EnqueuedBuild());
}
else if (jobStatus.State == JobState.InProgress)
{
var duration = _timeProvider.UtcNow - jobStatus.EnteredState;
return(BuildProgress.InProgressBuild(duration));
}
else
{
return(BuildProgress.UnknownBuild());
}
}
}
return(null);
}
private async void BuildCore(string projectFilePath, string customArguments, string searchText = null)
{
var progress = new BuildProgress {
IsIndeterminate = true
};
progress.ProgressText = $"Building {projectFilePath}...";
SetContent(progress);
var buildHost = new HostedBuild(projectFilePath, customArguments);
Build result = await buildHost.BuildAndGetResult(progress);
progress.ProgressText = "Analyzing build...";
await QueueAnalyzeBuild(result);
DisplayBuild(result);
if (searchText != null && CurrentBuildControl != null)
{
CurrentBuildControl.InitialSearchText = searchText;
}
}
private void OnReadoutUnitBuildProgressChanged(ReadoutUnit.BuildProgress unitBuildProgress)
{
int maxNumOfGroups = 0;
if (_oneTakesAllGroupCollection != null)
{
maxNumOfGroups = _oneTakesAllGroupCollection.Length;
}
//Prepare readout layer version
BuildProgress buildProgress = new BuildProgress(Math.Min(_buildReadoutUnitIdx + 1, ReadoutLayerCfg.ReadoutUnitsCfg.ReadoutUnitCfgCollection.Count),
ReadoutLayerCfg.ReadoutUnitsCfg.ReadoutUnitCfgCollection.Count,
unitBuildProgress,
0,
maxNumOfGroups,
null
);
//Raise event
RLBuildProgressChanged?.Invoke(buildProgress);
return;
}
static void GenerateSolution()
{
using (var progress = new BuildProgress(k_Title, "Please wait..."))
{
var result = BeeTools.Run("ProjectFiles", new DotsRuntimeBuildProfile().BeeRootDirectory, progress);
if (!result.Succeeded)
{
UnityEngine.Debug.LogError($"{k_Title} failed.\n{result.Error}");
return;
}
var scriptEditor = ScriptEditorUtility.GetExternalScriptEditor();
var projectPath = new NPath(UnityEngine.Application.dataPath).Parent;
var pi = new ProcessStartInfo();
pi.FileName = scriptEditor;
pi.Arguments = $"{projectPath.Combine(projectPath.FileName + "-Dots.sln").InQuotes()}";
var proc = new Process();
proc.StartInfo = pi;
proc.Start();
}
}
private void buildProcessor(BuildProgress progress)
{
switch (progress)
{
case BuildProgress.PickUpBuildAssets:
pickupBuildAssets();
break;
case BuildProgress.NameAssetLabel:
NameAssetLabel();
break;
case BuildProgress.BuildAssetBundle:
BuildAssetBundle();
break;
case BuildProgress.UnNameAssetLabel:
UnNameAssetLabel();
break;
case BuildProgress.ExportAssetBundle:
ExportAssetBundle();
break;
case BuildProgress.UpdateVersionFile:
CheckVersionFile();
UpdateVersionFile();
break;
case BuildProgress.SaveVersionFile:
SaveVersionFile();
break;
case BuildProgress.UploadAssetBundle:
UploadAssetBundle();
break;
}
}
public static BeeRunResult Run(string arguments, DirectoryInfo workingDirectory, BuildProgress progress = null)
{
var command = new StringBuilder();
var output = new StringBuilder();
var beeProgressInfo = Run(arguments, command, output, workingDirectory);
while (beeProgressInfo.MoveNext())
{
if (progress?.Update(beeProgressInfo.Current.Info, beeProgressInfo.Current.Progress) ?? false)
{
beeProgressInfo.Current.Process?.Process?.Kill();
return(new BeeRunResult(-1, command.ToString(), "Build was cancelled."));
}
}
return(new BeeRunResult(beeProgressInfo.Current.ExitCode, command.ToString(), output.ToString()));
}
private void RaiseBuildProgressChanged(BuildProgress progress)
{
BuildProgressChanged?.Invoke(this, progress);
}
/// <summary>
/// Builds the trained network.
/// </summary>
/// <returns>The trained network.</returns>
public TNRNet Build()
{
TNRNet bestNetwork = null;
int bestNetworkAttempt = 0;
int bestNetworkAttemptEpoch = 0;
int currNetworkLastImprovementEpoch = 0;
double currNetworkLastImprovementCombinedPrecisionError = 0d;
double currNetworkLastImprovementCombinedBinaryError = 0d;
//Create network and trainer
NonRecurrentNetUtils.CreateNetworkAndTrainer(_networkCfg,
_trainingBundle.InputVectorCollection,
_trainingBundle.OutputVectorCollection,
_rand,
out INonRecurrentNetwork net,
out INonRecurrentNetworkTrainer trainer
);
//Iterate training cycles
while (trainer.Iteration())
{
//Compute current error statistics after training iteration
//Training data part
TNRNet currNetwork = new TNRNet(_networkName, _networkOutput)
{
Network = net,
TrainerInfoMessage = trainer.InfoMessage,
TrainingErrorStat = net.ComputeBatchErrorStat(_trainingBundle.InputVectorCollection, _trainingBundle.OutputVectorCollection, out List <double[]> trainingComputedOutputsCollection)
};
if (TNRNet.IsBinErrorStatsOutputType(_networkOutput))
{
currNetwork.TrainingBinErrorStat = new BinErrStat(BoolBorder, trainingComputedOutputsCollection, _trainingBundle.OutputVectorCollection);
currNetwork.CombinedBinaryError = currNetwork.TrainingBinErrorStat.TotalErrStat.Sum;
}
currNetwork.CombinedPrecisionError = currNetwork.TrainingErrorStat.ArithAvg;
//Testing data part
currNetwork.TestingErrorStat = net.ComputeBatchErrorStat(_testingBundle.InputVectorCollection, _testingBundle.OutputVectorCollection, out List <double[]> testingComputedOutputsCollection);
currNetwork.CombinedPrecisionError = Math.Max(currNetwork.CombinedPrecisionError, currNetwork.TestingErrorStat.ArithAvg);
if (TNRNet.IsBinErrorStatsOutputType(_networkOutput))
{
currNetwork.TestingBinErrorStat = new BinErrStat(BoolBorder, testingComputedOutputsCollection, _testingBundle.OutputVectorCollection);
currNetwork.CombinedBinaryError = Math.Max(currNetwork.CombinedBinaryError, currNetwork.TestingBinErrorStat.TotalErrStat.Sum);
}
//Restart lastImprovementEpoch when new trainer's attempt started
if (trainer.AttemptEpoch == 1)
{
currNetworkLastImprovementEpoch = trainer.AttemptEpoch;
currNetworkLastImprovementCombinedPrecisionError = currNetwork.CombinedPrecisionError;
if (TNRNet.IsBinErrorStatsOutputType(_networkOutput))
{
currNetworkLastImprovementCombinedBinaryError = currNetwork.CombinedBinaryError;
}
}
//First initialization of the best network
if (bestNetwork == null)
{
bestNetwork = currNetwork.DeepClone();
bestNetworkAttempt = trainer.Attempt;
}
if ((TNRNet.IsBinErrorStatsOutputType(_networkOutput) && currNetwork.CombinedBinaryError < currNetworkLastImprovementCombinedBinaryError) ||
currNetwork.CombinedPrecisionError < currNetworkLastImprovementCombinedPrecisionError
)
{
currNetworkLastImprovementCombinedPrecisionError = currNetwork.CombinedPrecisionError;
if (TNRNet.IsBinErrorStatsOutputType(_networkOutput))
{
currNetworkLastImprovementCombinedBinaryError = currNetwork.CombinedBinaryError;
}
currNetworkLastImprovementEpoch = trainer.AttemptEpoch;
}
//BuildProgress instance
BuildProgress buildProgress = new BuildProgress(_networkName,
trainer.Attempt,
trainer.MaxAttempt,
trainer.AttemptEpoch,
trainer.MaxAttemptEpoch,
currNetwork,
currNetworkLastImprovementEpoch,
bestNetwork,
bestNetworkAttempt,
bestNetworkAttemptEpoch
);
//Call controller
BuildInstr instructions = _controller(buildProgress);
//Better?
if (instructions.CurrentIsBetter)
{
//Adopt current regression unit as a best one
bestNetwork = currNetwork.DeepClone();
bestNetworkAttempt = trainer.Attempt;
bestNetworkAttemptEpoch = trainer.AttemptEpoch;
//Update build progress
buildProgress.BestNetwork = bestNetwork;
buildProgress.BestNetworkAttemptNum = bestNetworkAttempt;
buildProgress.BestNetworkAttemptEpochNum = bestNetworkAttemptEpoch;
}
//Raise notification event
NetworkBuildProgressChanged?.Invoke(buildProgress);
//Process instructions
if (instructions.StopProcess)
{
break;
}
else if (instructions.StopCurrentAttempt)
{
if (!trainer.NextAttempt())
{
break;
}
}
}//while (iteration)
//Create statistics of the best network weights
bestNetwork.NetworkWeightsStat = bestNetwork.Network.ComputeWeightsStat();
return(bestNetwork);
}
// Use this for initialization
void Start()
{
buildprogress = PanelProgress.GetComponent <BuildProgress>();
}
/// <summary>
/// Updates the line index centred around 'filepos'.
/// If 'filepos' is within the current byte range of 'm_line_index' then an incremental search
/// for lines is done in the direction needed to re-centre the line list around 'filepos'.
/// If 'reload' is true a full rebuild of the cache is done</summary>
public void Build(long filepos, bool reload, IList <IFilter> filters)
{
// Incremental updates cannot interrupt full reloads
if (m_reload_in_progress && reload == false)
{
return;
}
m_reload_in_progress = reload;
// Increment the build issue number to cancel any previous builds
Interlocked.Increment(ref m_issue);
var issue = m_issue;
// Make a copy of the current state
var state = m_state;
// The index that 'filepos' would have in 'm_line_index'.
// -1 if 'filepos' is before the current cached range,
// or 'm_line_index.Count' if after
state.m_index_centre = FindIndex(m_line_index, filepos);
state.m_index_count = m_line_index.Count;
// Make a copy of the filters
state.m_filters = new List <IFilter>(filters);
// Start a build in a worker thread
ThreadPool.QueueUserWorkItem(BuildWorker);
void BuildWorker(object _)
{
try
{
// Open a new stream to the log data
using (var src = Src.OpenStream())
{
// Determine new state properties of the file
state.m_fileend = src.Length;
state.m_filepos = Math_.Clamp(filepos, 0, state.m_fileend);
// Do the update
var lines = BuildAsync(ref state, src, reload, Progress);
// Add the lines to the line index
m_dispatcher.BeginInvoke(new Action(() =>
{
MergeResults(state, lines, reload, Progress);
}));
}
}
catch (Exception ex)
{
m_dispatcher.BeginInvoke(new Action(() =>
{
BuildError?.Invoke(this, new BuildErrorEventArgs(ex));
}));
}
finally
{
m_dispatcher.BeginInvoke(new Action(() =>
{
if (!Progress(1, 1))
{
return;
}
m_reload_in_progress = false;
}));
}
}
bool Progress(long scanned, long length)
{
m_dispatcher.BeginInvoke(new Action(() =>
{
if (IsBuildCancelled(issue))
{
return;
}
BuildProgress?.Invoke(this, new BuildProgressEventArgs(scanned, length));
}));
return(!IsBuildCancelled(issue));
}
}
internal EasyAssetBundleBuildProcessor(EasyAssetBundleBuilderData data, BuildProgress start = BuildProgress.PickUpBuildAssets, BuildProgress end = BuildProgress.UploadAssetBundle)
{
Progress = start;
ProgressEnd = end;
buildData = data;
config = data.Config;
assetList = buildData.BuildAssets;
versionAssetList = buildData.VersionAssets;
assetListProcess = ( IAssetBundleList )buildData;
assetExportProcess = ( IAssetBundleExporter )buildData;
assetUploadProcess = ( IAssetBundleUploader )buildData;
}
private async void OpenLogFile(string filePath)
{
if (!File.Exists(filePath))
{
return;
}
DisplayBuild(null);
this.logFilePath = filePath;
SettingsService.AddRecentLogFile(filePath);
UpdateRecentItemsMenu();
Title = DefaultTitle + " - " + filePath;
var progress = new BuildProgress();
progress.ProgressText = "Opening " + filePath + "...";
SetContent(progress);
Build build = await System.Threading.Tasks.Task.Run(() =>
{
return Serialization.Read(filePath);
});
progress.ProgressText = "Analyzing " + filePath + "...";
await System.Threading.Tasks.Task.Run(() => BuildAnalyzer.AnalyzeBuild(build));
DisplayBuild(build);
}
private async void BuildCore(string projectFilePath, string customArguments)
{
var progress = new BuildProgress();
progress.ProgressText = $"Building {projectFilePath}...";
SetContent(progress);
var buildHost = new HostedBuild(projectFilePath, customArguments);
Build result = await buildHost.BuildAndGetResult(progress);
result.Name = projectFilePath;
progress.ProgressText = "Analyzing build...";
await System.Threading.Tasks.Task.Run(() => BuildAnalyzer.AnalyzeBuild(result));
DisplayBuild(result);
}
private async void OpenLogFile(string filePath)
{
if (!File.Exists(filePath))
{
return;
}
DisplayBuild(null);
this.logFilePath = filePath;
SettingsService.AddRecentLogFile(filePath);
UpdateRecentItemsMenu();
Title = filePath + " - " + DefaultTitle;
var progress = new BuildProgress();
progress.Progress.Updated += update =>
{
Dispatcher.InvokeAsync(() =>
{
progress.Value = update.Ratio;
}, DispatcherPriority.Background);
};
progress.ProgressText = "Opening " + filePath + "...";
SetContent(progress);
bool shouldAnalyze = true;
var stopwatch = Stopwatch.StartNew();
Build build = await System.Threading.Tasks.Task.Run(() =>
{
try
{
return(Serialization.Read(filePath, progress.Progress));
}
catch (Exception ex)
{
ex = ExceptionHandler.Unwrap(ex);
shouldAnalyze = false;
return(GetErrorBuild(filePath, ex.ToString()));
}
});
if (build == null)
{
build = GetErrorBuild(filePath, "");
shouldAnalyze = false;
}
if (shouldAnalyze)
{
progress.ProgressText = "Analyzing " + filePath + "...";
await QueueAnalyzeBuild(build);
}
progress.ProgressText = "Rendering tree...";
await Dispatcher.InvokeAsync(() => { }, DispatcherPriority.Loaded); // let the progress message be rendered before we block the UI again
DisplayBuild(build);
var elapsed = stopwatch.Elapsed;
if (currentBuild != null)
{
currentBuild.UpdateBreadcrumb($"Load time: {elapsed}");
}
}
