/// <summary>
/// The partitioned table is a partitioned file.
/// </summary>
/// <param name="statusReporter">Delegate used to report errors.</param>
private void ParsePartitionedFile(StatusReporter statusReporter)
{
this.EstimatedSize = 0;
try
{
if (!File.Exists(this.Uri))
{
this.Error = "File not found";
statusReporter("Cannot find file " + this.Uri, StatusKind.Error);
return;
}
PartitionedFileMetadata pfi = new PartitionedFileMetadata(new UNCPathname(this.Uri));
this.PartitionCount = pfi.NumberOfPartitions;
foreach (var p in pfi.Partitions)
{
StaticPartitionInformation spi = new StaticPartitionInformation(p.Number, p.Size, p.NumberOfReplicas);
this.partitions.Add(spi);
if (spi.PartitionSize >= 0)
{
this.EstimatedSize += spi.PartitionSize;
}
}
}
catch (Exception ex)
{
this.Error = ex.Message;
}
}
public void TestThatRoundRobinLoadBlancingWithOneServerWorks()
{
var server1 = new ServerConfig {
Name = "jat-web01"
};
_settingsDefault.Config.Servers = new[] { server1 };
var loadBalancer = new MockLoadBalancer {
Mode = LbMode.RoundRobin
};
var sequnceManager = new ExecutionSequenceManager(_settingsDefault.Config.Servers, loadBalancer);
sequnceManager.NewRemoteSequence("Test");
var status = new StatusReporter();
//remoteSequence.Execute(status, _settingsDefault, _token);
sequnceManager.Execute(status, _settingsDefault, _token);
Assert.That(loadBalancer.OnlineOfflineSequence.Count, Is.EqualTo(2));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item1, Is.EqualTo("jat-web01"));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item2, Is.EqualTo("offline"));
Assert.That(loadBalancer.OnlineOfflineSequence[1].Item1, Is.EqualTo("jat-web01"));
Assert.That(loadBalancer.OnlineOfflineSequence[1].Item2, Is.EqualTo("online"));
}
public TaskAttemptContextImpl(Configuration conf, TaskAttemptID taskId, StatusReporter
reporter)
: base(conf, taskId.GetJobID())
{
this.taskId = taskId;
this.reporter = reporter;
}
public void TestThatRoundRobinLoadBlancingWithOneServerAndManuelTestWorks()
{
var config = new ConDepConfig {
EnvironmentName = "bogusEnv"
};
var server1 = new ServerConfig {
Name = "jat-web01"
};
config.Servers = new[] { server1 };
var infrastructureSequence = new InfrastructureSequence();
var preOpsSequence = new PreOpsSequence(new WebDeployHandlerMock());
var loadBalancer = new MockLoadBalancer {
Mode = LbMode.RoundRobin
};
var remoteSequence = new RemoteSequence(infrastructureSequence, preOpsSequence, config.Servers, loadBalancer);
var status = new StatusReporter();
remoteSequence.Execute(status, new ConDepOptions(false, "", false, true, true, false, null));
Assert.That(loadBalancer.OnlineOfflineSequence.Count, Is.EqualTo(1));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item1, Is.EqualTo("jat-web01"));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item2, Is.EqualTo("offline"));
}
public DefaultAnthillStatusReporterTests()
{
_randomReporter1 = Substitute.For <StatusReporter>();
_randomReporter2 = Substitute.For <StatusReporter>();
_statusReporter = new DefaultAnthillStatusReporter(_randomReporter1, _randomReporter2);
}
/// <summary>
/// Clean up any resources being used.
/// </summary>
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
protected override void Dispose(bool disposing)
{
if (disposing && (components != null))
{
components.Dispose();
// If the process has started, all three disposable objects will be valid
// I would just call OnStop, but the code analysis tool doesn't look inside
// it and thinks the members aren't being Disposed.
if (Processors.Count > 0)
{
// Clean up the directory watcher and crash processor threads
foreach (var Processor in Processors)
{
Processor.Dispose();
}
Processors.Clear();
Watcher.Dispose();
Watcher = null;
StatusReporter.Dispose();
StatusReporter = null;
Slack.Dispose();
Slack = null;
// Flush the log to disk
Log.Dispose();
Log = null;
}
}
base.Dispose(disposing);
}
public void DoBattleAction()
{
StatusReporter.WriteLine($"действие запущено");
GameAction = new BattleAction(Castles, HeroUnits);
GameAction.Scene(UserInput, StatusReporter);
StatusReporter.WriteLine($"действие завершено");
}
public Game(GameSettings settings)
{
ExpressionWorker = new UserExpressionWorker();
UserInput = new ConsoleUserInput();
StatusReporter = new StatusReporter();
GameSettings = settings as GameSettings;
}
public void TestThatStickyLoadBlancingWithOneServerAndManuelTestWorks()
{
var server1 = new ServerConfig {
Name = "jat-web01"
};
_settingsStopAfterMarkedServer.Config.Servers = new[] { server1 };
var loadBalancer = new MockLoadBalancer {
Mode = LbMode.Sticky
};
var sequnceManager = new ExecutionSequenceManager(_settingsStopAfterMarkedServer.Config.Servers, loadBalancer);
sequnceManager.NewRemoteSequence("Test");
var status = new StatusReporter();
sequnceManager.Execute(status, _settingsStopAfterMarkedServer, _token);
Assert.That(loadBalancer.OnlineOfflineSequence.Count, Is.EqualTo(1));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item1, Is.EqualTo("jat-web01"));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item2, Is.EqualTo("offline"));
}
/// <summary>
/// Stop the service.
/// </summary>
protected override void OnStop()
{
StatusReporter.OnPreStopping();
// Clean up the directory watcher and crash processor threads
foreach (var Processor in Processors)
{
Processor.RequestStop();
}
foreach (var Processor in Processors)
{
Processor.Dispose();
}
Processors.Clear();
Watcher.Dispose();
Watcher = null;
StatusReporter.Dispose();
StatusReporter = null;
Slack.Dispose();
Slack = null;
// Flush the log to disk
Log.Dispose();
Log = null;
}
private void Cancel(ConDepSettings settings, StatusReporter status, CancellationToken token)
{
Logger.WithLogSection("Cancellation", () =>
{
try
{
var tokenSource = new CancellationTokenSource();
Logger.Warn("Cancelling execution gracefully!");
_cancelled = true;
if (_serverNodeInstalled)
{
ExecutePostOps(settings, status, tokenSource.Token);
}
}
catch (AggregateException aggEx)
{
foreach (var ex in aggEx.InnerExceptions)
{
Logger.Error("Failure during cancellation", ex);
}
}
catch (Exception ex)
{
Logger.Error("Failure during cancellation", ex);
}
});
}
/// <summary>
/// Initializes a new instance of the <see cref="Batch"/> class.
/// </summary>
/// <param name="statusReporter">The status reporter.</param>
/// <param name="logger">The logger.</param>
public Batch(StatusReporter statusReporter, ILogger logger)
{
this.statusReporter = statusReporter;
this.logger = logger;
coverageFiles = new MultiMap <string, string>();
}
/// <summary>
/// Write an exception message to the event log.
/// </summary>
static public void WriteException(string Message)
{
if (Message != null && Message.Length > 2)
{
Log.Print("[EXCEPT] " + Message);
StatusReporter.IncrementCount(StatusReportingEventNames.ExceptionEvent);
}
}
public void UpdateConnectionStatus(int code, string msg)
{
if (this.StatusReporter == null)
{
return;
}
StatusReporter.UpdateConnectionStatus(this.ConnectionState, code, msg);
}
public Batch(NCoverVersion version, StatusReporter statusReporter, ILogger logger)
{
this.version = version;
this.statusReporter = statusReporter;
this.logger = logger;
coverageFiles = new MultiMap <string, string>();
}
public MapContextImpl(Configuration conf, TaskAttemptID taskid, RecordReader <KEYIN
, VALUEIN> reader, RecordWriter <KEYOUT, VALUEOUT> writer, OutputCommitter committer
, StatusReporter reporter, InputSplit split)
: base(conf, taskid, writer, committer, reporter)
{
this.reader = reader;
this.split = split;
}
/// <inheritdoc />
public IDisposable BeginBatch(StatusReporter statusReporter)
{
if (statusReporter == null)
{
throw new ArgumentNullException("statusReporter");
}
return(BeginBatchImpl(statusReporter));
}
public MainWindowVM()
{
Reporter = new StatusReporter(Resources.WpfPlalyer_StartStatus);
_wpfPlayer = new WpfPlayer(VMLocator.GameFieldVM, VMLocator.GameControlsVM, Reporter);
_robotLauncher = new RobotLauncher(VMLocator.GameControlsVM.RobotTime.Value, Reporter);
AttachHandlers();
}
public EggsIncubatingReporterTests()
{
_eggsChecker = Substitute.For <IEggsChecker>();
var eggsCheckerFactory = Substitute.For <IEggsCheckerFactory>();
eggsCheckerFactory.Build().Returns(_eggsChecker);
_reporter = new EggsIncubatingReporter(eggsCheckerFactory);
}
public void TestThatPreCompileExecutesSuccessfully()
{
var operation = new PreCompileOperation("MyWebApp", @"C:\temp\MyWebApp", @"C:\temp\MyWebAppCompiled", _buildManager.Object);
var status = new StatusReporter();
operation.Execute(status, new ConDepConfig(), new ConDepOptions(false, "", false, false, false, false, null));
Assert.That(status.HasErrors, Is.False);
_buildManager.Verify(manager => manager.PrecompileApplication(It.IsAny <PreCompileCallback>()));
}
/// <summary>
/// Generate a new view of the same partitioned table.
/// </summary>
/// <returns>The new view.</returns>
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <param name="job">Job containing the table.</param>
/// <param name="showCancelled">Show the cancelled vertices.</param>
public StaticPartitionedTableInformation Refresh(DryadLinqJobInfo job, StatusReporter statusReporter, bool showCancelled)
{
if (this.constructorArguments.code != null)
{
return(new StaticPartitionedTableInformation(this.Config, this.UriType, this.Uri, this.constructorArguments.code, statusReporter));
}
else
{
return(StageOutput(job, this.constructorArguments.plan, this.constructorArguments.source, statusReporter, showCancelled));
}
}
public void TestThatPreCompileExecutesSuccessfully()
{
var operation = new PreCompileOperation("MyWebApp", @"C:\temp\MyWebApp", @"C:\temp\MyWebAppCompiled", _buildManager.Object);
var status = new StatusReporter();
operation.Execute(_settingsDefault, _token);
Assert.That(status.HasErrors, Is.False);
_buildManager.Verify(manager => manager.PrecompileApplication(It.IsAny <PreCompileCallback>()));
}
public void TestThatStickyWithContinueAfterManualTestExecutesOnCorrectServers()
{
var config = new ConDepConfig {
EnvironmentName = "bogusEnv"
};
var server1 = new ServerConfig {
Name = "jat-web01"
};
var server2 = new ServerConfig {
Name = "jat-web02"
};
var server3 = new ServerConfig {
Name = "jat-web03"
};
var server4 = new ServerConfig {
Name = "jat-web04"
};
var server5 = new ServerConfig {
Name = "jat-web05"
};
config.Servers = new[] { server1, server2, server3, server4, server5 };
var infrastructureSequence = new InfrastructureSequence();
var preOpsSequence = new PreOpsSequence(new WebDeployHandlerMock());
var loadBalancer = new MockLoadBalancer {
Mode = LbMode.Sticky
};
var remoteSequence = new RemoteSequence(infrastructureSequence, preOpsSequence, config.Servers, loadBalancer);
var status = new StatusReporter();
remoteSequence.Execute(status, new ConDepOptions(false, "", false, true, false, true, null));
Assert.That(loadBalancer.OnlineOfflineSequence.Count, Is.EqualTo(((config.Servers.Count - 1) * 2) + 1));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item1, Is.EqualTo("jat-web01"));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item2, Is.EqualTo("online"));
var serverNumber = 2;
for (int i = 1; i < loadBalancer.OnlineOfflineSequence.Count; i += 2)
{
Assert.That(loadBalancer.OnlineOfflineSequence[i].Item1, Is.EqualTo("jat-web0" + serverNumber));
Assert.That(loadBalancer.OnlineOfflineSequence[i].Item2, Is.EqualTo("offline"));
Assert.That(loadBalancer.OnlineOfflineSequence[i + 1].Item1, Is.EqualTo("jat-web0" + serverNumber));
Assert.That(loadBalancer.OnlineOfflineSequence[i + 1].Item2, Is.EqualTo("online"));
serverNumber++;
}
}
private void OnDispose()
{
bDisposing = true;
// Clean up the directory watcher and crash processor threads
WriteEvent("Shutdown: Stopping ReportProcessors...");
foreach (var Processor in Processors)
{
Processor.RequestStop();
}
WriteEvent("Shutdown: Disposing ReportProcessors...");
foreach (var Processor in Processors)
{
Processor.Dispose();
}
Processors.Clear();
WriteEvent("Shutdown: ReportProcessors stopped and disposed.");
WriteEvent("Shutdown: Disposing ReportWatcher...");
Watcher.Dispose();
Watcher = null;
WriteEvent("Shutdown: ReportWatcher disposed.");
WriteEvent("Shutdown: Writing ReportIndex.");
ReportIndex.WriteToFile();
ReportIndex = null;
WriteEvent("Shutdown: ReportIndex written.");
WriteEvent("Shutdown: Disposing AmazonClients...");
OutputAWS.Dispose();
OutputAWS = null;
DataRouterAWS.Dispose();
DataRouterAWS = null;
WriteEvent("Shutdown: AmazonClients disposed.");
WriteEvent("Shutdown: Disposing StatusReporter...");
StatusReporter.Dispose();
StatusReporter = null;
WriteEvent("Shutdown: StatusReporter disposed.");
WriteEvent("Shutdown: Disposing SlackWriter...");
Slack.Dispose();
Slack = null;
WriteEvent("Shutdown: SlackWriter disposed.");
// Flush the log to disk
WriteEvent("Shutdown: Disposing LogWriter.");
Log.Dispose();
Log = null;
bDisposed = true;
}
public void Test()
{
var config = new LoadBalancerConfig
{
Name = "https://10.64.6.74:8443/anm/OperationManager",
UserName = "******",
Password = "******"
};
var loadBalancer = new AceLoadBalancer_Anm_v41(config);
var status = new StatusReporter();
loadBalancer.BringOffline("z63os2swb01-t", "test_env_FARM", LoadBalancerSuspendMethod.Suspend, status);
}
public void TestThatStickyWithContinueAfterManualTestExecutesOnCorrectServers()
{
var server1 = new ServerConfig {
Name = "jat-web01"
};
var server2 = new ServerConfig {
Name = "jat-web02"
};
var server3 = new ServerConfig {
Name = "jat-web03"
};
var server4 = new ServerConfig {
Name = "jat-web04"
};
var server5 = new ServerConfig {
Name = "jat-web05"
};
_settingsContinueAfterMarkedServer.Config.Servers = new[] { server1, server2, server3, server4, server5 };
var loadBalancer = new MockLoadBalancer {
Mode = LbMode.Sticky
};
var sequnceManager = new ExecutionSequenceManager(_settingsContinueAfterMarkedServer.Config.Servers, loadBalancer);
sequnceManager.NewRemoteSequence("Test");
var status = new StatusReporter();
//remoteSequence.Execute(status, _settingsContinueAfterMarkedServer, _token);
sequnceManager.Execute(status, _settingsContinueAfterMarkedServer, _token);
Assert.That(loadBalancer.OnlineOfflineSequence.Count, Is.EqualTo(((_settingsContinueAfterMarkedServer.Config.Servers.Count - 1) * 2) + 1));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item1, Is.EqualTo("jat-web01"));
Assert.That(loadBalancer.OnlineOfflineSequence[0].Item2, Is.EqualTo("online"));
var serverNumber = 2;
for (int i = 1; i < loadBalancer.OnlineOfflineSequence.Count; i += 2)
{
Assert.That(loadBalancer.OnlineOfflineSequence[i].Item1, Is.EqualTo("jat-web0" + serverNumber));
Assert.That(loadBalancer.OnlineOfflineSequence[i].Item2, Is.EqualTo("offline"));
Assert.That(loadBalancer.OnlineOfflineSequence[i + 1].Item1, Is.EqualTo("jat-web0" + serverNumber));
Assert.That(loadBalancer.OnlineOfflineSequence[i + 1].Item2, Is.EqualTo("online"));
serverNumber++;
}
}
/// <inheritdoc />
public object RunIsolatedTask <TIsolatedTask>(HostSetup hostSetup, StatusReporter statusReporter, object[] args)
where TIsolatedTask : IsolatedTask, new()
{
if (hostSetup == null)
{
throw new ArgumentNullException("hostSetup");
}
if (statusReporter == null)
{
throw new ArgumentNullException("statusReporter");
}
return(RunIsolatedTaskImpl <TIsolatedTask>(hostSetup, statusReporter, args));
}
public void TestReport()
{
var reporter = new StatusReporter();
for (var i = 0; i < 32; i++)
{
reporter.RecordUnique();
}
for (var i = 0; i < 4; i++)
{
reporter.RecordDuplicate();
}
using (var writer = new StringWriter())
{
reporter.Report(writer);
Assert.Equal(
"Received 32 unique numbers, 4 duplicates. Unique total: 32",
writer.ToString());
}
Assert.Equal(32, reporter.TotalUnique);
Assert.Equal(4, reporter.TotalDuplicates);
Assert.Equal(0, reporter.IncrementalUnique);
Assert.Equal(0, reporter.IncrementalDuplicates);
for (var i = 0; i < 31; i++)
{
reporter.RecordUnique();
}
for (var i = 0; i < 3; i++)
{
reporter.RecordDuplicate();
}
using (var writer = new StringWriter())
{
reporter.Report(writer);
Assert.Equal(
"Received 31 unique numbers, 3 duplicates. Unique total: 63",
writer.ToString());
}
Assert.Equal(63, reporter.TotalUnique);
Assert.Equal(7, reporter.TotalDuplicates);
Assert.Equal(0, reporter.IncrementalUnique);
Assert.Equal(0, reporter.IncrementalDuplicates);
}
public void TestRecordUnique()
{
var reporter = new StatusReporter();
Assert.Equal(0, reporter.IncrementalUnique);
Assert.Equal(0, reporter.TotalUnique);
for (var i = 0; i < 10; i++)
{
reporter.RecordUnique();
Assert.Equal(i + 1, reporter.IncrementalUnique);
Assert.Equal(i + 1, reporter.TotalUnique);
}
}
/// <summary>
/// Инициациализация игры
/// </summary>
public void Init()
{
Castles = new List <Castle>
{
new Castle("Оплот", new [] { "Кентавр", "Эльф", "Пегас" }),
new Castle("Некрополис", new [] { "Скелет", "Зомби", "Вампир" })
};
HeroUnits = new Units();
StatusReporter.WriteLine("Пользователь должнен выбрать замок, за который он будет играть.");
StatusReporter.WriteLine($"Варианты для выбора:");
Castles.ForEach((castle) => { StatusReporter.WriteLine(castle); });
GenerateUnits();
}
/// <summary>
/// Creates a new instance of this class in preparation for debugging a vertex locally.
/// </summary>
/// <param name="guid">Guid for the vertex to debug.</param>
/// <param name="config">Cluster where job debugged is running.</param>
/// <param name="vertexWorkingDirPath">Path to the (remote) working directory of the vertex.</param>
/// <param name="statusWriter">Used to display status messages nicely.</param>
/// <param name="version">Vertex version to debug.</param>
/// <param name="managed">If true debug managed version.</param>
/// <param name="cpuSampling">If true perform cpu sampling based profiling.</param>
/// <param name="number">Vertex number.</param>
public LocalDebuggingAndProfiling(ClusterConfiguration config,
string guid,
int number,
int version,
IClusterResidentObject vertexWorkingDirPath,
bool managed,
bool cpuSampling,
StatusReporter statusWriter)
{
this.cluster = config;
this.workingDirPath = (vertexWorkingDirPath as UNCFile).Pathname;
this.guid = guid;
this.reporter = statusWriter;
this.cpuSampling = cpuSampling;
this.number = number;
this.version = version;
if (!managed)
throw new Exception("Unmanaged debugging not supported");
}
/// <summary> /// Discover the dryadlinq job given a url from the cluster scheduler. /// (Does not make sense for some cluster architectures, e.g., HPC.) /// </summary> /// <param name="url">URL pointing to the job.</param> /// <returns>The dryadlinq job summary.</returns> /// <param name="reporter">Delegate used to report errors.</param> // ReSharper disable UnusedParameter.Global public abstract DryadLinqJobSummary DiscoverDryadLinqJobFromURL(string url, StatusReporter reporter);
/// <summary> /// Discover the (unique) dryadlinq job corresponding to a cluster job. /// </summary> /// <param name="clusterJob">Cluster Job.</param> /// <returns>The job description.</returns> /// <param name="reporter">Delegate used to report errors.</param> public abstract DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter);
/// <summary>
/// The partitioned table is a partitioned file.
/// </summary>
/// <param name="statusReporter">Delegate used to report errors.</param>
private void ParsePartitionedFile(StatusReporter statusReporter)
{
this.EstimatedSize = 0;
try
{
if (!File.Exists(this.Uri))
{
this.Error = "File not found";
statusReporter("Cannot find file " + this.Uri, StatusKind.Error);
return;
}
PartitionedFileMetadata pfi = new PartitionedFileMetadata(new UNCPathname(this.Uri));
this.PartitionCount = pfi.NumberOfPartitions;
foreach (var p in pfi.Partitions)
{
StaticPartitionInformation spi = new StaticPartitionInformation(p.Number, p.Size, p.NumberOfReplicas);
this.partitions.Add(spi);
if (spi.PartitionSize >= 0)
this.EstimatedSize += spi.PartitionSize;
}
}
catch (Exception ex)
{
this.Error = ex.Message;
}
}
/// <summary>
/// Discover the dryadlinq job given a url from the cluster scheduler.
/// (Does not make sense for some cluster architectures, e.g., HPC.)
/// </summary>
/// <param name="url">URL pointing to the job.</param>
/// <returns>The dryadlinq job summary.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public override DryadLinqJobSummary DiscoverDryadLinqJobFromURL(string url, StatusReporter reporter)
{
throw new InvalidOperationException();
}
/// <summary>
/// Create a class representing the set of edges between two stages in the job plan.
/// These collectively look like a partitioned table.
/// If the source stage has multiple outputs there is not enough information to return meaningful information.
/// </summary>
/// <param name="job">Job whose slice we are displaying.</param>
/// <param name="plan">Static plan of the job.</param>
/// <param name="source">Stage in the job which produces the data.</param>
/// <param name="status">Delegate used to report errors.</param>
/// <param name="showCancelled">If true include cancelled vertices.</param>
public static StaticPartitionedTableInformation StageOutput(
DryadLinqJobInfo job,
DryadJobStaticPlan plan,
DryadJobStaticPlan.Stage source,
StatusReporter status,
bool showCancelled)
{
string header = "Output of " + source.Name;
// First check whether in the static plan this is virtual
while (source.IsTee)
{
var sourceInputs = plan.GetStageConnections(source, true).ToList();
if (sourceInputs.Count() != 1)
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
source = sourceInputs.First().From;
}
// If we reached the input return information about that input
if (source.IsInput)
{
status("Scanning " + source.Name, StatusKind.LongOp);
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status);
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
return result;
}
else
{
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation();
result.Name = "Output of vertices in stage " + source.Name;
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
// Check whether this stage has multiple outputs; this can only happen for 'Fork' operators.
var destinations = plan.GetStageConnections(source, false);
if (destinations.Count() > 1)
{
result.Error = "Cannot provide information about one of multiple outputs of a stage.";
return result;
}
DryadLinqJobStage stage = job.GetStage(source.Name);
if (stage == null)
{
result.Error = "There is no information about the output of stage " + source.Name;
return result;
}
result.EstimatedSize = 0;
result.PartitionCount = stage.TotalInitiatedVertices;
int count = 0;
foreach (ExecutedVertexInstance vi in stage.Vertices)
{
if (vi.State == ExecutedVertexInstance.VertexState.Successful ||
vi.State == ExecutedVertexInstance.VertexState.Failed ||
(showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled))
{
StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version);
result.AddPartition(spi);
if (vi.DataWritten != -1)
result.EstimatedSize += vi.DataWritten;
}
}
return result;
}
}
/// <summary>
/// Parse a log file of the job manager and extract useful information.
/// </summary>
/// <param name="logfile">Log file to parse.</param>
/// <param name="statusReporter">Delegate used to parse errors.</param>
/// <returns>True if parsing succeeds.</returns>
internal bool ParseJMLogFile(IClusterResidentObject logfile, StatusReporter statusReporter)
{
bool success = true;
ISharedStreamReader sr = logfile.GetStream();
if (sr.Exception != null)
{
statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error);
return false;
}
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
if (!line.Contains("DryadProfiler")) continue;
DryadLogEntry le = new DryadLogEntry(line);
if (le.Subsystem != "DryadProfiler") continue;
if (!le.Message.EndsWith("channel status")) continue;
Dictionary<string, string> kvp = Utilities.ParseCommaSeparatedKeyValuePair(le.ExtraInfo);
string verver = kvp["Vertex"];
string[] numbers = verver.Split('.');
int vertex = int.Parse(numbers[0]);
int version = int.Parse(numbers[1]);
ExecutedVertexInstance vi = this.jobVertices.FindVertex(vertex, version);
if (vi == null)
{
// We have overshot the information about the vertices parsed from stdout; stop parsing here
success = false;
break;
}
if (le.Message == "Input channel status")
{
// Vertex=69.0, Name=Merge__446[3], MachPod=sherwood-005:pod1, TotalRead=1470802, TotalReadFromMach=1470802, TotalReadCrossMach=1470802, TotalReadCrossPod=0
long info = long.Parse(kvp["TotalRead"]);
vi.DataRead = info;
}
else if (le.Message == "Output channel status")
{
// Vertex=69.0, Name=Merge__446[3], MachPod=sherwood-005:pod1, TotalWrite=1213418
long info = long.Parse(kvp["TotalWrite"]);
vi.DataWritten = info;
}
}
sr.Close();
return success;
}
/// <summary>
/// Parse the logs generated by the Job Manager and learn more information from them.
/// This function should be called after parsing the stdout.
/// This function is extremely slow; it may be invoked on a background thread.
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <returns>True on success.</returns>
/// <param name="updateProgress">Delegate used to report progress.</param>
/// </summary>
public bool ParseJMLogs(StatusReporter statusReporter, Action<int> updateProgress)
{
IClusterResidentObject dir = this.ClusterConfiguration.ProcessLogDirectory(this.Summary.ManagerProcessGuid, this.ManagerVertex.VertexIsCompleted, this.Summary.Machine, this.Summary);
if (dir.Exception != null)
{
statusReporter("Exception finding logs in " + dir, StatusKind.Error);
return false;
}
string pattern = this.ClusterConfiguration.JMLogFilesPattern(false, this.Summary);
List<IClusterResidentObject> logfiles = dir.GetFilesAndFolders(pattern).ToList();
long totalWork = 0;
foreach (var logfile in logfiles)
{
if (logfile.Size >= 0 && totalWork > 0)
totalWork += logfile.Size;
else
totalWork = -1;
}
bool success = true;
statusReporter(string.Format("Parsing {0} log files", logfiles.Count - this.logFilesParsed), StatusKind.OK);
int currentFile = 0;
bool invalidateCache = false;
foreach (var logfile in logfiles)
{
if (currentFile >= this.logFilesParsed)
{
invalidateCache = true;
success = this.ParseJMLogFile(logfile, statusReporter);
}
if (!success)
// stop at first failure
break;
currentFile++;
updateProgress(100 * currentFile / logfiles.Count);
}
updateProgress(100);
if (invalidateCache)
this.InvalidateCaches();
// reparse the last one again
this.logFilesParsed = currentFile - 1;
return success;
}
/// <summary>
/// Create a class to discover the information about a partitioned table given its uri.
/// </summary>
/// <param name="uri">Partitioned table uri.</param>
/// <param name="uriType">Type of URI.</param>
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <param name="code">Code associated to the stage (the uri does not contain the options, they may still be in the code).</param>
/// <param name="config">Cluster where the job accessing the stream resides.</param>
public StaticPartitionedTableInformation(ClusterConfiguration config, string uriType, string uri, string[] code, StatusReporter statusReporter)
{
this.Config = config;
this.UriType = uriType;
this.Error = "";
this.constructorArguments = new SaveConstructorArguments
{
// this is all we need
code = code
};
// really ugly, but the uri in the table does not longer contain the options; they were stripped by the DryadLINQ compiler.
if (code.Length > 0)
{
string firstline = code[0];
firstline = firstline.Trim('[', ']');
if (firstline.StartsWith("PartitionedTable:"))
firstline = firstline.Substring("PartitionedTable:".Length).Trim();
if (firstline.StartsWith(uri))
uri = firstline; // this may contain the options.
}
int indexoptions = uri.IndexOf("?");
if (indexoptions > 0)
{
this.Uri = uri.Substring(0, indexoptions);
this.Options = uri.Substring(indexoptions + 1);
}
else
{
this.Uri = uri;
this.Options = "";
}
this.Name = Path.GetFileName(this.Uri);
// default values
this.PartitionCount = -1;
this.partitions = new List<StaticPartitionInformation>();
this.EstimatedSize = -1;
this.Header = Path.GetFileName(this.Uri);
switch (uriType)
{
case "PartitionedFile":
this.ParsePartitionedFile(statusReporter);
break;
}
}
/// <summary>
/// Copies all the files matching the 'pattern' from the 'fromDir' to the 'toDir'.
/// </summary>
/// <param name="fromDir">Source directory.</param>
/// <param name="toDir">Destination directory.</param>
/// <param name="yespattern">Cope only the files matching this shell pattern.</param>
/// <returns>Number of copied files, or -1 on failure.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="skipRegex">Skip files matching this regex; if empty string don't skip anything.</param>
private static int copyFiles(string fromDir, string toDir, string yespattern, string skipRegex, StatusReporter reporter)
{
try
{
FileInfo[] fromFiles;
if (yespattern == null)
{
fromFiles = Directory.CreateDirectory(fromDir).GetFiles();
}
else
{
fromFiles = Directory.CreateDirectory(fromDir).GetFiles(yespattern);
}
int copied = 0;
Regex skip = new Regex(skipRegex);
foreach (FileInfo fromFile in fromFiles)
{
if (!string.IsNullOrEmpty(skipRegex) &&
skip.IsMatch(fromFile.Name))
continue;
copied++;
string destFile = Path.Combine(toDir, fromFile.Name);
if (File.Exists(destFile))
{
string bakfile = destFile + ".bak";
FileInfo destFileInfo = new FileInfo(destFile);
if (destFileInfo.LastWriteTime >= fromFile.LastWriteTime
&& !File.Exists(bakfile) // file may have been modified
&& (destFileInfo.Length == fromFile.Length))
{
reporter("Skipping copying " + fromFile.Name + " since it seems to be already present.", StatusKind.OK);
continue;
}
}
reporter("Copying " + fromFile.Name + " (" + string.Format("{0:N0}", fromFile.Length) + " bytes)", StatusKind.LongOp);
fromFile.CopyTo(Path.Combine(toDir, fromFile.Name), true);
}
return copied;
}
catch (Exception ex)
{
reporter("Error during file copy: " + ex.Message, StatusKind.Error);
return -1;
}
}
/// <summary>
/// Check whether the debugging/profiling can be done.
/// </summary>
/// <returns>True if the architecture allows debugging.</returns>
// ReSharper disable once UnusedParameter.Global
internal bool CheckArchitecture(StatusReporter status)
{
return true;
}
/// <summary>
/// Not needed, all summaries are already known.
/// </summary>
/// <param name="clusterJob">Cluster job information.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <returns>Throws an exception.</returns>
public override DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter)
{
throw new InvalidOperationException();
}
/// <summary>
/// Create a channel endpoint description
/// </summary>
/// <param name="isInput">True if the channel endpoint is an input.</param>
/// <param name="number">The input/output number.</param>
/// <param name="uri">URI to channel contents.</param>
/// <param name="uripathprefix">Relative uris will need this prefix appended.</param>
/// <param name="fast">If true the channel size is not computed (this is much faster).</param>
/// <param name="reporter">Delegate used to report errors.</param>
// ReSharper disable once UnusedParameter.Local
public ChannelEndpointDescription(bool isInput, int number, string uri, string uripathprefix, bool fast, StatusReporter reporter)
{
this.IsInput = isInput;
this.Number = number;
int sepindex = uri.IndexOf("://");
if (sepindex < 0)
throw new DryadException("Channel URI " + uri + " does not contain separator ://");
this.UriType = uri.Substring(0, sepindex);
// some HPC URIs use the compression scheme as an "option" (not really defined for file:// uris, but...)
// strip it here
int option = uri.IndexOf('?');
if (option >= 0)
{
uri = uri.Substring(0, option);
}
this.LocalPath = uri.Substring(sepindex + 3);
if (uripathprefix != null) {
// Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here.
this.LocalPath = Path.Combine(uripathprefix, this.LocalPath);
}
if (fast)
this.Size = 0;
else
{
switch (this.UriType)
{
case "file":
if (File.Exists(this.LocalPath))
{
this.Size = new FileInfo(this.LocalPath).Length;
}
else
{
this.Size = -1;
}
break;
default:
this.Size = 0;
break;
}
}
}
/// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
/// </summary>
/// <param name="clusterJob">Cluster Job.</param>
/// <returns>The job description.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public override DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter)
{
DryadLinqJobSummary result = new DryadLinqJobSummary(
clusterJob.Cluster,
this.Config.TypeOfCluster,
"", // virtual cluster
"", // machine
clusterJob.ClusterJobID, // jobId
clusterJob.ClusterJobID, // clusterJobId
new DryadProcessIdentifier("jm"), // jmProcessGuid
clusterJob.Name,
clusterJob.User,
clusterJob.Date,
clusterJob.Date + clusterJob.EstimatedRunningTime,
clusterJob.Status);
return result;
}
/// <summary>
/// Copies all the files from the 'fromDir' to the 'toDir'.
/// </summary>
/// <param name="fromDir">Source directory.</param>
/// <param name="toDir">Destination directory.</param>
/// <returns>Number of copied files, or -1 on failure.</returns>
/// <param name="skipRegex">Skip files whose name matches this regex.</param>
/// <param name="reporter">Delegate used to report errors.</param>
private static int copyFiles(string fromDir, string toDir, string skipRegex, StatusReporter reporter)
{
return copyFiles(fromDir, toDir, null, skipRegex, reporter);
}
/// <summary>
/// Generate a layout suitable for drawing the plan.
/// </summary>
/// <returns>A graph layout.</returns>
/// <param name="statusReporter">Delegate used to report errors.</param>
// ReSharper disable once UnusedParameter.Global
public GraphLayout ComputePlanLayout(StatusReporter statusReporter)
{
IEnumerable<DryadLinqJobStage> stages = this.AllStages().OrderBy(s => s.StartTime).ToList();
if (!stages.Any())
// no layout to compute
return null;
DateTime jobStartTime = this.StartJMTime;
DateTime lastTime = stages.Max(s => s.EndTime);
if (lastTime == jobStartTime)
// avoid the degenerate case
lastTime = jobStartTime + new TimeSpan(0, 0, 1);
GraphLayout result = new GraphLayout((lastTime - jobStartTime).TotalSeconds, stages.Count()*2);
int currentStage = 0;
foreach (DryadLinqJobStage s in stages)
{
// node represents the schedule: horizontal position is starttime - endtime
DateTime endTime = s.EndTime;
DateTime startTime = s.StartTime;
if (endTime <= jobStartTime) // unknown time?
endTime = lastTime; // assume still running
if (startTime <= jobStartTime)
startTime = jobStartTime;
GraphLayout.GraphNode node = new GraphLayout.GraphNode(
(startTime - jobStartTime).TotalSeconds, currentStage*2, (endTime - startTime).TotalSeconds, 1);
node.Shape = GraphLayout.GraphNode.NodeShape.Box;
node.Label = s.Name;
node.Stage = s.Name;
result.Add(node);
currentStage++;
}
return result;
}
/// <summary>
/// Generate a new view of the same partitioned table.
/// </summary>
/// <returns>The new view.</returns>
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <param name="job">Job containing the table.</param>
/// <param name="showCancelled">Show the cancelled vertices.</param>
public StaticPartitionedTableInformation Refresh(DryadLinqJobInfo job, StatusReporter statusReporter, bool showCancelled)
{
if (this.constructorArguments.code != null)
return new StaticPartitionedTableInformation(this.Config, this.UriType, this.Uri, this.constructorArguments.code, statusReporter);
else
return StageOutput(job, this.constructorArguments.plan, this.constructorArguments.source, statusReporter, showCancelled);
}