/// <summary>
/// Looks at current list of properties to get latest information
/// </summary>
/// <param name="getPropLabel">property label</param>
/// <param name="property">property info output</param>
/// <param name="index">property info index where found if found</param>
/// <returns>found property = true</returns>
private bool TryGetProperty(string getPropLabel, out ProcessPropertyInfo property, out int index)
{
index = 0;
property = null;
if (info.propertyInfos != null)
{
lock (syncRoot)
{
//
// Look through each known property for one sharing the same label.
//
foreach (ProcessPropertyInfo p in info.propertyInfos)
{
if (String.Compare(p.propertyLabel, getPropLabel, StringComparison.OrdinalIgnoreCase) == 0)
{
//
// If found, set output parameter and return true
//
CopyProp(p, out property);
return(true);
}
index++;
}
}
}
return(false);
}
/// <summary>
/// Constructor - fills in properties
/// </summary>
/// <param name="uri"></param>
/// <param name="infos"></param>
/// <param name="blockOnLabel"></param>
/// <param name="blockOnVersion"></param>
/// <param name="maxBlockTime"></param>
/// <param name="getPropLabel"></param>
/// <param name="ProcessStatistics"></param>
public PropertyRequest(string uri, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
this.infos = infos;
this.blockOnLabel = blockOnLabel;
this.blockOnVersion = blockOnVersion;
this.maxBlockTime = maxBlockTime;
this.getPropLabel = getPropLabel;
this.ProcessStatistics = ProcessStatistics;
this.replyUri = uri;
}
/// <summary>
/// Copy the information from one ProcessPropertyInfo object to another
/// </summary>
/// <param name="propertySrc">Source of info</param>
/// <param name="propertyDst">destination of info</param>
private void CopyProp(ProcessPropertyInfo propertySrc, out ProcessPropertyInfo propertyDst)
{
propertyDst = new ProcessPropertyInfo();
propertyDst.propertyLabel = propertySrc.propertyLabel;
propertyDst.propertyVersion = propertySrc.propertyVersion;
propertyDst.propertyString = propertySrc.propertyString;
if (propertySrc.propertyBlock != null)
{
propertyDst.propertyBlock = new byte[propertySrc.propertyBlock.Length];
Array.Copy(propertySrc.propertyBlock, propertyDst.propertyBlock, propertySrc.propertyBlock.Length);
}
}
public bool SetGetProps(int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics, GetSetPropertyEventHandler handler)
{
if (this.processTable.ContainsKey(processId))
{
if (infos != null && infos.Length > 0)
{
// Only add for the first property info since we only want to fire completion once per request
this.processTable[processId].AddPropertyListener(infos[0].propertyLabel, infos[0].propertyVersion, handler);
}
else if (getPropLabel != null && getPropLabel.Length > 0)
{
this.processTable[processId].AddPropertyListener(getPropLabel, 0, handler);
}
else
{
DryadLogger.LogError(0, null, "infos and getPropLabel both empty");
return false;
}
lock (this.processTable[processId].SyncRoot)
{
if (this.processTable[processId].Dispatcher != null)
{
if (this.processTable[processId].Dispatcher.SetGetProps(replyUri, processId, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics))
{
return true;
}
}
}
// Keep returning error to GM and let its fault-tolerance kick in
if (dispatcherPool.Count == 0)
{
DryadLogger.LogCritical(0, null, "All dispatchers are faulted.");
}
return false;
}
else
{
DryadLogger.LogError(0, null, "process id {0} not found in process table", processId);
return false;
}
}
public bool SetGetProps(string replyUri, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
bool faultDispatcher = true;
for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
{
try
{
if (!Faulted)
{
return this.m_client.SetGetProps(replyUri, processId, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
}
return false;
}
catch (FaultException<UnknownProcessError>)
{
DryadLogger.LogWarning("Set Get Process Properties", "Attempt to get or set properties for unknown process {0} on node {1}", processId, this.m_nodeName);
faultDispatcher = false;
break;
}
catch (FaultException<VertexServiceError> vse)
{
DryadLogger.LogWarning("Set Get Process Properties", "Error setting or getting properties for process {0} on node {1}: {2}", processId, this.m_nodeName, vse.Reason);
faultDispatcher = false;
break;
}
catch (TimeoutException te)
{
DryadLogger.LogWarning("Set Get Process Properties", "Timeout communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, te.ToString());
if (!SafeOpenConnection())
{
faultDispatcher = true;
break;
}
}
catch (CommunicationException ce)
{
DryadLogger.LogWarning("Set Get Process Properties", "Error communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, ce.ToString());
if (!SafeOpenConnection())
{
faultDispatcher = true;
break;
}
}
catch (Exception e)
{
DryadLogger.LogError(0, e, "Error calling SetGetProps for process {0} on node {1}", processId, m_nodeName);
faultDispatcher = false;
break;
}
}
if (faultDispatcher)
{
RaiseFaultedEvent();
}
return false;
}
/// <summary>
/// Set a property and record the version
/// </summary>
/// <param name="property"></param>
/// <param name="newVersion"></param>
private void SetProperty(ProcessPropertyInfo property, out ulong newVersion)
{
DryadLogger.LogMethodEntry(property.propertyLabel);
ProcessPropertyInfo oldProperty = null;
lock (syncRoot)
{
int index;
if (TryGetProperty(property.propertyLabel, out oldProperty, out index))
{
//
// If property found in local array, then we are setting a new version of existing property
// Copy the new property information into the array
//
oldProperty.propertyVersion++;
newVersion = oldProperty.propertyVersion;
if (property.propertyBlock != null && property.propertyBlock.Length > 0)
{
oldProperty.propertyBlock = property.propertyBlock;
}
oldProperty.propertyString = property.propertyString;
CopyProp(oldProperty, out info.propertyInfos[index]);
}
else
{
//
// if property not found in local array, then setting a new property
// use version 1, unless valid value specified
//
if (property.propertyVersion == ulong.MaxValue || property.propertyVersion == 0)
{
property.propertyVersion = 1;
}
newVersion = property.propertyVersion;
//
// Create or resize the local info array as necessary and append this property
//
if (info.propertyInfos == null)
{
info.propertyInfos = new ProcessPropertyInfo[1];
}
else
{
Array.Resize(ref info.propertyInfos, info.propertyInfos.Length + 1);
}
info.propertyInfos[info.propertyInfos.Length - 1] = property;
}
//
// If there was a vertex completed event, record the latest vertex status
//
if (StatusMessageContainsDryadError_VertexCompleted( property.propertyLabel))
{
CopyProp(property, out latestVertexStatusReceived);
latestVertexStatusReceived.propertyVersion = newVersion;
}
//
// Wake up anyone waiting for a property change by adding a new wait event for this property if needed
//
if (propertyWaitEvents.ContainsKey(property.propertyLabel) == false)
{
propertyWaitEvents.Add(property.propertyLabel, new Dictionary<ulong, ManualResetEvent>());
}
//
// Wake up anyone waiting for this version of the property
//
if (propertyWaitEvents[property.propertyLabel].ContainsKey(newVersion - 1))
{
propertyWaitEvents[property.propertyLabel][newVersion - 1].Set();
}
else
{
propertyWaitEvents[property.propertyLabel].Add(newVersion - 1, new ManualResetEvent(true));
}
//
// Wake up anyone waiting for any version of this property
//
if (newVersion > 1)
{
if (propertyWaitEvents[property.propertyLabel].ContainsKey(0))
{
propertyWaitEvents[property.propertyLabel][0].Set();
}
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Set all requested properties
/// </summary>
/// <param name="infos">Properties to set</param>
/// <param name="labels">Output - labels of properties set</param>
/// <param name="versions">Output - version of properties set</param>
private void SetProperties(ProcessPropertyInfo[] infos, out string[] labels, out ulong[] versions)
{
//
// Return null if infos list contains no properties
//
labels = null;
versions = null;
if (infos != null && infos.Length > 0)
{
versions = new ulong[infos.Length];
labels = new string[infos.Length];
for (int i = 0; i < infos.Length; i++)
{
//
// Set each property and update version and label
//
ulong newVersion = 0;
SetProperty(infos[i], out newVersion);
versions[i] = newVersion;
labels[i] = infos[i].propertyLabel;
}
}
}
/// <summary>
/// Copy the information from one ProcessPropertyInfo object to another
/// </summary>
/// <param name="propertySrc">Source of info</param>
/// <param name="propertyDst">destination of info</param>
private void CopyProp(ProcessPropertyInfo propertySrc, out ProcessPropertyInfo propertyDst)
{
propertyDst = new ProcessPropertyInfo();
propertyDst.propertyLabel = propertySrc.propertyLabel;
propertyDst.propertyVersion = propertySrc.propertyVersion;
propertyDst.propertyString = propertySrc.propertyString;
if (propertySrc.propertyBlock != null)
{
propertyDst.propertyBlock = new byte[ propertySrc.propertyBlock.Length ];
Array.Copy(propertySrc.propertyBlock, propertyDst.propertyBlock, propertySrc.propertyBlock.Length);
}
}
/// <summary>
/// Creates a request for property update from one vertex to another (one vertex is usually GM)
/// </summary>
/// <param name="replyEpr">Address to send response to</param>
/// <param name="infos"></param>
/// <param name="blockOnLabel"></param>
/// <param name="blockOnVersion"></param>
/// <param name="maxBlockTime"></param>
/// <param name="getPropLabel"></param>
/// <param name="ProcessStatistics"></param>
/// <returns>Returns success/failure of thread startup</returns>
public bool SetGetProps(string replyEpr, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
DryadLogger.LogMethodEntry(replyEpr, this.DryadId);
//
// Check if graph manager is the one calling. If so, increment propertyWaiter count
//
if (ReplyDispatcher.IsGraphMrgUri(replyEpr))
{
int n = Interlocked.Increment(ref propertyWaiters);
}
//
// Create new property request with provided parameters and queue up request sending
//
PropertyRequest req = new PropertyRequest(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(SetGetPropThreadProc), req);
DryadLogger.LogMethodExit(result);
return result;
}
/// <summary>
/// Looks at current list of properties to get latest information
/// </summary>
/// <param name="getPropLabel">property label</param>
/// <param name="property">property info output</param>
/// <param name="index">property info index where found if found</param>
/// <returns>found property = true</returns>
private bool TryGetProperty(string getPropLabel, out ProcessPropertyInfo property, out int index)
{
index = 0;
property = null;
if (info.propertyInfos != null)
{
lock (syncRoot)
{
//
// Look through each known property for one sharing the same label.
//
foreach (ProcessPropertyInfo p in info.propertyInfos)
{
if (String.Compare(p.propertyLabel, getPropLabel, StringComparison.OrdinalIgnoreCase) == 0)
{
//
// If found, set output parameter and return true
//
CopyProp(p, out property);
return true;
}
index ++;
}
}
}
return false;
}
/// <summary>
/// Update properties
/// </summary>
/// <param name="replyEpr">callback URI</param>
/// <param name="processId">vertex process id</param>
/// <param name="infos">property information</param>
/// <param name="blockOnLabel">property update label</param>
/// <param name="blockOnVersion">property update version</param>
/// <param name="maxBlockTime">maximum time to wait for update</param>
/// <param name="getPropLabel">property to get</param>
/// <param name="ProcessStatistics">vertex host process statistics</param>
/// <returns>success/failure of property update</returns>
bool IDryadVertexService.SetGetProps(string replyEpr, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
DryadLogger.LogMethodEntry(replyEpr, processId);
bool success = false;
try
{
// Get the vertex process ID
VertexProcess vp = FindByDryadId(processId);
if (vp != null)
{
success = vp.SetGetProps(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
}
else
{
DryadLogger.LogError(0, null, "Failed to set / get process properties: Unknown process id {0}", processId);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Set Or Get Process Properties", "Operation threw exception: {0}", e.ToString());
throw new FaultException<VertexServiceError>(new VertexServiceError("SetGetProps", e.ToString()));
}
DryadLogger.LogMethodExit(success);
return success;
}
/// <summary>
/// Set a property and record the version
/// </summary>
/// <param name="property"></param>
/// <param name="newVersion"></param>
private void SetProperty(ProcessPropertyInfo property, out ulong newVersion)
{
DryadLogger.LogMethodEntry(property.propertyLabel);
ProcessPropertyInfo oldProperty = null;
lock (syncRoot)
{
int index;
if (TryGetProperty(property.propertyLabel, out oldProperty, out index))
{
//
// If property found in local array, then we are setting a new version of existing property
// Copy the new property information into the array
//
oldProperty.propertyVersion++;
newVersion = oldProperty.propertyVersion;
if (property.propertyBlock != null && property.propertyBlock.Length > 0)
{
oldProperty.propertyBlock = property.propertyBlock;
}
oldProperty.propertyString = property.propertyString;
CopyProp(oldProperty, out info.propertyInfos[index]);
}
else
{
//
// if property not found in local array, then setting a new property
// use version 1, unless valid value specified
//
if (property.propertyVersion == ulong.MaxValue || property.propertyVersion == 0)
{
property.propertyVersion = 1;
}
newVersion = property.propertyVersion;
//
// Create or resize the local info array as necessary and append this property
//
if (info.propertyInfos == null)
{
info.propertyInfos = new ProcessPropertyInfo[1];
}
else
{
Array.Resize(ref info.propertyInfos, info.propertyInfos.Length + 1);
}
info.propertyInfos[info.propertyInfos.Length - 1] = property;
}
//
// If there was a vertex completed event, record the latest vertex status
//
if (StatusMessageContainsDryadError_VertexCompleted(property.propertyLabel))
{
CopyProp(property, out latestVertexStatusReceived);
latestVertexStatusReceived.propertyVersion = newVersion;
}
//
// Wake up anyone waiting for a property change by adding a new wait event for this property if needed
//
if (propertyWaitEvents.ContainsKey(property.propertyLabel) == false)
{
propertyWaitEvents.Add(property.propertyLabel, new Dictionary <ulong, ManualResetEvent>());
}
//
// Wake up anyone waiting for this version of the property
//
if (propertyWaitEvents[property.propertyLabel].ContainsKey(newVersion - 1))
{
propertyWaitEvents[property.propertyLabel][newVersion - 1].Set();
}
else
{
propertyWaitEvents[property.propertyLabel].Add(newVersion - 1, new ManualResetEvent(true));
}
//
// Wake up anyone waiting for any version of this property
//
if (newVersion > 1)
{
if (propertyWaitEvents[property.propertyLabel].ContainsKey(0))
{
propertyWaitEvents[property.propertyLabel][0].Set();
}
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Adds specified property to property wait list and waits for it.
/// </summary>
/// <param name="blockOnLabel">Property label to wait for</param>
/// <param name="blockOnVersion">Version of property to wait for</param>
/// <param name="maxBlockTime">Time to wait for property</param>
/// <returns>False if property was requested but none was returned</returns>
private bool BlockOnProperty(string blockOnLabel, ulong blockOnVersion, long maxBlockTime)
{
DryadLogger.LogMethodEntry();
//
// Return true if no label is provided
//
if (String.IsNullOrEmpty(blockOnLabel))
{
DryadLogger.LogMethodExit(true);
return(true);
}
DryadLogger.LogInformation("Block on property", "Label {0} Version {1} maxBlockTime {2}", blockOnLabel, blockOnVersion, maxBlockTime);
ProcessPropertyInfo prop = null;
//
// If the process already exited, don't bother adding a wait event for
// this property - if it's not already set it never will be.
//
lock (syncRoot)
{
if (!exited)
{
//
// Add this label and version to the wait events list if needed
//
if (propertyWaitEvents.ContainsKey(blockOnLabel) == false)
{
propertyWaitEvents.Add(blockOnLabel, new Dictionary <ulong, ManualResetEvent>());
}
if (propertyWaitEvents[blockOnLabel].ContainsKey(blockOnVersion) == false)
{
propertyWaitEvents[blockOnLabel].Add(blockOnVersion, new ManualResetEvent(false));
}
}
else
{
DryadLogger.LogInformation("Block on property", "Process {0} already exited, not adding waiter", this.DryadId);
}
}
// todo: We still may want to implement timeouts to deal with deadlocks in the service / host but it hasn't been an issue yet.
//if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(new TimeSpan(maxBlockTime), false))
//
// Wait forever (or until process exits or is disposed) for the property to be set or interrupted
//
while (!exited)
{
try
{
if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(100, false))
{
break;
}
}
catch (ObjectDisposedException)
{
DryadLogger.LogWarning("Block on property", "Process {0} disposed while waiting for label {1}, version {2}", DryadId, blockOnLabel, blockOnVersion);
DryadLogger.LogMethodExit(false);
return(false);
}
}
// Did we get the property, or did the process
// terminate?
int index;
if (TryGetProperty(blockOnLabel, out prop, out index))
{
//
// If a property was successfully returned, return true
//
if ((blockOnVersion == 0) || (prop.propertyVersion > blockOnVersion))
{
DryadLogger.LogMethodExit(true);
return(true);
}
if (state == ProcessState.Completed)
{
DryadLogger.LogInformation("Block on property", "Vertex completed (wait) requested version:{0} returned version:{1} of label {2}", blockOnVersion, prop.propertyVersion, blockOnLabel);
DryadLogger.LogMethodExit(true);
return(true);
}
}
//
// Return false if property was requested but none was found
//
DryadLogger.LogMethodExit(false);
return(false);
}
