/// <summary> /// Looks at current list of properties to get latest information /// </summary> /// <param name="getPropLabel">property label</param> /// <param name="property">property info output</param> /// <param name="index">property info index where found if found</param> /// <returns>found property = true</returns> private bool TryGetProperty(string getPropLabel, out ProcessPropertyInfo property, out int index) { index = 0; property = null; if (info.propertyInfos != null) { lock (syncRoot) { // // Look through each known property for one sharing the same label. // foreach (ProcessPropertyInfo p in info.propertyInfos) { if (String.Compare(p.propertyLabel, getPropLabel, StringComparison.OrdinalIgnoreCase) == 0) { // // If found, set output parameter and return true // CopyProp(p, out property); return(true); } index++; } } } return(false); }
/// <summary> /// Constructor - fills in properties /// </summary> /// <param name="uri"></param> /// <param name="infos"></param> /// <param name="blockOnLabel"></param> /// <param name="blockOnVersion"></param> /// <param name="maxBlockTime"></param> /// <param name="getPropLabel"></param> /// <param name="ProcessStatistics"></param> public PropertyRequest(string uri, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { this.infos = infos; this.blockOnLabel = blockOnLabel; this.blockOnVersion = blockOnVersion; this.maxBlockTime = maxBlockTime; this.getPropLabel = getPropLabel; this.ProcessStatistics = ProcessStatistics; this.replyUri = uri; }
/// <summary> /// Copy the information from one ProcessPropertyInfo object to another /// </summary> /// <param name="propertySrc">Source of info</param> /// <param name="propertyDst">destination of info</param> private void CopyProp(ProcessPropertyInfo propertySrc, out ProcessPropertyInfo propertyDst) { propertyDst = new ProcessPropertyInfo(); propertyDst.propertyLabel = propertySrc.propertyLabel; propertyDst.propertyVersion = propertySrc.propertyVersion; propertyDst.propertyString = propertySrc.propertyString; if (propertySrc.propertyBlock != null) { propertyDst.propertyBlock = new byte[propertySrc.propertyBlock.Length]; Array.Copy(propertySrc.propertyBlock, propertyDst.propertyBlock, propertySrc.propertyBlock.Length); } }
public bool SetGetProps(int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics, GetSetPropertyEventHandler handler) { if (this.processTable.ContainsKey(processId)) { if (infos != null && infos.Length > 0) { // Only add for the first property info since we only want to fire completion once per request this.processTable[processId].AddPropertyListener(infos[0].propertyLabel, infos[0].propertyVersion, handler); } else if (getPropLabel != null && getPropLabel.Length > 0) { this.processTable[processId].AddPropertyListener(getPropLabel, 0, handler); } else { DryadLogger.LogError(0, null, "infos and getPropLabel both empty"); return false; } lock (this.processTable[processId].SyncRoot) { if (this.processTable[processId].Dispatcher != null) { if (this.processTable[processId].Dispatcher.SetGetProps(replyUri, processId, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics)) { return true; } } } // Keep returning error to GM and let its fault-tolerance kick in if (dispatcherPool.Count == 0) { DryadLogger.LogCritical(0, null, "All dispatchers are faulted."); } return false; } else { DryadLogger.LogError(0, null, "process id {0} not found in process table", processId); return false; } }
public bool SetGetProps(string replyUri, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { bool faultDispatcher = true; for (int numRetries = 0; numRetries < MaxRetries; numRetries++) { try { if (!Faulted) { return this.m_client.SetGetProps(replyUri, processId, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics); } return false; } catch (FaultException<UnknownProcessError>) { DryadLogger.LogWarning("Set Get Process Properties", "Attempt to get or set properties for unknown process {0} on node {1}", processId, this.m_nodeName); faultDispatcher = false; break; } catch (FaultException<VertexServiceError> vse) { DryadLogger.LogWarning("Set Get Process Properties", "Error setting or getting properties for process {0} on node {1}: {2}", processId, this.m_nodeName, vse.Reason); faultDispatcher = false; break; } catch (TimeoutException te) { DryadLogger.LogWarning("Set Get Process Properties", "Timeout communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, te.ToString()); if (!SafeOpenConnection()) { faultDispatcher = true; break; } } catch (CommunicationException ce) { DryadLogger.LogWarning("Set Get Process Properties", "Error communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, ce.ToString()); if (!SafeOpenConnection()) { faultDispatcher = true; break; } } catch (Exception e) { DryadLogger.LogError(0, e, "Error calling SetGetProps for process {0} on node {1}", processId, m_nodeName); faultDispatcher = false; break; } } if (faultDispatcher) { RaiseFaultedEvent(); } return false; }
/// <summary> /// Set a property and record the version /// </summary> /// <param name="property"></param> /// <param name="newVersion"></param> private void SetProperty(ProcessPropertyInfo property, out ulong newVersion) { DryadLogger.LogMethodEntry(property.propertyLabel); ProcessPropertyInfo oldProperty = null; lock (syncRoot) { int index; if (TryGetProperty(property.propertyLabel, out oldProperty, out index)) { // // If property found in local array, then we are setting a new version of existing property // Copy the new property information into the array // oldProperty.propertyVersion++; newVersion = oldProperty.propertyVersion; if (property.propertyBlock != null && property.propertyBlock.Length > 0) { oldProperty.propertyBlock = property.propertyBlock; } oldProperty.propertyString = property.propertyString; CopyProp(oldProperty, out info.propertyInfos[index]); } else { // // if property not found in local array, then setting a new property // use version 1, unless valid value specified // if (property.propertyVersion == ulong.MaxValue || property.propertyVersion == 0) { property.propertyVersion = 1; } newVersion = property.propertyVersion; // // Create or resize the local info array as necessary and append this property // if (info.propertyInfos == null) { info.propertyInfos = new ProcessPropertyInfo[1]; } else { Array.Resize(ref info.propertyInfos, info.propertyInfos.Length + 1); } info.propertyInfos[info.propertyInfos.Length - 1] = property; } // // If there was a vertex completed event, record the latest vertex status // if (StatusMessageContainsDryadError_VertexCompleted( property.propertyLabel)) { CopyProp(property, out latestVertexStatusReceived); latestVertexStatusReceived.propertyVersion = newVersion; } // // Wake up anyone waiting for a property change by adding a new wait event for this property if needed // if (propertyWaitEvents.ContainsKey(property.propertyLabel) == false) { propertyWaitEvents.Add(property.propertyLabel, new Dictionary<ulong, ManualResetEvent>()); } // // Wake up anyone waiting for this version of the property // if (propertyWaitEvents[property.propertyLabel].ContainsKey(newVersion - 1)) { propertyWaitEvents[property.propertyLabel][newVersion - 1].Set(); } else { propertyWaitEvents[property.propertyLabel].Add(newVersion - 1, new ManualResetEvent(true)); } // // Wake up anyone waiting for any version of this property // if (newVersion > 1) { if (propertyWaitEvents[property.propertyLabel].ContainsKey(0)) { propertyWaitEvents[property.propertyLabel][0].Set(); } } } DryadLogger.LogMethodExit(); }
/// <summary> /// Set all requested properties /// </summary> /// <param name="infos">Properties to set</param> /// <param name="labels">Output - labels of properties set</param> /// <param name="versions">Output - version of properties set</param> private void SetProperties(ProcessPropertyInfo[] infos, out string[] labels, out ulong[] versions) { // // Return null if infos list contains no properties // labels = null; versions = null; if (infos != null && infos.Length > 0) { versions = new ulong[infos.Length]; labels = new string[infos.Length]; for (int i = 0; i < infos.Length; i++) { // // Set each property and update version and label // ulong newVersion = 0; SetProperty(infos[i], out newVersion); versions[i] = newVersion; labels[i] = infos[i].propertyLabel; } } }
/// <summary> /// Copy the information from one ProcessPropertyInfo object to another /// </summary> /// <param name="propertySrc">Source of info</param> /// <param name="propertyDst">destination of info</param> private void CopyProp(ProcessPropertyInfo propertySrc, out ProcessPropertyInfo propertyDst) { propertyDst = new ProcessPropertyInfo(); propertyDst.propertyLabel = propertySrc.propertyLabel; propertyDst.propertyVersion = propertySrc.propertyVersion; propertyDst.propertyString = propertySrc.propertyString; if (propertySrc.propertyBlock != null) { propertyDst.propertyBlock = new byte[ propertySrc.propertyBlock.Length ]; Array.Copy(propertySrc.propertyBlock, propertyDst.propertyBlock, propertySrc.propertyBlock.Length); } }
/// <summary> /// Creates a request for property update from one vertex to another (one vertex is usually GM) /// </summary> /// <param name="replyEpr">Address to send response to</param> /// <param name="infos"></param> /// <param name="blockOnLabel"></param> /// <param name="blockOnVersion"></param> /// <param name="maxBlockTime"></param> /// <param name="getPropLabel"></param> /// <param name="ProcessStatistics"></param> /// <returns>Returns success/failure of thread startup</returns> public bool SetGetProps(string replyEpr, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { DryadLogger.LogMethodEntry(replyEpr, this.DryadId); // // Check if graph manager is the one calling. If so, increment propertyWaiter count // if (ReplyDispatcher.IsGraphMrgUri(replyEpr)) { int n = Interlocked.Increment(ref propertyWaiters); } // // Create new property request with provided parameters and queue up request sending // PropertyRequest req = new PropertyRequest(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics); bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(SetGetPropThreadProc), req); DryadLogger.LogMethodExit(result); return result; }
/// <summary> /// Looks at current list of properties to get latest information /// </summary> /// <param name="getPropLabel">property label</param> /// <param name="property">property info output</param> /// <param name="index">property info index where found if found</param> /// <returns>found property = true</returns> private bool TryGetProperty(string getPropLabel, out ProcessPropertyInfo property, out int index) { index = 0; property = null; if (info.propertyInfos != null) { lock (syncRoot) { // // Look through each known property for one sharing the same label. // foreach (ProcessPropertyInfo p in info.propertyInfos) { if (String.Compare(p.propertyLabel, getPropLabel, StringComparison.OrdinalIgnoreCase) == 0) { // // If found, set output parameter and return true // CopyProp(p, out property); return true; } index ++; } } } return false; }
/// <summary> /// Update properties /// </summary> /// <param name="replyEpr">callback URI</param> /// <param name="processId">vertex process id</param> /// <param name="infos">property information</param> /// <param name="blockOnLabel">property update label</param> /// <param name="blockOnVersion">property update version</param> /// <param name="maxBlockTime">maximum time to wait for update</param> /// <param name="getPropLabel">property to get</param> /// <param name="ProcessStatistics">vertex host process statistics</param> /// <returns>success/failure of property update</returns> bool IDryadVertexService.SetGetProps(string replyEpr, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { DryadLogger.LogMethodEntry(replyEpr, processId); bool success = false; try { // Get the vertex process ID VertexProcess vp = FindByDryadId(processId); if (vp != null) { success = vp.SetGetProps(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics); } else { DryadLogger.LogError(0, null, "Failed to set / get process properties: Unknown process id {0}", processId); } } catch (Exception e) { DryadLogger.LogWarning("Set Or Get Process Properties", "Operation threw exception: {0}", e.ToString()); throw new FaultException<VertexServiceError>(new VertexServiceError("SetGetProps", e.ToString())); } DryadLogger.LogMethodExit(success); return success; }
/// <summary> /// Set a property and record the version /// </summary> /// <param name="property"></param> /// <param name="newVersion"></param> private void SetProperty(ProcessPropertyInfo property, out ulong newVersion) { DryadLogger.LogMethodEntry(property.propertyLabel); ProcessPropertyInfo oldProperty = null; lock (syncRoot) { int index; if (TryGetProperty(property.propertyLabel, out oldProperty, out index)) { // // If property found in local array, then we are setting a new version of existing property // Copy the new property information into the array // oldProperty.propertyVersion++; newVersion = oldProperty.propertyVersion; if (property.propertyBlock != null && property.propertyBlock.Length > 0) { oldProperty.propertyBlock = property.propertyBlock; } oldProperty.propertyString = property.propertyString; CopyProp(oldProperty, out info.propertyInfos[index]); } else { // // if property not found in local array, then setting a new property // use version 1, unless valid value specified // if (property.propertyVersion == ulong.MaxValue || property.propertyVersion == 0) { property.propertyVersion = 1; } newVersion = property.propertyVersion; // // Create or resize the local info array as necessary and append this property // if (info.propertyInfos == null) { info.propertyInfos = new ProcessPropertyInfo[1]; } else { Array.Resize(ref info.propertyInfos, info.propertyInfos.Length + 1); } info.propertyInfos[info.propertyInfos.Length - 1] = property; } // // If there was a vertex completed event, record the latest vertex status // if (StatusMessageContainsDryadError_VertexCompleted(property.propertyLabel)) { CopyProp(property, out latestVertexStatusReceived); latestVertexStatusReceived.propertyVersion = newVersion; } // // Wake up anyone waiting for a property change by adding a new wait event for this property if needed // if (propertyWaitEvents.ContainsKey(property.propertyLabel) == false) { propertyWaitEvents.Add(property.propertyLabel, new Dictionary <ulong, ManualResetEvent>()); } // // Wake up anyone waiting for this version of the property // if (propertyWaitEvents[property.propertyLabel].ContainsKey(newVersion - 1)) { propertyWaitEvents[property.propertyLabel][newVersion - 1].Set(); } else { propertyWaitEvents[property.propertyLabel].Add(newVersion - 1, new ManualResetEvent(true)); } // // Wake up anyone waiting for any version of this property // if (newVersion > 1) { if (propertyWaitEvents[property.propertyLabel].ContainsKey(0)) { propertyWaitEvents[property.propertyLabel][0].Set(); } } } DryadLogger.LogMethodExit(); }
/// <summary> /// Adds specified property to property wait list and waits for it. /// </summary> /// <param name="blockOnLabel">Property label to wait for</param> /// <param name="blockOnVersion">Version of property to wait for</param> /// <param name="maxBlockTime">Time to wait for property</param> /// <returns>False if property was requested but none was returned</returns> private bool BlockOnProperty(string blockOnLabel, ulong blockOnVersion, long maxBlockTime) { DryadLogger.LogMethodEntry(); // // Return true if no label is provided // if (String.IsNullOrEmpty(blockOnLabel)) { DryadLogger.LogMethodExit(true); return(true); } DryadLogger.LogInformation("Block on property", "Label {0} Version {1} maxBlockTime {2}", blockOnLabel, blockOnVersion, maxBlockTime); ProcessPropertyInfo prop = null; // // If the process already exited, don't bother adding a wait event for // this property - if it's not already set it never will be. // lock (syncRoot) { if (!exited) { // // Add this label and version to the wait events list if needed // if (propertyWaitEvents.ContainsKey(blockOnLabel) == false) { propertyWaitEvents.Add(blockOnLabel, new Dictionary <ulong, ManualResetEvent>()); } if (propertyWaitEvents[blockOnLabel].ContainsKey(blockOnVersion) == false) { propertyWaitEvents[blockOnLabel].Add(blockOnVersion, new ManualResetEvent(false)); } } else { DryadLogger.LogInformation("Block on property", "Process {0} already exited, not adding waiter", this.DryadId); } } // todo: We still may want to implement timeouts to deal with deadlocks in the service / host but it hasn't been an issue yet. //if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(new TimeSpan(maxBlockTime), false)) // // Wait forever (or until process exits or is disposed) for the property to be set or interrupted // while (!exited) { try { if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(100, false)) { break; } } catch (ObjectDisposedException) { DryadLogger.LogWarning("Block on property", "Process {0} disposed while waiting for label {1}, version {2}", DryadId, blockOnLabel, blockOnVersion); DryadLogger.LogMethodExit(false); return(false); } } // Did we get the property, or did the process // terminate? int index; if (TryGetProperty(blockOnLabel, out prop, out index)) { // // If a property was successfully returned, return true // if ((blockOnVersion == 0) || (prop.propertyVersion > blockOnVersion)) { DryadLogger.LogMethodExit(true); return(true); } if (state == ProcessState.Completed) { DryadLogger.LogInformation("Block on property", "Vertex completed (wait) requested version:{0} returned version:{1} of label {2}", blockOnVersion, prop.propertyVersion, blockOnLabel); DryadLogger.LogMethodExit(true); return(true); } } // // Return false if property was requested but none was found // DryadLogger.LogMethodExit(false); return(false); }