/// <summary>
/// Creates an instance of the OktoQueue class.
/// </summary>
/// <param name="flowId">FlowId uniqueue within this tenant.</param>
/// <param name="conn">Connection for comms with the remote network agent.</param>
internal OktoQueue(uint flowId, Connection conn)
{
this.flowId = flowId;
this.conn = conn;
vecLen = 1;
TokSecVec[0] = ulong.MaxValue;
bucketCapacity[0] = ulong.MaxValue;
listRap = new List<RAP>();
conn.ListQueues.Add(this);
costFuncVecRead = new CostFuncVec();
costFuncVecWrite = new CostFuncVec();
}
/// <summary>
/// Send given message on given connection then block until a (n)ack is received.
/// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message is to be sent.</param>
/// <param name="funcPtrSerialize"></param>
/// <returns></returns>
private bool SendSequential(Connection conn, FuncPtrSerialize funcPtrSerialize, uint seqNo)
{
int countBytesToSend = funcPtrSerialize(conn.sendBuffer, 0);
int countBytesSent = conn.Send(conn.sendBuffer, 0, countBytesToSend);
if (countBytesSent != countBytesToSend)
{
Console.WriteLine("SendSynchronous Err: attempt {0} sent {1}.", countBytesToSend, countBytesSent);
return false;
}
int countRecv = conn.Recv(conn.receiveBuffer, 0, (int)MessageAck.TOTAL_MESSAGE_SIZE);
if (countRecv != MessageAck.TOTAL_MESSAGE_SIZE)
{
Console.WriteLine("SendSynchronous Err: attempt {0} recv {1}.", MessageAck.TOTAL_MESSAGE_SIZE, countRecv);
return false;
}
conn.MessageAck.InitFromNetBytes(conn.receiveBuffer, 0);
Console.WriteLine("SendSynchronous rcv MessageAck({0},{1})", conn.MessageAck.SeqNo, conn.MessageAck.Result);
if (conn.MessageAck.SeqNo != seqNo)
{
Console.WriteLine("SendSynchronous Err: SN {0} != expected {1}.", conn.MessageAck.SeqNo, seqNo);
return false;
}
return true;
}
/// <summary>
/// Send given message on given connection, start async receive and immediately return to caller.
/// Caller must hold LockPendingReplies for the reply message type.
/// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message is to be sent.</param>
/// <param name="funcPtrSerialize">Serilaize() method of the message we want to send.</param>
/// <returns>True on success, false on error.</returns>
private bool SendParallel(Connection conn, FuncPtrSerialize funcPtrSerialize, MessageTypes replyType, uint seqNo)
{
DictPendingReplies.Add(seqNo, replyType);
Interlocked.Increment(ref CountPendingReplies[(int)replyType]);
int countBytesToSend = funcPtrSerialize(conn.sendBuffer, 0);
int countBytesSent = conn.Send(conn.sendBuffer, 0, countBytesToSend);
if (countBytesSent != countBytesToSend)
{
Console.WriteLine("SendParallel Err: attempt {0} sent {1}.", countBytesToSend, countBytesSent);
return false;
}
return true;
}
/// <summary>
/// Callback when slave receives MessageRegister.
/// </summary>
/// <param name="tenantId"></param>
/// <param name="alertVec"></param>
public void CallbackMessageRegister(Connection conn, uint tenantId, UInt64 alertVec)
{
Console.WriteLine("CallbackMessageRegister({0},{1},{1:X8})", conn.RemoteEndPoint, tenantId, alertVec);
TenantId = tenantId;
}
public void Close()
{
if (conn != null)
conn.Close();
conn = null;
}
public OktoResultCodes RegisterMessageSupport(Connection conn, FuncPtrSerialize funcPtrSerialize, uint seqNo)
{
int countBytesToSend = funcPtrSerialize(conn.sendBuffer, 0);
int countBytesSent = conn.Send(conn.sendBuffer, 0, countBytesToSend);
if (countBytesSent != countBytesToSend)
{
Console.WriteLine("SendMessageRegister Err: attempt {0} sent {1}.", countBytesToSend, countBytesSent);
return OktoResultCodes.OKTO_RESULT_SOCK_ERROR;
}
int countRecv = conn.Recv(conn.receiveBuffer, 0, (int)MessageAck.TOTAL_MESSAGE_SIZE);
if (countRecv != MessageAck.TOTAL_MESSAGE_SIZE)
{
Console.WriteLine("SendMessageRegister Err: attempt {0} recv {1}.", MessageAck.TOTAL_MESSAGE_SIZE, countRecv);
return OktoResultCodes.OKTO_RESULT_INVALID_MESSAGE_LENGTH;
}
MessageAck ack = MessageAck.CreateFromNetBytes(conn.receiveBuffer, 0);
//Console.WriteLine("SendSynchronous rcv MessageAck({0},{1})", ack.SeqNo, ack.Result);
if (ack.SeqNo != seqNo)
{
Console.WriteLine("SendMessageRegister Err: SN {0} != expected {1}.", ack.SeqNo, seqNo);
return OktoResultCodes.OKTO_RESULT_INVALID_MESSAGE;
}
else if (ack.Result != (uint)OktoResultCodes.OKTO_RESULT_SUCCESS)
{
Console.WriteLine("SendMessageRegister Err: TID {0} not available", TenantId);
}
return (OktoResultCodes)ack.Result;
}
public void ReceiveError(Connection conn, int errNo)
{
string msg = String.Format("BridgeOktofsAgent comms error: conn = {0}, err = {1}", conn.HostName, errNo);
throw new ApplicationException(msg);
}
/// <summary>
/// Called to initialize the list of RAPS.
/// </summary>
/// <param name="listEndpoints">List of (addr,NodeDetails) pairs typically ex ParseConfigFile().</param>
/// <returns></returns>
public List<RAP> InitListRap(List<Endpoint> listEndpoints)
{
ValidateState(RateControllerState.Init, "InitTrafficMatrix");
if (listEndpoints.Count == 0)
throw new ArgumentException("err: input cannot have zero length.");
if (IsMatrixGenerated)
throw new ApplicationException("err: InitListRap() can only be called once.");
matrixDimension = listEndpoints.Count;
//
// Setup TCP connection to each network agent and then register with each network agent.
// Each row in table represents a VM, so all entries in a row use the same network agent.
//
foreach (Endpoint ep in listEndpoints)
{
Connection conn;
string serverName = ep.FilterServer;
if (ep.IsOktofsH && !AgentNameToConn.TryGetValue(serverName, out conn))
{
conn = new Connection(this, serverName, 0, AgentPort);
AgentNameToConn.Add(serverName, conn);
MessageRegister mRegister = new MessageRegister(++SeqNo, TenantId, AlertVec);
const MessageTypes replyType = MessageTypes.MessageTypeRegisterAck;
const int typeIndex = (int)replyType;
lock (LockPendingReplies[typeIndex])
{
// Registration messages actually sent sequentially, but using parallel paradigm.
// This because SendSequential will break if interleaved with alerts.
SendParallel(conn, mRegister.Serialize, replyType, mRegister.SeqNo);
conn.BeginReceive();
WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS);
}
}
}
foreach (Endpoint ep in listEndpoints)
{
Connection conn;
string hyperVName = ep.HyperVserver;
//
// Connections to remote OktofsAgent socket apps.
//
if ((ep.IsOktofsC || ep.IsOktofsH) && !AgentNameToConn.TryGetValue(hyperVName, out conn))
{
conn = new Connection(this, hyperVName, 0, AgentPort);
AgentNameToConn.Add(hyperVName, conn);
MessageRegister mRegister = new MessageRegister(++SeqNo, TenantId, AlertVec);
const MessageTypes replyType = MessageTypes.MessageTypeRegisterAck;
const int typeIndex = (int)replyType;
lock (LockPendingReplies[typeIndex])
{
// Registration messages actually sent sequentially, but using parallel paradigm.
// This because SendSequential will break if interleaved with alerts.
SendParallel(conn, mRegister.Serialize, replyType, mRegister.SeqNo);
conn.BeginReceive();
WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS);
}
}
//
// Connections to remote IoFlow socket apps.
//
if (ep.IsIoFlowD && !IoFlowNameToConn.TryGetValue(hyperVName, out conn))
{
conn = new Connection(this, hyperVName, 0, Parameters.IOFLOWAGENT_TCP_PORT_NUMBER);
IoFlowNameToConn.Add(hyperVName, conn);
MessageRegister mRegister = new MessageRegister(++SeqNo, TenantId, AlertVec);
const MessageTypes replyType = MessageTypes.MessageTypeRegisterAck;
const int typeIndex = (int)replyType;
lock (LockPendingReplies[typeIndex])
{
// Registration messages actually sent sequentially, but using parallel paradigm.
// This because SendSequential will break if interleaved with alerts.
SendParallel(conn, mRegister.Serialize, replyType, mRegister.SeqNo);
conn.BeginReceive();
WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS);
}
}
}
//
// NT account names in the config file need to be translated into SIDs.
// This because SIDS in config files are too unfriendly and error-prone.
// Names and SIDS from an untrusted NT domain are legit, e.g. VMs in a workgroup.
// Such names cannot be resolved locally, so ask the Hyper-V server to do it for us.
// If the Hyper-V server is unknown, e.g. H-only config., config must use valid SIDS.
//
//
// Ask Hyper-V servers to translate VM names to SIDS in case VM in untrusted NT domain.
//
Dictionary<string, Connection> DictIoFlowVmLookup = new Dictionary<string, Connection>(); // Any not resolved by Oktofs agent.
foreach (Endpoint nDtls in listEndpoints)
{
// Nothing to do if config file claims to know the correct string SID.
if (nDtls.SidOrAccount.ToUpper().StartsWith("S-1-5-"))
continue;
// Only attempt SID translation at Hyper-V servers i.e. at point C in stack.
if (!nDtls.Tag.ToUpper().Contains("-VM-SHARE-VOL") &&
!nDtls.Tag.ToUpper().Contains("-VM-FILE-VOL") )
continue;
// Maybe we already queried this SID with the given Hyper-V server.
string VmKey = MakeVmKey(nDtls.HyperVserver, nDtls.SidOrAccount);
if (DictVmNameToSid.ContainsKey(VmKey))
{
nDtls.StringSid = DictVmNameToSid[VmKey];
continue;
}
// Ask remote Hyper-V server for the sid: our DC may not trust VMs NT domain (e.g. workgroup).
Connection conn = null;
if (AgentNameToConn.TryGetValue(nDtls.HyperVserver, out conn))
{
MessageNameToStringSidQuery mQuerySid =
new MessageNameToStringSidQuery(++SeqNo, (uint)nDtls.SidOrAccount.Length, nDtls.SidOrAccount);
const MessageTypes replyType = MessageTypes.MessageTypeNameToStringSidReply;
const int typeIndex = (int)replyType;
lock (LockPendingReplies[typeIndex])
{
// For now (dbg) VmName to SID translation messages actually sent sequentially.
SendParallel(conn, mQuerySid.Serialize, replyType, mQuerySid.SeqNo);
WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS);
}
// Fatal error if SidQuery failed to update DictVmNameToSid.
if (!DictVmNameToSid.ContainsKey(VmKey))
throw new ApplicationException(String.Format("Panic: failed to get SID for {0}", VmKey));
continue;
}
// If we are using IoFlowAgent but not an OktoFsAgent try the IoFlowagent for the sid.
if (IoFlowNameToConn.TryGetValue(nDtls.HyperVserver, out conn))
{
if (!DictIoFlowVmLookup.ContainsKey(nDtls.SidOrAccount)){
DictIoFlowVmLookup.Add(nDtls.SidOrAccount, conn);
}
}
}
// Any unresolved Vm-to-sid lookups are attempted via IoFlow network agent in batches in parallel.
if (DictIoFlowVmLookup.Count != 0)
{
const MessageTypes replyType = MessageTypes.MessageTypeNameToStringSidBatchReply;
const int typeIndex = (int)replyType;
lock (LockPendingReplies[typeIndex])
{
foreach (Connection conn in IoFlowNameToConn.Values)
{
List<string> listVmNames = new List<string>();
foreach (KeyValuePair<string, Connection> kvp in DictIoFlowVmLookup)
if (kvp.Value == conn)
listVmNames.Add(kvp.Key);
MessageNameToStringSidBatchQuery bq =
new MessageNameToStringSidBatchQuery(++SeqNo, listVmNames);
SendParallel(conn, bq.Serialize, replyType, bq.SeqNo);
}
WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS);
} // lock
}
//
// Set the SID for each RAP. Any trusted account names will be translated to SIDS at agent.
//
foreach (Endpoint nDtls in listEndpoints)
{
if (nDtls.SidOrAccount.ToUpper().StartsWith("S-1-5-"))
nDtls.StringSid = nDtls.SidOrAccount.ToUpper();
else if (nDtls.Tag.ToUpper().Equals("H-HYPERV-VOL"))
nDtls.StringSid = nDtls.SidOrAccount.ToUpper();
else
nDtls.StringSid = DictVmNameToSid[MakeVmKey(nDtls.HyperVserver, nDtls.SidOrAccount)];
}
//
// Initialize RAPs and RAP lookup, and point Connections at each RAP.
// More than one Connection may ref a RAP (e.g. Oktofs and IoFlow both).
// Note RAPs not yet bound to flows.
//
ListRap = new List<RAP>();
for (int i = 0; i < listEndpoints.Count; i++)
{
string locServername = listEndpoints[i].FilterServer;
RAP rap = new RAP(listEndpoints[i], listEndpoints[i]);
Connection conn = null;
if (AgentNameToConn.TryGetValue(locServername, out conn))
conn.ListRap.Add(rap);
if (IoFlowNameToConn.TryGetValue(locServername, out conn))
conn.ListRap.Add(rap);
DictEpKeyToRap[listEndpoints[i].Key] = rap;
ListRap.Add(rap);
}
Console.WriteLine("InitListRap made {0} RAPs", ListRap.Count);
return ListRap;
}
public IoFlowAgent(IIoFlowAgentClient client)
{
this.client = client;
conn = new Connection(this);
}
/// <summary>
/// Send given message on given connection.
/// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message is to be sent.</param>
/// <param name="funcPtrSerialize">Serilaize() method of the message we want to send.</param>
/// <returns>True on success, false on error.</returns>
private bool SendSynchronous(Connection conn, FuncPtrSerialize funcPtrSerialize)
{
int countBytesToSend = funcPtrSerialize(conn.sendBuffer, 0);
int countBytesSent = conn.Send(conn.sendBuffer, 0, countBytesToSend);
if (countBytesSent != countBytesToSend)
{
Console.WriteLine("SendSynchronous Err: attempt {0} sent {1}.", countBytesToSend, countBytesSent);
return false;
}
return true;
}
/// <summary>
/// Called by a connection when it has received an intact and complete message in wire-format.
/// Parses the supplied byte-array to generate a typed message for processing.
/// On return from this routine the connection is free to overwrite the buffer contents.
/// /// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message arrived.</param>
/// <param name="buff">Buffer encoding the message.</param>
/// <param name="offset">Offset to start of message in the supplied buffer.</param>
/// <param name="length">Length of message encoding in supplied buffer</param>
public void ReceiveMessage(Connection conn, MessageTypes messageType, byte[] buff, int offset, int length)
{
switch (messageType)
{
case MessageTypes.MessageTypeRegister:
{
MessageRegister msgRegister = MessageRegister.CreateFromNetBytes(buff, offset);
TenantId = msgRegister.TenantId;
client.CallbackMessageRegister(conn, msgRegister.TenantId, msgRegister.AlertVec);
MessageAck ack = new MessageAck(msgRegister.SeqNo,
MessageTypes.MessageTypeRegisterAck,
(uint)OktoResultCodes.OKTO_RESULT_SUCCESS);
SendSynchronous(conn, ack.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeIoFlowCreate:
{
MessageIoFlowCreate msgFlowc = MessageIoFlowCreate.CreateFromNetBytes(buff, offset);
lock (LockLocal)
{
foreach (IoFlowMessageParams flowc in msgFlowc.ListParams)
{
Console.WriteLine("Agent MessageIoFlowCreate({0},{1})", flowc.FlowId, flowc.ParameterString);
if (DictFlowCreateParams.ContainsKey(flowc.FlowId))
throw new ArgumentOutOfRangeException(string.Format("Agent flowc dup FlowId {0}", flowc.FlowId));
DictFlowCreateParams.Add(flowc.FlowId, flowc);
}
}
MessageAck ack = new MessageAck(msgFlowc.SeqNo,
MessageTypes.MessageTypeIoFlowCreateAck,
(uint)OktoResultCodes.OKTO_RESULT_SUCCESS);
SendSynchronous(conn, ack.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeRapFsCreate:
{
MessageRapFsCreate msgRapc = MessageRapFsCreate.CreateFromNetBytes(buff, offset);
lock (LockLocal)
{
foreach (MsgRapArg rapc in msgRapc.ListMsgRapArg)
{
Console.WriteLine("Agent MessageRapFsCreate({0},{1},{2})", rapc.FlowId, rapc.stringSid, rapc.ShareOrVolume);
if (!DictFlowCreateParams.ContainsKey(rapc.FlowId))
throw new ArgumentOutOfRangeException(string.Format("Agent rapc invalid FlowId {0}", rapc.FlowId));
if (DictRapCreateParams.ContainsKey(rapc.FlowId))
throw new ArgumentOutOfRangeException(string.Format("Agent rapc dup FlowId {0}", rapc.FlowId));
if (!DictFlowCreateParams.ContainsKey(rapc.FlowId))
throw new ArgumentOutOfRangeException(string.Format("Agent rapc unmatched FlowId {0}", rapc.FlowId));
DictRapCreateParams.Add(rapc.FlowId, rapc);
}
}
//
// Params look reasonable and FlowCreate and RapCreate match up.
// Now we can invite the client to create its IoFlows.
//
OktoResultCodes result = client.CallbackIoFlowCreate(DictFlowCreateParams, DictRapCreateParams);
MessageAck ack = new MessageAck(msgRapc.SeqNo,
MessageTypes.MessageTypeRapFsCreateAck,
(uint)result);
SendSynchronous(conn, ack.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeStatsZero:
{
MessageStatsZero msgZero = MessageStatsZero.CreateFromNetBytes(buff, offset);
OktoResultCodes result = client.CallbackMessageStatsZero();
MessageAck ack = new MessageAck(msgZero.SeqNo,
MessageTypes.MessageTypeStatsZeroAck,
(uint)result);
SendSynchronous(conn, ack.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeIoFlowUpdate:
{
MessageIoFlowUpdate msgFlowc = MessageIoFlowUpdate.CreateFromNetBytes(buff, offset);
lock (LockLocal)
{
foreach (IoFlowMessageParams flowu in msgFlowc.ListParams)
if (!DictFlowCreateParams.ContainsKey(flowu.FlowId))
throw new ArgumentOutOfRangeException(string.Format("Agent flowu invalid FlowId {0}", flowu.FlowId));
}
OktoResultCodes result = client.CallbackMessageIoFlowUpdate(msgFlowc.ListParams);
MessageAck ack = new MessageAck(msgFlowc.SeqNo,
MessageTypes.MessageTypeIoFlowUpdateAck,
(uint)result);
SendSynchronous(conn, ack.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeTenantDelete:
{
MessageTenantDelete msgTend = MessageTenantDelete.CreateFromNetBytes(buff, offset);
OktoResultCodes result = client.CallbackMessageTenantDelete();
MessageAck ack = new MessageAck(msgTend.SeqNo,
MessageTypes.MessageTypeTenantDeleteAck,
(uint)result);
SendSynchronous(conn, ack.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeIoFlowStatsQuery:
{
MessageIoFlowStatsQuery msgStatsQ = MessageIoFlowStatsQuery.CreateFromNetBytes(buff, offset);
List<IoFlowMessageParams> listStats = client.CallbackMessageIoFlowStatsQuery();
lock (LockLocal)
{
foreach (IoFlowMessageParams stats in listStats)
if (!DictFlowCreateParams.ContainsKey(stats.FlowId))
throw new ArgumentOutOfRangeException(string.Format("Stats reply invalid FlowId {0}", stats.FlowId));
}
MessageIoFlowStatsReply msgStatsReply = new MessageIoFlowStatsReply(msgStatsQ.SeqNo, listStats, TenantId);
SendSynchronous(conn, msgStatsReply.Serialize);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeNameToStringSidBatchQuery:
{
MessageNameToStringSidBatchQuery msgSidBatchQuery =
MessageNameToStringSidBatchQuery.CreateFromNetBytes(buff, offset);
Dictionary<string, string> DictVmNameToSid = new Dictionary<string, string>();
System.Security.Principal.SecurityIdentifier sid = null;
foreach (string vmName in msgSidBatchQuery.ListVmNames)
{
//XXXET: following two lines are for account names of type europe\
sid = VmSID.GetVMnameSid(vmName);
if (sid == null)
{
try
{
NTAccount ntaccount = new NTAccount(vmName);
sid = (SecurityIdentifier)ntaccount.Translate(typeof(SecurityIdentifier));
}
catch (Exception e)
{
Debug.Assert(0 == 1, e.Message);
}
}
Console.WriteLine("MessageTypeNameToStringSidBatchQuery: {0} sid {1}", vmName, sid.ToString());
DictVmNameToSid.Add(vmName, sid.ToString());
}
MessageVmNameToStringSidBatchReply msgSidReply =
new MessageVmNameToStringSidBatchReply(msgSidBatchQuery.SeqNo,
(uint)OktoResultCodes.OKTO_RESULT_SUCCESS,
DictVmNameToSid);
SendSynchronous(conn, msgSidReply.Serialize);
conn.BeginReceive();
break;
}
default:
{
string msg = string.Format("ReceiveMessage: unexpected message type {0}", messageType);
throw new ApplicationException(msg);
}
} // switch
}
/// <summary>
/// Callback when socket reports and error.
/// Typically clear down local state and driver state.
/// </summary>
/// <param name="conn"></param>
/// <param name="errNo"></param>
public void CallbackReceiveError(Connection conn, int errNo)
{
string msg = String.Format("MoiraiSlave CallbackReceiveError TCP comms error: conn = {0}, err = {1}", conn.HostName, errNo);
Reset();
}
/// <summary>
/// Callback when socket reports close.
/// Typically clear down local state and driver state.
/// </summary>
/// <param name="conn"></param>
public void CallbackReceiveClose(Connection conn)
{
Console.WriteLine("MoiraiSlave CallbackReceiveClose: received close (zero bytes) from conn {0}", conn.HostName);
Reset();
}
internal RcWorkItem(RcWorkItemType type, object args, long timerUSecs, Connection connection)
{
Type = type;
Args = args;
TimerUSecs = timerUSecs;
this.connection = connection;
}
/// <summary>
/// Callback invoked by a connection catches a socket exception.
/// </summary>
/// <param name="conn">Connection that caught the socket exception.</param>
/// <param name="sockEx">The socket exception.</param>
public void CatchSocketException(Connection conn, SocketException sockEx)
{
Console.WriteLine("IoFlowAgent caught SocketException({0},{1})", conn.HostName, sockEx.Message);
client.CallbackCatchSocketException(sockEx);
Reset();
}
/// <summary>
/// Callback invoked by a connection catches a socket exception.
/// </summary>
/// <param name="conn">Connection that caught the socket exception.</param>
/// <param name="sockEx">The socket exception.</param>
public void CatchSocketException(Connection conn, SocketException sockEx)
{
string msg = String.Format("OktofsRateController.CatchSocketException({0},{1})", conn.HostName, sockEx.Message);
throw sockEx;
}
/// <summary>
/// Called by a Connection (TCP to a specific network agent) when TCP has closed the connection.
/// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message arrived.</param>
public void ReceiveClose(Connection conn)
{
Console.WriteLine("IoFlowAgent ReceiveClose: received close (zero bytes) from conn {0}", conn.HostName);
client.CallbackReceiveClose(conn);
Reset();
}
public void ReceiveClose(Connection conn)
{
throw new NotImplementedException();
}
/// <summary>
/// Called by a Connection (TCP to a specific network agent) when it encounters a TCP receive error.
/// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message arrived.</param>
public void ReceiveError(Connection conn, int errNo)
{
string msg = String.Format("IoFlowAgent TCP comms error: conn = {0}, err = {1}", conn.HostName, errNo);
client.CallbackReceiveError(conn, errNo);
Reset();
}
public void ReceiveMessage(Connection conn, MessageTypes messageType, byte[] buff, int offset, int length)
{
switch (messageType)
{
case MessageTypes.MessageTypeNameToStringSidReply:
messageVmNameToStringSidReply
= MessageVmNameToStringSidReply.CreateFromNetBytes(conn.receiveBuffer, 0);
autoResetEvent.Set();
break;
default:
throw new NotImplementedException();
}
}
/// <summary>
/// Called by a Connection (TCP to a specific network agent) when TCP has closed the connection.
/// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message arrived.</param>
public void ReceiveClose(Connection conn)
{
Console.WriteLine("RM shutdown: received close (zero bytes) from conn {0}", conn.HostName);
StateTransition(RateControllerState.Fin);
#if gregos // tidy RC shutdown exceptions
//DeleteTenant();
foreach (Connection connection in AgentNameToConn.Values)
if (connection != conn)
conn.Close();
AgentNameToConn = null;
#endif
EventPolicyModuleThreadBlocked.Set();
}
public void CatchSocketException(Connection conn, SocketException sockEx)
{
string msg = String.Format("BridgeOktofsAgent CatchSocketException({0},{1})", conn.HostName, sockEx.Message);
throw sockEx;
}
/// <summary>
/// Called by a connection when it has received an intact and complete message in wire-format.
/// Parses the supplied byte-array to generate a typed message for processing.
/// On return from this routine the connection is free to overwrite the buffer contents.
/// /// </summary>
/// <param name="conn">Connection (think TCP to specific network agent) on which message arrived.</param>
/// <param name="buff">Buffer encoding the message.</param>
/// <param name="offset">Offset to start of message in the supplied buffer.</param>
/// <param name="length">Length of message encoding in supplied buffer</param>
public void ReceiveMessage(Connection conn, MessageTypes messageType, byte[] buff, int offset, int length)
{
bool unblock = false;
switch (messageType)
{
case MessageTypes.MessageTypeAck:
{
conn.MessageAck.InitFromNetBytes(buff, 0);
//Console.WriteLine("SendParallel rcv MessageAck({0},{1})", conn.MessageAck.SeqNo, conn.MessageAck.Result);
if (conn.MessageAck.Result != (uint)OktoResultCodes.OKTO_RESULT_SUCCESS)
{
string msg = String.Format("ACK err: conn {0} ResultCode {1} subytpe {2}",
conn.HostName, conn.MessageAck.Result, (MessageTypes)conn.MessageAck.SubType);
Console.WriteLine(msg);
}
MessageTypes AckSubType = (MessageTypes)conn.MessageAck.SubType;
unblock = DowndatePendingReplies(AckSubType, conn.MessageAck.SeqNo);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeStatsReplyDelta:
{
MessageStatsReplyDelta mStatsDelta = MessageStatsReplyDelta.CreateFromNetBytes(conn.receiveBuffer, 0, conn, this);
//Console.WriteLine("ReceiveRaw MessageStatsReplyDelta({0},{1})", mStatsDelta.SeqNo, mStatsDelta.Result);
ulong intervalUSecs = mStatsDelta.IntervalUSecs;
foreach (KeyValuePair<ushort, FlowStats> kvp in mStatsDelta.dictFlowStats)
{
RAP rap = conn.ListRap[kvp.Key];
FlowStats flowStats = kvp.Value;
rap.SetFlowStats(flowStats);
}
unblock = DowndatePendingReplies(messageType, mStatsDelta.SeqNo);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeNameToStringSidReply:
{
MessageVmNameToStringSidReply mNameToSidReply
= MessageVmNameToStringSidReply.CreateFromNetBytes(conn.receiveBuffer, 0);
string vmKey = conn.HostName + "." + mNameToSidReply.VmName;
lock (DictVmNameToSid)
{
if (!DictVmNameToSid.ContainsKey(vmKey))
DictVmNameToSid.Add(vmKey, mNameToSidReply.SidString);
}
unblock = DowndatePendingReplies(messageType, mNameToSidReply.SeqNo);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeAlert:
{
MessageAlert mAlert = MessageAlert.CreateFromNetBytes(conn.receiveBuffer, 0);
//Console.WriteLine("ReceiveMessage rx alert {0}", mAlert);
// Push message to the sequential work queue for upcall into Policy Module.
RcWorkItem rcWorkItem = new RcWorkItem(RcWorkItemType.Alert, mAlert, 0, conn);
RcWorkQueue.Enqueue(rcWorkItem);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeIoFlowStatsReply:
{
MessageIoFlowStatsReply mIoFlowStats =
MessageIoFlowStatsReply.CreateFromNetBytes(conn.receiveBuffer, 0);
foreach (IoFlowMessageParams stats in mIoFlowStats.ListParams)
conn.DictIoFlows[stats.FlowId].RapD.SetIoFlowStats(stats.ParameterString);
unblock = DowndatePendingReplies(messageType, mIoFlowStats.SeqNo);
conn.BeginReceive();
break;
}
case MessageTypes.MessageTypeNameToStringSidBatchReply:
{
MessageVmNameToStringSidBatchReply mNameToSidReply
= MessageVmNameToStringSidBatchReply.CreateFromNetBytes(conn.receiveBuffer, 0);
lock (DictVmNameToSid)
{
foreach (KeyValuePair<string, string> kvp in mNameToSidReply.DictVmNameToSid)
{
string vmKey = MakeVmKey(conn.hostName, kvp.Key);
if (!DictVmNameToSid.ContainsKey(vmKey))
{
Console.WriteLine("sidBatchReply adding ({0},{1}", vmKey, kvp.Value);
DictVmNameToSid.Add(vmKey, kvp.Value);
}
}
}
unblock = DowndatePendingReplies(messageType, mNameToSidReply.SeqNo);
conn.BeginReceive();
break;
}
default:
{
string msg = string.Format("SendParallel: unexpected message type {0}", messageType);
throw new ApplicationException(msg);
}
//break;
}
}
public bool Connect()
{
string localHost = System.Environment.MachineName;
int oktofsAgentPort = Parameters.OKTOFSAGENT_TCP_PORT_NUMBER;
try
{
const int locPort = 0;
conn = new Connection(this, localHost, locPort, oktofsAgentPort);
return true;
}
catch (Exception except)
{
const string OKTFSAGENT_NOT_RUNNING =
"No connection could be made because the target machine actively refused it";
if (!except.Message.StartsWith(OKTFSAGENT_NOT_RUNNING))
throw;
}
return false;
}
private const int OKTO_FLT_INFO_BUFF_BYTE_LEN = 256; // ref oktofsuser.h
#endregion Fields
#region Methods
public static MessageStatsReplyDelta CreateFromNetBytes(
byte[] buffer,
int offset,
Connection conn,
OktofsRateController rateController)
{
//
// Get header info and lengths of the Device and Stats arrays.
//
MessageStatsReplyDelta msg = new MessageStatsReplyDelta();
msg.Length = (uint)Utils.Int32FromNetBytes(buffer, offset); offset += 4;
msg.SeqNo = (uint)Utils.Int32FromNetBytes(buffer, offset); offset += 4;
msg.MessageType = buffer[offset++];
msg.Result = (uint)Utils.Int32FromNetBytes(buffer, offset); offset += 4;
msg.IntervalUSecs = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
msg.CountDeviceInfo = (ushort)Utils.Int16FromNetBytes(buffer, offset); offset += 2;
msg.CountFlowStats = (ushort)Utils.Int16FromNetBytes(buffer, offset); offset += 2;
//
// Get the array of Device stats.
//
char[] charsToTrim = { '\0' };
for (int i = 0; i < msg.CountDeviceInfo; i++)
{
string deviceName = UnicodeEncoding.Unicode.GetString(buffer, offset, (OKTO_FLT_INFO_BUFF_BYTE_LEN + 2));
offset += (OKTO_FLT_INFO_BUFF_BYTE_LEN + 2);
deviceName = deviceName.TrimEnd(charsToTrim);
Device device = rateController.FindOrCreateDevice(conn.HostName, deviceName);
lock (device)
{
device.AveInFlightTick.Read.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightTick.Read.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightTick.Write.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightTick.Write.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightEvent.Read.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightEvent.Read.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightEvent.Write.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveInFlightEvent.Write.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenTick.Read.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenTick.Read.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenTick.Write.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenTick.Write.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenEvent.Read.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenEvent.Read.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenEvent.Write.Iops = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
device.AveQueueLenEvent.Write.Tokens = (ulong)Utils.Int64FromNetBytes(buffer, offset); offset += 8;
}
}
//
// Get the array of FlowStats.
//
for (int i = 0; i < msg.CountFlowStats; i++)
{
ushort idxRap = (ushort)Utils.Int16FromNetBytes(buffer, offset); offset += 2;
FlowStats flowStats = FlowStats.CreateFromNetBytes(buffer, offset, out offset);
msg.dictFlowStats.Add(idxRap, flowStats);
}
return msg;
}
