/// <summary> /// Sends messages to network agents instructing them to create RAPs. /// The messages are sent synchronously in parallel and control is not returned to the /// caller until all network agents have replied. /// </summary> public void InstallRaps() { ValidateState(RateControllerState.Init, "InstallRaps"); ManualResetEvent NetMessagesComplete = NetBeginMessagePairs(netRateController.InstallRaps); const MessageTypes replyType = MessageTypes.MessageTypeRapFsCreateAck; const int typeIndex = (int)replyType; lock (LockPendingReplies[typeIndex]) { foreach (Connection conn in AgentNameToConn.Values) { List<RAP> listOktofsRap = new List<RAP>(); foreach (RAP rap in conn.ListRap) if (rap.LocEndpointDetails.IsOktofsC || rap.LocEndpointDetails.IsOktofsH) listOktofsRap.Add(rap); if (listOktofsRap.Count > 0) { Console.WriteLine("InstallRaps {0} oktofs raps on host {1}", listOktofsRap.Count, conn.HostName); RAP[] arrayRapFsArg = new RAP[listOktofsRap.Count]; for (int i = 0; i < listOktofsRap.Count; i++) arrayRapFsArg[i] = conn.ListRap[i]; MessageRapFsCreate mRapFsCreate = new MessageRapFsCreate(++SeqNo, arrayRapFsArg); SendParallel(conn, mRapFsCreate.Serialize, replyType, mRapFsCreate.SeqNo); } } foreach (Connection conn in IoFlowNameToConn.Values) { if (conn.DictIoFlows.Count == 0) continue; Console.WriteLine("InstallRaps {0} ioflow raps on host {1}", conn.DictIoFlows.Count, conn.HostName); RAP[] arrayRapFsArg = new RAP[conn.DictIoFlows.Count]; int i = 0; foreach (Flow flow in conn.DictIoFlows.Values) arrayRapFsArg[i++] = flow.RapD; MessageRapFsCreate mRapFsCreate = new MessageRapFsCreate(++SeqNo, arrayRapFsArg); SendParallel(conn, mRapFsCreate.SerializeIoFlow, replyType, mRapFsCreate.SeqNo); } WaitForParallelReplies(replyType, Parameters.RAPC_MESSAGE_TIMEOUT_MS); } // lock NetMessagesComplete.WaitOne(); }
private int SerializeRapFsArg(RAP rap, byte[] buffer, int offset, bool isOktofs) { int startOffset = offset; string stringSid = rap.LocEndpointDetails.StringSid; Utils.Int32ToNetBytes((int)rap.Queue.FlowId, buffer, offset); offset += 4; // FlowId. Utils.Int32ToNetBytes((int)stringSid.Length, buffer, offset); offset += 4; // StringSid int shareLen = (isOktofs ? (int)rap.ShareOrVolume.Length : (int)rap.LocEndpointDetails.FileName.Length); Utils.Int32ToNetBytes(shareLen, buffer, offset); offset += 4; // ShareNameCZLen Utils.Int32ToNetBytes(rap.i_index, buffer, offset); offset += 4; // i_index Utils.Int32ToNetBytes(rap.j_index, buffer, offset); offset += 4; // j_index for (int i = 0; i < Parameters.OKTO_VM_NAME_MAX_CHARS; i++) buffer[offset + i] = 0; // ensure chars zero terminated in buffer. int encByteCount = Enc.GetBytes(stringSid, 0, stringSid.Length, buffer, offset); if (encByteCount != stringSid.Length || encByteCount > Parameters.OKTO_VM_NAME_MAX_CHARS) throw new ApplicationException(String.Format("SerializeRapFsArg VM enc({0}) overrun {1}.", encByteCount, rap.LocEndpointDetails.StringSid)); offset += Parameters.OKTO_VM_NAME_MAX_CHARS; for (int i = 0; i < Parameters.OKTO_SHARE_NAME_MAX_CHARS; i++) buffer[offset + i] = 0; // ensure chars zero terminated in buffer. string strShare = (isOktofs ? rap.ShareOrVolume : rap.LocEndpointDetails.FileName); encByteCount = Enc.GetBytes(strShare, 0, shareLen, buffer, offset); if (encByteCount != shareLen || encByteCount > Parameters.OKTO_SHARE_NAME_MAX_CHARS) { Console.WriteLine("SerializeRapFsArg err serializing {0}", rap.ShareOrVolume); throw new ApplicationException(String.Format("SerializeRapFsArg SHARE enc({0}) overrun {1}.", encByteCount, rap.ShareOrVolume.Length)); } offset += Parameters.OKTO_SHARE_NAME_MAX_CHARS; Debug.Assert(offset - startOffset == SIZEOF_RAP_FS_ARG); return SIZEOF_RAP_FS_ARG; }
/// <summary> /// Called to initialize the list of RAPS. /// </summary> /// <param name="listEndpoints">List of (addr,NodeDetails) pairs typically ex ParseConfigFile().</param> /// <returns></returns> public List<RAP> InitListRap(List<Endpoint> listEndpoints) { ValidateState(RateControllerState.Init, "InitTrafficMatrix"); if (listEndpoints.Count == 0) throw new ArgumentException("err: input cannot have zero length."); if (IsMatrixGenerated) throw new ApplicationException("err: InitListRap() can only be called once."); matrixDimension = listEndpoints.Count; // // Setup TCP connection to each network agent and then register with each network agent. // Each row in table represents a VM, so all entries in a row use the same network agent. // foreach (Endpoint ep in listEndpoints) { Connection conn; string serverName = ep.FilterServer; if (ep.IsOktofsH && !AgentNameToConn.TryGetValue(serverName, out conn)) { conn = new Connection(this, serverName, 0, AgentPort); AgentNameToConn.Add(serverName, conn); MessageRegister mRegister = new MessageRegister(++SeqNo, TenantId, AlertVec); const MessageTypes replyType = MessageTypes.MessageTypeRegisterAck; const int typeIndex = (int)replyType; lock (LockPendingReplies[typeIndex]) { // Registration messages actually sent sequentially, but using parallel paradigm. // This because SendSequential will break if interleaved with alerts. SendParallel(conn, mRegister.Serialize, replyType, mRegister.SeqNo); conn.BeginReceive(); WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS); } } } foreach (Endpoint ep in listEndpoints) { Connection conn; string hyperVName = ep.HyperVserver; // // Connections to remote OktofsAgent socket apps. // if ((ep.IsOktofsC || ep.IsOktofsH) && !AgentNameToConn.TryGetValue(hyperVName, out conn)) { conn = new Connection(this, hyperVName, 0, AgentPort); AgentNameToConn.Add(hyperVName, conn); MessageRegister mRegister = new MessageRegister(++SeqNo, TenantId, AlertVec); const MessageTypes replyType = MessageTypes.MessageTypeRegisterAck; const int typeIndex = (int)replyType; lock (LockPendingReplies[typeIndex]) { // Registration messages actually sent sequentially, but using parallel paradigm. // This because SendSequential will break if interleaved with alerts. SendParallel(conn, mRegister.Serialize, replyType, mRegister.SeqNo); conn.BeginReceive(); WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS); } } // // Connections to remote IoFlow socket apps. // if (ep.IsIoFlowD && !IoFlowNameToConn.TryGetValue(hyperVName, out conn)) { conn = new Connection(this, hyperVName, 0, Parameters.IOFLOWAGENT_TCP_PORT_NUMBER); IoFlowNameToConn.Add(hyperVName, conn); MessageRegister mRegister = new MessageRegister(++SeqNo, TenantId, AlertVec); const MessageTypes replyType = MessageTypes.MessageTypeRegisterAck; const int typeIndex = (int)replyType; lock (LockPendingReplies[typeIndex]) { // Registration messages actually sent sequentially, but using parallel paradigm. // This because SendSequential will break if interleaved with alerts. SendParallel(conn, mRegister.Serialize, replyType, mRegister.SeqNo); conn.BeginReceive(); WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS); } } } // // NT account names in the config file need to be translated into SIDs. // This because SIDS in config files are too unfriendly and error-prone. // Names and SIDS from an untrusted NT domain are legit, e.g. VMs in a workgroup. // Such names cannot be resolved locally, so ask the Hyper-V server to do it for us. // If the Hyper-V server is unknown, e.g. H-only config., config must use valid SIDS. // // // Ask Hyper-V servers to translate VM names to SIDS in case VM in untrusted NT domain. // Dictionary<string, Connection> DictIoFlowVmLookup = new Dictionary<string, Connection>(); // Any not resolved by Oktofs agent. foreach (Endpoint nDtls in listEndpoints) { // Nothing to do if config file claims to know the correct string SID. if (nDtls.SidOrAccount.ToUpper().StartsWith("S-1-5-")) continue; // Only attempt SID translation at Hyper-V servers i.e. at point C in stack. if (!nDtls.Tag.ToUpper().Contains("-VM-SHARE-VOL") && !nDtls.Tag.ToUpper().Contains("-VM-FILE-VOL") ) continue; // Maybe we already queried this SID with the given Hyper-V server. string VmKey = MakeVmKey(nDtls.HyperVserver, nDtls.SidOrAccount); if (DictVmNameToSid.ContainsKey(VmKey)) { nDtls.StringSid = DictVmNameToSid[VmKey]; continue; } // Ask remote Hyper-V server for the sid: our DC may not trust VMs NT domain (e.g. workgroup). Connection conn = null; if (AgentNameToConn.TryGetValue(nDtls.HyperVserver, out conn)) { MessageNameToStringSidQuery mQuerySid = new MessageNameToStringSidQuery(++SeqNo, (uint)nDtls.SidOrAccount.Length, nDtls.SidOrAccount); const MessageTypes replyType = MessageTypes.MessageTypeNameToStringSidReply; const int typeIndex = (int)replyType; lock (LockPendingReplies[typeIndex]) { // For now (dbg) VmName to SID translation messages actually sent sequentially. SendParallel(conn, mQuerySid.Serialize, replyType, mQuerySid.SeqNo); WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS); } // Fatal error if SidQuery failed to update DictVmNameToSid. if (!DictVmNameToSid.ContainsKey(VmKey)) throw new ApplicationException(String.Format("Panic: failed to get SID for {0}", VmKey)); continue; } // If we are using IoFlowAgent but not an OktoFsAgent try the IoFlowagent for the sid. if (IoFlowNameToConn.TryGetValue(nDtls.HyperVserver, out conn)) { if (!DictIoFlowVmLookup.ContainsKey(nDtls.SidOrAccount)){ DictIoFlowVmLookup.Add(nDtls.SidOrAccount, conn); } } } // Any unresolved Vm-to-sid lookups are attempted via IoFlow network agent in batches in parallel. if (DictIoFlowVmLookup.Count != 0) { const MessageTypes replyType = MessageTypes.MessageTypeNameToStringSidBatchReply; const int typeIndex = (int)replyType; lock (LockPendingReplies[typeIndex]) { foreach (Connection conn in IoFlowNameToConn.Values) { List<string> listVmNames = new List<string>(); foreach (KeyValuePair<string, Connection> kvp in DictIoFlowVmLookup) if (kvp.Value == conn) listVmNames.Add(kvp.Key); MessageNameToStringSidBatchQuery bq = new MessageNameToStringSidBatchQuery(++SeqNo, listVmNames); SendParallel(conn, bq.Serialize, replyType, bq.SeqNo); } WaitForParallelReplies(replyType, Parameters.DEFAULT_MESSAGE_TIMEOUT_MS); } // lock } // // Set the SID for each RAP. Any trusted account names will be translated to SIDS at agent. // foreach (Endpoint nDtls in listEndpoints) { if (nDtls.SidOrAccount.ToUpper().StartsWith("S-1-5-")) nDtls.StringSid = nDtls.SidOrAccount.ToUpper(); else if (nDtls.Tag.ToUpper().Equals("H-HYPERV-VOL")) nDtls.StringSid = nDtls.SidOrAccount.ToUpper(); else nDtls.StringSid = DictVmNameToSid[MakeVmKey(nDtls.HyperVserver, nDtls.SidOrAccount)]; } // // Initialize RAPs and RAP lookup, and point Connections at each RAP. // More than one Connection may ref a RAP (e.g. Oktofs and IoFlow both). // Note RAPs not yet bound to flows. // ListRap = new List<RAP>(); for (int i = 0; i < listEndpoints.Count; i++) { string locServername = listEndpoints[i].FilterServer; RAP rap = new RAP(listEndpoints[i], listEndpoints[i]); Connection conn = null; if (AgentNameToConn.TryGetValue(locServername, out conn)) conn.ListRap.Add(rap); if (IoFlowNameToConn.TryGetValue(locServername, out conn)) conn.ListRap.Add(rap); DictEpKeyToRap[listEndpoints[i].Key] = rap; ListRap.Add(rap); } Console.WriteLine("InitListRap made {0} RAPs", ListRap.Count); return ListRap; }
public MessageRapFsCreate(uint seqNo, RAP[] arrayRapArg) : base(0, seqNo, (byte)MessageTypes.MessageTypeRapFsCreate) { ArrayRapArg = arrayRapArg; int length = (ArrayRapArg.Length * SIZEOF_RAP_FS_ARG); Debug.Assert(length <= int.MaxValue); Length = (uint)length; }