//XXXIS add more things if we need them? public EssesFlowCacheContext(Flow flow, UInt64 cacheSizeAllocated, CacheWritePolicy writePolicy, CacheWriteBuffer cacheWrites, string cacheCurvePointsFile, string tenantID) { Flow = flow; this.cacheSizeAllocated = cacheSizeAllocated; this.writePolicy = writePolicy; this.cacheWrites = cacheWrites; this.hitRate = 0.0F; this.tenantID = tenantID; this.demandPointsSerializationFile = cacheCurvePointsFile; if (File.Exists(this.demandPointsSerializationFile)) { readInCacheCurve(); this.useSerializedCurve = true; } else { this.cacheDemandCurvePoints = null; this.cacheDemandFunc = null; this.useSerializedCurve = false; } #if COLLECT_CACHE_CURVE Console.CancelKeyPress += new ConsoleCancelEventHandler(Controller_CancelKeyPress); #endif }
private void processFlowCacheStats(Flow flow, string statsString) { //Stats strings look like: "cacheSizeAllocated={0} cacheSizeUsed={1} cacheAccessesTotal={2} flowBytesAccessed={3} cacheAccessesHits={4} cacheEvictions={5} {6}" EssesFlowCacheContext fCC = (EssesFlowCacheContext)flow.Context; string[] statsStringSeparators = new string[] { " ", "\t" }; char[] tokenSeparator = new char[] { '=' }; string[] statsTokens = statsString.Split(statsStringSeparators, StringSplitOptions.RemoveEmptyEntries); Debug.Assert(statsTokens.Length > 0); //Extract the stats from the string sent by the client slave fCC.cacheSizeAllocated = Convert.ToUInt64(statsTokens[0].Split(tokenSeparator)[1]); fCC.cacheSizeUsed = Convert.ToUInt64(statsTokens[1].Split(tokenSeparator)[1]); UInt64 cacheAccessesNow, flowBytesAccessedNow, cacheAccessesHitsNow, cacheEvictionsNow; cacheAccessesNow = Convert.ToUInt64(statsTokens[2].Split(tokenSeparator)[1]); flowBytesAccessedNow = Convert.ToUInt64(statsTokens[3].Split(tokenSeparator)[1]); cacheAccessesHitsNow = Convert.ToUInt64(statsTokens[4].Split(tokenSeparator)[1]); cacheEvictionsNow = Convert.ToUInt64(statsTokens[5].Split(tokenSeparator)[1]); Debug.Assert(cacheAccessesNow >= fCC.cacheAccessesTotal); Debug.Assert(flowBytesAccessedNow >= fCC.flowBytesAccessedTotal); Debug.Assert(cacheAccessesHitsNow >= fCC.cacheAccessesHitsTotal); if (cacheAllocControlCounter == 0) { fCC.cacheAccessesLastAllocInterval = 0; fCC.flowBytesAccessedLasAllocInterval = 0; fCC.cacheAccessesHitsLastAllocInterval = 0; fCC.cacheEvictionsLastAllocInterval = 0; } else if (cacheAllocControlCounter < cacheAllocControlFreq) { fCC.cacheAccessesLastAllocInterval += (cacheAccessesNow - fCC.cacheAccessesTotal); fCC.flowBytesAccessedLasAllocInterval += (flowBytesAccessedNow - fCC.flowBytesAccessedTotal); fCC.cacheAccessesHitsLastAllocInterval += (cacheAccessesHitsNow - fCC.cacheAccessesHitsTotal); fCC.cacheEvictionsLastAllocInterval += (cacheEvictionsNow - fCC.cacheEvictionsTotal); } fCC.flowBytesAccessedLastSampleInterval = flowBytesAccessedNow - fCC.flowBytesAccessedTotal; fCC.cacheAccessesTotal = cacheAccessesNow; fCC.flowBytesAccessedTotal = flowBytesAccessedNow; fCC.cacheAccessesHitsTotal = cacheAccessesHitsNow; fCC.cacheEvictionsTotal = cacheEvictionsNow; fCC.hitRate = (fCC.cacheAccessesTotal > 0 ? (float)fCC.cacheAccessesHitsTotal / (float)fCC.cacheAccessesTotal : 0.0F); if (statsTokens.Length > 6) //cache demand string is included { string[] cacheDemandPairs = statsTokens[6].Split(';'); Double[] xVals = new Double[cacheDemandPairs.Length]; Double[] yVals = new Double[cacheDemandPairs.Length]; for (int i = 0; i < cacheDemandPairs.Length; i++) { xVals[i] = Convert.ToDouble(cacheDemandPairs[i].Split(',')[0]); yVals[i] = Convert.ToDouble(cacheDemandPairs[i].Split(',')[1]); } if (!fCC.useSerializedCurve) //only save curve if we don't have a curve at the moment { fCC.cacheDemandCurvePoints = new CacheCurvePoints(xVals, yVals); //fCC.cacheDemandFunc = Interpolate.LinearBetweenPoints(xVals, yVals); //fCC.cacheDemandFunc = Interpolate.Linear(xVals, yVals); fCC.cacheDemandFunc = Interpolate.PolynomialEquidistant(xVals, yVals); } #if COLLECT_CACHE_CURVE //////////XXXIS: quick test to see if we got the function fitted properly (since graphing's a pain...) //XXXIS change fileCacheSize accordingly to print the curve on the screen, IN ADDITION to it being collected when you Ctrl+C the //controller at the end of the run UInt64 fileCacheSize = 10737418240; int numCurvePoints = 500; float stepSize = 1.0F / (float)numCurvePoints; Console.WriteLine("Flow ID {0}", flow.FlowId); for (float frac = 0.0F; frac <= (1.0F + stepSize); frac += stepSize) { UInt32 ghostCacheBlockSize = 4096; //Compute and block-align the cache sizes; XXXIS: THIS ASSUMES ghost cache block size is a power of two!!! UInt64 specCacheSize = ((UInt64)(frac * (float)fileCacheSize)) & (~((UInt64)(ghostCacheBlockSize - 1))); Console.WriteLine("{0}, {1}", specCacheSize, fCC.cacheDemandFunc.Interpolate((double)specCacheSize)); //rateController.Log(String.Format("{0}, {1}", specCacheSize, fCC.cacheDemandFunc.Interpolate((double)specCacheSize))); } #endif } //Console.WriteLine("Flow {0}, SizeAllocated {1}, SizeUsed {2}, flowBytesAccessedTotal {3}, AccessesTotal {4}, AccessesHits {5}, Evictions {6}, HitRate {7}", // flow.FlowId, fCC.cacheSizeAllocated, fCC.cacheSizeUsed, fCC.flowBytesAccessedTotal, fCC.cacheAccessesTotal, // fCC.cacheAccessesHitsTotal, fCC.cacheEvictionsTotal, fCC.hitRate); }
/// <summary> /// Parses config file and returns list of Enpoints. /// </summary> /// <param name="fileName">Name of input file.</param> /// <param name="validTags">Acceptable input record types.</param> /// <returns>List containing one Endpoint for each input record.</returns> public List<Endpoint> ParseConfigFile(string fileName, string[] validTags) { List<Endpoint> listEndpoints = null; StreamReader streamIn = new StreamReader(fileName); string inputRecord; string[] separators = new string[] { " ", "\t" }; ValidateState(RateControllerState.Init, "ParseConfigFile"); while ((inputRecord = streamIn.ReadLine()) != null) { string[] inputTokens = inputRecord.Split(separators, StringSplitOptions.RemoveEmptyEntries); if (inputTokens.Length == 0) continue; else if (inputTokens[0].StartsWith(@"#") || inputTokens[0].StartsWith(@"//")) continue; else if (inputTokens[0].ToLower().Equals(Parameters.VOLNAME_ALIAS_REC_HEADER)) continue; else if (inputTokens.Length > 4 && IsValidTag(inputTokens[0], validTags) && !inputTokens[0].StartsWith("-") && (inputTokens[0].ToLower().Contains("-vm-share-vol") || inputTokens[0].ToLower().Contains("-vm-file-vol"))) { // // Parse records. // Each such record generates a Flow containing one or more queue : e.g. one at C and one at H. // string tag = inputTokens[0].ToLower(); string StageIds = tag.Substring(0, tag.IndexOf("-")); bool isOktofsC = StageIds.Contains("c"); bool isOktofsH = StageIds.Contains("h"); bool isIoFlowD = StageIds.Contains("d"); Endpoint endpointC = null; Endpoint endpointH = null; if (listEndpoints == null) listEndpoints = new List<Endpoint>(); string vmName = inputTokens[1]; string hypervName = inputTokens[2]; string shareName = inputTokens[3]; string volName = inputTokens[4]; // Windows keeps breaking our conf files by changing the volume names at H. // Provide an optional alias func based on share name to avoid admin chaos. volName = GetVolumeNameFromShareName(shareName, volName); if (!StageIds.Replace("c", "").Replace("h", "").Replace("d", "").Equals("")) { streamIn.Close(); streamIn = null; throw new ApplicationException(String.Format("input rec illegal prefix {0}", tag)); } if (vmName.Length > Parameters.OKTO_VM_NAME_MAX_CHARS) { streamIn.Close(); streamIn = null; throw new ApplicationException(String.Format("VmName too long {0}", vmName)); } if (shareName.Length > Parameters.OKTO_SHARE_NAME_MAX_CHARS) { streamIn.Close(); streamIn = null; throw new ApplicationException(String.Format("ShareName too long {0}", shareName)); } if (volName.Length > Parameters.OKTO_SHARE_NAME_MAX_CHARS) { streamIn.Close(); streamIn = null; throw new ApplicationException(String.Format("VolumeName too long {0}", volName)); } string storageServer = GetHostNameFromShareName(shareName); const int LastHeaderIndex = 4; // Index of last std header field in "ch-vm-share-vol" record. if (isOktofsC || isIoFlowD) { endpointC = new Endpoint(tag, hypervName, hypervName, vmName, shareName, NextEndpointKey++); listEndpoints.Add(endpointC); } if (isOktofsH) { endpointH = new Endpoint(tag, storageServer, hypervName, vmName, volName, NextEndpointKey++); listEndpoints.Add(endpointH); } Flow flow = new Flow(endpointC, endpointH, inputRecord, inputTokens, LastHeaderIndex, this); if (ListFlows == null) ListFlows = new List<Flow>(); ListFlows.Add(flow); } else if (IsValidTag(inputTokens[0], validTags)) { // Silently ignore tags that we don't understand if caller said they are valid. } else { string msg = String.Format("Illegal config file line: {0}", inputRecord); streamIn.Close(); streamIn = null; throw new ApplicationException(msg); } } streamIn.Close(); // // Plumb index values into ordered list of Endpoints. // if (listEndpoints != null) for (int i = 0; i < listEndpoints.Count; i++) listEndpoints[i].SetIndex(i); return listEndpoints; }
/// <summary> /// Init TMEs and Queues within a Flow: 1 or 2 of each depends on whether Flow defines C and/or H. /// </summary> /// <param name="flow"></param> private void AddTmesAndQueuesToFlow(Flow flow) { ValidateState(RateControllerState.Init, "AddTmesAndQueuesToEdge"); if (flow.EndpointC == null && flow.EndpointH == null) throw new ApplicationException("Flow cannot have both C and H are null."); // Find the RAP for each valid (non-null) Endpoint in the flow. // Wire the two RAP into the flow. if (flow.EndpointC != null && flow.EndpointC.IsOktofsC) flow.RapC = DictEpKeyToRap[flow.EndpointC.Key]; if (flow.EndpointC != null && flow.EndpointC.IsIoFlowD) flow.RapD = DictEpKeyToRap[flow.EndpointC.Key]; if (flow.EndpointH != null) flow.RapH = DictEpKeyToRap[flow.EndpointH.Key]; Connection conn; uint flowId = NextFreeFlowId++; if (flow.EndpointC != null && flow.EndpointC.IsOktofsC && AgentNameToConn.TryGetValue(flow.EndpointC.FilterServer, out conn)) { flow.RapC.AssignQueue(new OktoQueue(flowId, conn)); // so msg rx path can find dest flows. flow.RapC.Queue.SetFlagOn(OktoFlowFlags.FlowFlagIsAtC); } else if (flow.EndpointC != null && flow.EndpointC.IsOktofsC) { string msg = String.Format("AddQueuesToFlow({0}) unregistered agent C", flow.EndpointC.FilterServer); throw new ArgumentException(msg); } if (flow.EndpointH != null && flow.EndpointH.IsOktofsH && AgentNameToConn.TryGetValue(flow.EndpointH.FilterServer, out conn)) { flow.RapH.AssignQueue(new OktoQueue(flowId, conn)); // so msg rx path can find dest flows. flow.RapH.Queue.SetFlagOn(OktoFlowFlags.FlowFlagIsAtH); } else if (flow.EndpointH != null && flow.EndpointH.IsOktofsH) { string msg = String.Format("AddQueuesToFlow({0}) unregistered agent H", flow.EndpointH.FilterServer); throw new ArgumentException(msg); } if (flow.EndpointC != null && flow.EndpointC.IsIoFlowD && IoFlowNameToConn.TryGetValue(flow.EndpointC.FilterServer, out conn)) { flow.RapD.AssignQueue(new OktoQueue(flowId, conn)); // unused except by msg serializer. conn.DictIoFlows.Add(flow.FlowId, flow); // so msg rx path can find dest flows. } else if (flow.EndpointC != null && flow.EndpointC.IsIoFlowD) { string msg = String.Format("AddQueuesToFlow({0}) unregistered IoFlow agent at HyperV ", flow.EndpointC.FilterServer); throw new ArgumentException(msg); } flow.SetDefaultFlags(); }