Exemplo n.º 1
0
            //XXXIS add more things if we need them?

            public EssesFlowCacheContext(Flow flow, UInt64 cacheSizeAllocated, CacheWritePolicy writePolicy, CacheWriteBuffer cacheWrites, string cacheCurvePointsFile, string tenantID)
            {
                Flow = flow;
                this.cacheSizeAllocated = cacheSizeAllocated;
                this.writePolicy = writePolicy;
                this.cacheWrites = cacheWrites;
                this.hitRate = 0.0F;
                this.tenantID = tenantID;

                this.demandPointsSerializationFile = cacheCurvePointsFile;
                
                if (File.Exists(this.demandPointsSerializationFile))
                {
                    readInCacheCurve();
                    this.useSerializedCurve = true;
                }
                else
                {
                    this.cacheDemandCurvePoints = null;
                    this.cacheDemandFunc = null;
                    this.useSerializedCurve = false;
                }

#if COLLECT_CACHE_CURVE
                Console.CancelKeyPress += new ConsoleCancelEventHandler(Controller_CancelKeyPress);
#endif
            }
Exemplo n.º 2
0
        private void processFlowCacheStats(Flow flow, string statsString)
        {
            //Stats strings look like: "cacheSizeAllocated={0} cacheSizeUsed={1} cacheAccessesTotal={2} flowBytesAccessed={3} cacheAccessesHits={4} cacheEvictions={5} {6}"

            EssesFlowCacheContext fCC = (EssesFlowCacheContext)flow.Context;
            string[] statsStringSeparators = new string[] { " ", "\t" };
            char[] tokenSeparator = new char[] { '=' };
            string[] statsTokens = statsString.Split(statsStringSeparators, StringSplitOptions.RemoveEmptyEntries);
            Debug.Assert(statsTokens.Length > 0);

            //Extract the stats from the string sent by the client slave
            fCC.cacheSizeAllocated = Convert.ToUInt64(statsTokens[0].Split(tokenSeparator)[1]);
            fCC.cacheSizeUsed = Convert.ToUInt64(statsTokens[1].Split(tokenSeparator)[1]);

            UInt64 cacheAccessesNow, flowBytesAccessedNow, cacheAccessesHitsNow, cacheEvictionsNow;

            cacheAccessesNow = Convert.ToUInt64(statsTokens[2].Split(tokenSeparator)[1]);
            flowBytesAccessedNow = Convert.ToUInt64(statsTokens[3].Split(tokenSeparator)[1]);
            cacheAccessesHitsNow = Convert.ToUInt64(statsTokens[4].Split(tokenSeparator)[1]);
            cacheEvictionsNow = Convert.ToUInt64(statsTokens[5].Split(tokenSeparator)[1]);

            Debug.Assert(cacheAccessesNow >= fCC.cacheAccessesTotal);
            Debug.Assert(flowBytesAccessedNow >= fCC.flowBytesAccessedTotal);
            Debug.Assert(cacheAccessesHitsNow >= fCC.cacheAccessesHitsTotal);

            if (cacheAllocControlCounter == 0)
            {
                fCC.cacheAccessesLastAllocInterval = 0;
                fCC.flowBytesAccessedLasAllocInterval = 0;
                fCC.cacheAccessesHitsLastAllocInterval = 0;
                fCC.cacheEvictionsLastAllocInterval = 0;
            } 
            else if (cacheAllocControlCounter < cacheAllocControlFreq)
            {
                fCC.cacheAccessesLastAllocInterval += (cacheAccessesNow - fCC.cacheAccessesTotal);
                fCC.flowBytesAccessedLasAllocInterval += (flowBytesAccessedNow - fCC.flowBytesAccessedTotal);
                fCC.cacheAccessesHitsLastAllocInterval += (cacheAccessesHitsNow - fCC.cacheAccessesHitsTotal);
                fCC.cacheEvictionsLastAllocInterval += (cacheEvictionsNow - fCC.cacheEvictionsTotal);
            }

            fCC.flowBytesAccessedLastSampleInterval = flowBytesAccessedNow - fCC.flowBytesAccessedTotal;
            
            fCC.cacheAccessesTotal = cacheAccessesNow;
            fCC.flowBytesAccessedTotal = flowBytesAccessedNow;
            fCC.cacheAccessesHitsTotal = cacheAccessesHitsNow;
            fCC.cacheEvictionsTotal = cacheEvictionsNow;

            fCC.hitRate = (fCC.cacheAccessesTotal > 0 ? (float)fCC.cacheAccessesHitsTotal / (float)fCC.cacheAccessesTotal : 0.0F);

            if (statsTokens.Length > 6) //cache demand string is included
            {
                string[] cacheDemandPairs = statsTokens[6].Split(';');
                Double[] xVals = new Double[cacheDemandPairs.Length];
                Double[] yVals = new Double[cacheDemandPairs.Length];
                for (int i = 0; i < cacheDemandPairs.Length; i++)
                {
                    xVals[i] = Convert.ToDouble(cacheDemandPairs[i].Split(',')[0]);
                    yVals[i] = Convert.ToDouble(cacheDemandPairs[i].Split(',')[1]);
                }

                if (!fCC.useSerializedCurve) //only save curve if we don't have a curve at the moment
                {
                    fCC.cacheDemandCurvePoints = new CacheCurvePoints(xVals, yVals);
                    //fCC.cacheDemandFunc = Interpolate.LinearBetweenPoints(xVals, yVals);
                    //fCC.cacheDemandFunc = Interpolate.Linear(xVals, yVals);
                    fCC.cacheDemandFunc = Interpolate.PolynomialEquidistant(xVals, yVals);

                }

#if COLLECT_CACHE_CURVE
                //////////XXXIS: quick test to see if we got the function fitted properly (since graphing's a pain...)

                //XXXIS change fileCacheSize accordingly to print the curve on the screen, IN ADDITION to it being collected when you Ctrl+C the
                //controller at the end of the run

                UInt64 fileCacheSize = 10737418240;
                int numCurvePoints = 500;
                float stepSize = 1.0F / (float)numCurvePoints;
                Console.WriteLine("Flow ID {0}", flow.FlowId);

                for (float frac = 0.0F; frac <= (1.0F + stepSize); frac += stepSize)
                {
                    UInt32 ghostCacheBlockSize = 4096;
                    //Compute and block-align the cache sizes; XXXIS: THIS ASSUMES ghost cache block size is a power of two!!!
                    UInt64 specCacheSize = ((UInt64)(frac * (float)fileCacheSize)) & (~((UInt64)(ghostCacheBlockSize - 1)));
                    Console.WriteLine("{0}, {1}", specCacheSize, fCC.cacheDemandFunc.Interpolate((double)specCacheSize));
                    //rateController.Log(String.Format("{0}, {1}", specCacheSize, fCC.cacheDemandFunc.Interpolate((double)specCacheSize)));
                }
#endif
            }

            //Console.WriteLine("Flow {0}, SizeAllocated {1}, SizeUsed {2}, flowBytesAccessedTotal {3}, AccessesTotal {4}, AccessesHits {5}, Evictions {6}, HitRate {7}",
            //    flow.FlowId, fCC.cacheSizeAllocated, fCC.cacheSizeUsed, fCC.flowBytesAccessedTotal, fCC.cacheAccessesTotal,
            //    fCC.cacheAccessesHitsTotal, fCC.cacheEvictionsTotal, fCC.hitRate);
        }
        /// <summary>
        /// Parses config file and returns list of Enpoints.
        /// </summary>
        /// <param name="fileName">Name of input file.</param>
        /// <param name="validTags">Acceptable input record types.</param>
        /// <returns>List containing one Endpoint for each input record.</returns>
        public List<Endpoint> ParseConfigFile(string fileName, string[] validTags)
        {
            List<Endpoint> listEndpoints = null;
            StreamReader streamIn = new StreamReader(fileName);
            string inputRecord;
            string[] separators = new string[] { " ", "\t" };

            ValidateState(RateControllerState.Init, "ParseConfigFile");
            while ((inputRecord = streamIn.ReadLine()) != null)
            {
                string[] inputTokens = inputRecord.Split(separators, StringSplitOptions.RemoveEmptyEntries);
                if (inputTokens.Length == 0)
                    continue;
                else if (inputTokens[0].StartsWith(@"#") || inputTokens[0].StartsWith(@"//"))
                    continue;
                else if (inputTokens[0].ToLower().Equals(Parameters.VOLNAME_ALIAS_REC_HEADER))
                    continue;
                else if (inputTokens.Length > 4 &&
                    IsValidTag(inputTokens[0], validTags) &&
                    !inputTokens[0].StartsWith("-") &&
                    (inputTokens[0].ToLower().Contains("-vm-share-vol") ||
                     inputTokens[0].ToLower().Contains("-vm-file-vol")))
                {
                    //
                    // Parse records.
                    // Each such record generates a Flow containing one or more queue : e.g. one at C and one at H.
                    //
                    string tag = inputTokens[0].ToLower();
                    string StageIds = tag.Substring(0, tag.IndexOf("-"));
                    bool isOktofsC = StageIds.Contains("c");
                    bool isOktofsH = StageIds.Contains("h");
                    bool isIoFlowD = StageIds.Contains("d");

                    Endpoint endpointC = null;
                    Endpoint endpointH = null;
                    if (listEndpoints == null)
                        listEndpoints = new List<Endpoint>();
                    string vmName = inputTokens[1];
                    string hypervName = inputTokens[2];
                    string shareName = inputTokens[3];
                    string volName = inputTokens[4];
                    // Windows keeps breaking our conf files by changing the volume names at H.
                    // Provide an optional alias func based on share name to avoid admin chaos.
                    volName = GetVolumeNameFromShareName(shareName, volName);

                    if (!StageIds.Replace("c", "").Replace("h", "").Replace("d", "").Equals(""))
                    {
                        streamIn.Close();
                        streamIn = null;
                        throw new ApplicationException(String.Format("input rec illegal prefix {0}", tag));
                    }
                    if (vmName.Length > Parameters.OKTO_VM_NAME_MAX_CHARS)
                    {
                        streamIn.Close();
                        streamIn = null;
                        throw new ApplicationException(String.Format("VmName too long {0}", vmName));
                    }
                    if (shareName.Length > Parameters.OKTO_SHARE_NAME_MAX_CHARS)
                    {
                        streamIn.Close();
                        streamIn = null;
                        throw new ApplicationException(String.Format("ShareName too long {0}", shareName));
                    }
                    if (volName.Length > Parameters.OKTO_SHARE_NAME_MAX_CHARS)
                    {
                        streamIn.Close();
                        streamIn = null;
                        throw new ApplicationException(String.Format("VolumeName too long {0}", volName));
                    }

                    string storageServer = GetHostNameFromShareName(shareName);
                    const int LastHeaderIndex = 4; // Index of last std header field in "ch-vm-share-vol" record.

                    if (isOktofsC || isIoFlowD)
                    {
                        endpointC = new Endpoint(tag, hypervName, hypervName, vmName, shareName, NextEndpointKey++);
                        listEndpoints.Add(endpointC);
                    }
                    if (isOktofsH)
                    {
                        endpointH = new Endpoint(tag, storageServer, hypervName, vmName, volName, NextEndpointKey++);
                        listEndpoints.Add(endpointH);
                    }
                    Flow flow = new Flow(endpointC, endpointH, inputRecord, inputTokens, LastHeaderIndex, this);
                    if (ListFlows == null)
                        ListFlows = new List<Flow>();
                    ListFlows.Add(flow);

                }
                else if (IsValidTag(inputTokens[0], validTags))
                {
                    // Silently ignore tags that we don't understand if caller said they are valid.
                }
                else
                {
                    string msg = String.Format("Illegal config file line: {0}", inputRecord);
                    streamIn.Close();
                    streamIn = null;
                    throw new ApplicationException(msg);
                }
            }
            streamIn.Close();
            //
            // Plumb index values into ordered list of Endpoints.
            //
            if (listEndpoints != null)
                for (int i = 0; i < listEndpoints.Count; i++)
                    listEndpoints[i].SetIndex(i);

            return listEndpoints;
        }
        /// <summary>
        /// Init TMEs and Queues within a Flow: 1 or 2 of each depends on whether Flow defines C and/or H.
        /// </summary>
        /// <param name="flow"></param>
        private void AddTmesAndQueuesToFlow(Flow flow)
        {
            ValidateState(RateControllerState.Init, "AddTmesAndQueuesToEdge");

            if (flow.EndpointC == null && flow.EndpointH == null)
                throw new ApplicationException("Flow cannot have both C and H are null.");

            // Find the RAP for each valid (non-null) Endpoint in the flow.
            // Wire the two RAP into the flow.
            if (flow.EndpointC != null && flow.EndpointC.IsOktofsC)
                flow.RapC = DictEpKeyToRap[flow.EndpointC.Key];
            if (flow.EndpointC != null && flow.EndpointC.IsIoFlowD)
                flow.RapD = DictEpKeyToRap[flow.EndpointC.Key];
            if (flow.EndpointH != null)
                flow.RapH = DictEpKeyToRap[flow.EndpointH.Key];

            Connection conn;
            uint flowId = NextFreeFlowId++;
            if (flow.EndpointC != null &&
                flow.EndpointC.IsOktofsC &&
                AgentNameToConn.TryGetValue(flow.EndpointC.FilterServer, out conn))
            {
                flow.RapC.AssignQueue(new OktoQueue(flowId, conn));     // so msg rx path can find dest flows.
                flow.RapC.Queue.SetFlagOn(OktoFlowFlags.FlowFlagIsAtC);
            }
            else if (flow.EndpointC != null && flow.EndpointC.IsOktofsC)
            {
                string msg = String.Format("AddQueuesToFlow({0}) unregistered agent C", flow.EndpointC.FilterServer);
                throw new ArgumentException(msg);
            }
            if (flow.EndpointH != null &&
                flow.EndpointH.IsOktofsH &&
                AgentNameToConn.TryGetValue(flow.EndpointH.FilterServer, out conn))
            {
                flow.RapH.AssignQueue(new OktoQueue(flowId, conn));     // so msg rx path can find dest flows.
                flow.RapH.Queue.SetFlagOn(OktoFlowFlags.FlowFlagIsAtH);
            }
            else if (flow.EndpointH != null && flow.EndpointH.IsOktofsH)
            {
                string msg = String.Format("AddQueuesToFlow({0}) unregistered agent H", flow.EndpointH.FilterServer);
                throw new ArgumentException(msg);
            }
            if (flow.EndpointC != null &&
                flow.EndpointC.IsIoFlowD &&
                IoFlowNameToConn.TryGetValue(flow.EndpointC.FilterServer, out conn))
            {
                flow.RapD.AssignQueue(new OktoQueue(flowId, conn)); // unused except by msg serializer.
                conn.DictIoFlows.Add(flow.FlowId, flow);            // so msg rx path can find dest flows.
            }
            else if (flow.EndpointC != null && flow.EndpointC.IsIoFlowD)
            {
                string msg = String.Format("AddQueuesToFlow({0}) unregistered IoFlow agent at HyperV ", flow.EndpointC.FilterServer);
                throw new ArgumentException(msg);
            }
            flow.SetDefaultFlags();
        }