Пример #1
0
        public void Start(string[] args)
        {
            m_Parameters = ParameterParse.ParseArguments(args, false /* firstOpFlag */, true /* multipleFiles */);
            foreach (KeyValuePair <string, string> kvp in m_Parameters)
            {
                switch (kvp.Key)
                {
                case "-p":
                case "--period":
                    m_bucketSeconds = int.Parse(kvp.Value);
                    break;

                case "-o":
                case "--output":
                    m_outFile = kvp.Value;
                    break;

                case "-h":
                case "--cmHosts":
                    m_cmHosts = kvp.Value;
                    break;

                case "--verbose":
                    m_ParamVerbose++;
                    break;

                case ParameterParse.LAST_PARAM:
                    m_inFiles = kvp.Value;
                    break;

                case ParameterParse.ERROR_PARAM:
                    // if we get here, the parser found an error
                    Logger.Log("Parameter error: " + kvp.Value);
                    Logger.Log(Invocation());
                    return;

                default:
                    Logger.Log("ERROR: UNKNOWN PARAMETER: " + kvp.Key);
                    Logger.Log(Invocation());
                    return;
                }
            }

            List <HTTPRecord> records = new List <HTTPRecord>();

            // Read in the records
            if (!String.IsNullOrEmpty(m_inFiles))
            {
                string[] files = m_inFiles.Split(',');
                foreach (string fileName in files)
                {
                    records.AddRange(HTTPRecord.Read(fileName));
                }
            }

            // Find high and low dates and compute the number of buckets
            double minDate = double.MaxValue;
            double maxDate = double.MinValue;

            foreach (HTTPRecord rec in records)
            {
                minDate = Math.Min(minDate, rec.time);
                maxDate = Math.Max(maxDate, rec.time);
            }

            double bucketBaseTime = minDate;
            int    numBuckets     = ((int)((maxDate - minDate) * LongDate.secondsPerDay)) / m_bucketSeconds;

            numBuckets += 1; // add a last bucket for rounding error at the end.
            Logger.Log("Number of buckets = {0}", numBuckets);

            // Loop through all the records and assign each to a bucket
            foreach (HTTPRecord rec in records)
            {
                rec.bucket = ((int)((rec.time - bucketBaseTime) * LongDate.secondsPerDay)) / m_bucketSeconds;
            }

            // Specify individual hosts to count accesses with CSV list "host,host,host"
            List <string> cmHosts  = m_cmHosts.Split(',').ToList <string>();
            int           numHosts = cmHosts.Count;

            if (numHosts == 0)
            {
                Logger.Log("NUMBER OF Client Manager HOSTS MUST NOT BE ZERO!!");
                return;
            }

            // Initialize each bucket line with the variable sized structures
            BucketLine[] bucketLines = new BucketLine[numBuckets];
            for (int ii = 0; ii < numBuckets; ii++)
            {
                bucketLines[ii].bucket    = ii;
                bucketLines[ii].time      = bucketBaseTime + ((double)(ii * m_bucketSeconds) / LongDate.secondsPerDay);
                bucketLines[ii].hostCount = new int[numHosts];
                bucketLines[ii].logins    = 0;
            }

            // Loop through all the records and fill the bucket info
            foreach (HTTPRecord rec in records)
            {
                int nHost = cmHosts.IndexOf(rec.source);
                if (nHost >= 0)
                {
                    bucketLines[rec.bucket].hostCount[nHost]++;
                }
                if (rec.remain.Contains("/Grid/login/ "))
                {
                    bucketLines[rec.bucket].logins++;
                }
            }

            // Print out all the buckets
            bool       firstLine = true;
            TextWriter outWriter = new StreamWriter(File.Open(m_outFile, FileMode.Create));

            if (outWriter != null)
            {
                using (outWriter)
                {
                    if (firstLine)
                    {
                        StringBuilder buff = new StringBuilder();
                        buff.Append("bucket");
                        buff.Append(",");
                        buff.Append("time");
                        buff.Append(",");
                        buff.Append("logins");
                        buff.Append(",");
                        for (int ii = 0; ii < cmHosts.Count; ii++)
                        {
                            buff.Append(cmHosts[ii]);
                            buff.Append(",");
                        }
                        outWriter.WriteLine(buff.ToString());
                        firstLine = false;
                    }

                    foreach (BucketLine buck in bucketLines)
                    {
                        StringBuilder buff = new StringBuilder();
                        buff.Append(buck.bucket.ToString());
                        buff.Append(",");
                        buff.Append(buck.time.ToString());
                        buff.Append(",");
                        buff.Append(buck.logins.ToString());
                        buff.Append(",");
                        for (int ii = 0; ii < buck.hostCount.Length; ii++)
                        {
                            buff.Append(buck.hostCount[ii].ToString());
                            buff.Append(",");
                        }
                        outWriter.WriteLine(buff.ToString());
                    }
                }
            }
        }
Пример #2
0
        static public List <HTTPRecord> Read(string filename)
        {
            List <HTTPRecord> ret = new List <HTTPRecord>();

            TextReader inReader = new StreamReader(File.Open(filename, FileMode.Open));

            if (inReader == null)
            {
                Logger.Log("{0} Read: Failed opening stat file '{1}'", LogHeader, filename);
                return(null);
            }

            string inLine;

            Regex pattern = new Regex(@"^(\d+\.\d+\.\d+\.\d+) - - \[(\d+)/(...)/(\d+):(\d+):(\d+):(\d+) .*""(\w+) (.*)$");

            using (inReader)
            {
                while ((inLine = inReader.ReadLine()) != null)
                {
                    try
                    {
// SAMPLE
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 4703 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 213 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 211 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/ HTTP/1.1" 200 211 "-" "-"
// 98.229.237.99 - - [17/Mar/2013:15:30:43 -0400] "POST /Grid/login/ HTTP/1.1" 200 2586 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:47 -0400] "POST /Grid/ HTTP/1.1" 200 548 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:47 -0400] "POST /Grid/?id=9f54505f-b5aa-4e16-a1ef-d3152a252e08 HTTP/1.1" 200 293 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:47 -0400] "POST /Grid/ HTTP/1.1" 200 290 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:30:48 -0400] "GET /Grid/?id=cce0f112-878f-4586-a2e2-a8f104bba271 HTTP/1.1" 404 234 "-" "-"
// 10.10.10.1 - - [17/Mar/2013:15:31:10 -0400] "POST /Grid/ HTTP/1.1" 200 223 "-" "-"
// 98.229.237.99 - - [17/Mar/2013:15:31:10 -0400] "GET /GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429) HTTP/1.1" 200 1432 "-" "Mozilla/5.0 (Windows; U; Windows NT6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 10.10.10.1 - - [17/Mar/2013:15:31:10 -0400] "POST /Grid/ HTTP/1.1" 200 223 "-" "-"
// 98.229.237.99 - - [17/Mar/2013:15:31:10 -0400] "GET /GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429) HTTP/1.1" 200 1432 "-" "Mozilla/5.0 (Windows; U; Windows NT6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/styles/default/style.css HTTP/1.1" 200 897 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429)" "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/javascript/jquery.qtip.js HTTP/1.1" 200 10148 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429)" "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/styles/default/jquery-ui.css HTTP/1.1" 200 5985 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sourceid=&version=4.4.0%20(33429)" "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) SecondLife/4.4.0.33429 (Firestorm-Moses; firestorm skin) Safari/533.3"
// 98.229.237.99 - - [17/Mar/2013:15:31:11 -0400] "GET /GridFrontend//static/styles/default/jquery.qtip.css HTTP/1.1" 200 1695 "http://107.7.21.234/GridFrontend/index.php?channel=Firestorm-Moses&firstlogin=TRUE&grid=mosesdsg&lang=en&os=Microsoft%20Windows%207%2064-bit%20&sstring[] pieces = inLine.Split(',');

                        Match fields = pattern.Match(inLine);
                        if (!fields.Success)
                        {
                            Logger.Log("PARSING FAILURE: {0}", inLine);
                            continue;
                        }

                        HTTPRecord aRec = new HTTPRecord();
                        aRec.line   = inLine;
                        aRec.bucket = 0;

                        int nMonth = "xxJanFebMarAprMayJunJulAugSepOctNovDec".IndexOf(fields.Groups[3].Value) / 3;
                        aRec.timeDateTime = new DateTime(
                            int.Parse(fields.Groups[4].Value), // year
                            nMonth,                            // month
                            int.Parse(fields.Groups[2].Value), // month day
                            int.Parse(fields.Groups[5].Value), // hour
                            int.Parse(fields.Groups[6].Value), // minute
                            int.Parse(fields.Groups[7].Value)  // second
                            );

                        aRec.time   = LongDate.DateTimeToExcelDate(aRec.timeDateTime);
                        aRec.source = fields.Groups[1].Value;
                        aRec.op     = fields.Groups[8].Value;
                        aRec.remain = fields.Groups[9].Value;

                        ret.Add(aRec);
                        // Logger.Log(aRec.ToString());
                    }
                    catch (Exception e)
                    {
                        Logger.Log("{0} Exception parsing line: '{1}'", LogHeader, inLine);
                        Logger.Log("{0} Exception parsing line: e: {1} ", LogHeader, e);
                        continue;
                    }
                }
            }

            return(ret);
        }