コード例 #1
0
        private AccessDataDetail DeserializeJsonDataInLocal(string deviceId, string accessDate)
        {
            string jsonName = ComputeJsonName(deviceId, accessDate);

            if (!Directory.Exists(localPath))
            {
                return(null);
            }

            string fullPath = Path.Combine(localPath, jsonName);

            if (File.Exists(fullPath))
            {
                var content = File.ReadAllText(fullPath);
                AccessDataDetail deserializedObj = JsonConvert.DeserializeObject <AccessDataDetail>(content);
                return(deserializedObj);
            }

            return(null);
        }
コード例 #2
0
        public AccessDataDetail DeserializeJsonDataInCloud(string deviceId, string accessDate)
        {
            string jsonName = ComputeJsonName(deviceId, accessDate);

            // connect to our storage account and create a blob client
            var storageAccount = CloudStorageAccount.Parse(connectionString);
            var blobClient     = storageAccount.CreateCloudBlobClient();

            // get a reference to the container
            var blobcontainer = blobClient.GetContainerReference(containerName);

            blobcontainer.CreateIfNotExists();

            foreach (var blob in blobcontainer.ListBlobs())
            {
                string flName = blob.Uri.Segments.Last().ToString();
                Console.WriteLine(flName);

                if (jsonName == flName)
                {
                    CloudBlob cblob = blobcontainer.GetBlobReference(flName);
                    using (StreamReader reader = new StreamReader(cblob.OpenRead()))
                    {
                        string        content = "";
                        StringBuilder strjson = new StringBuilder();
                        while ((content = reader.ReadLine()) != null)
                        {
                            strjson.Append(content);
                        }
                        AccessDataDetail overlapAddObj = JsonConvert.DeserializeObject <AccessDataDetail>(strjson.ToString());
                        return(overlapAddObj);
                    }
                }
            }

            return(null);
        }
コード例 #3
0
        public IList <string> Parse(string blobName, Stream accessLogBlob)
        {
            string deviceId = GetDeviceId(blobName);

            IList <string> result    = new List <string>();
            string         lineEntry = string.Empty;

            //Initialize the prev record to some random date
            DateTime prevRecord = DateTime.Parse("01/01/1900");

            var file = new StreamReader(accessLogBlob);

            while ((lineEntry = file.ReadLine()) != null)
            {
                string[] tokens       = lineEntry.Split(' ');
                bool     isMainModule = false;

                //TODO: We skip this for now - need to check if this is an error and log appropriately
                if (tokens.Length <= 6)
                {
                    continue;
                }

                //If the first token (token[0]) is not of type IP address - skip processing - evaluate via Regex
                string          clientIP    = tokens[0];
                Regex           ipRegex     = new Regex(@"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b");
                MatchCollection regexResult = ipRegex.Matches(clientIP);
                if (regexResult.Count == 0)
                {
                    continue;
                }

                //Ignore Token 2(token[1]) & Token 3(token[2]) - Irrelevant

                //Token 4(token[3]) - Date information
                string      dateValue = tokens[3].Substring(1);
                CultureInfo provider  = CultureInfo.InvariantCulture;
                DateTime    dtObj     = DateTime.ParseExact(dateValue, "dd/MMM/yyyy:HH:mm:ss", provider);

                //TODO return if date is not valid
                string formattedDateValue = dtObj.ToString("yyyyMMdd");

                //Ignore Token 5(token[4]) & Token 6(token[5]) - Irrelevant

                //Token 7(token[6]) Contains Module Information.
                string module = tokens[6];

                string[] moduleTokens = module.Split('/', '?');
                //TODO: We skip this for now - need to check if this is an error and log appropriately
                if (moduleTokens.Length <= 3)
                {
                    continue;
                }

                //Specification says, the module url should start with "modules" - if that's not the case skip
                if (!moduleTokens.Contains("modules"))
                {
                    continue;
                }

                //Ignore all the Image based URLs
                if (moduleTokens[moduleTokens.Length - 1].Contains(".png") || moduleTokens[moduleTokens.Length - 1].Contains(".jpg") ||
                    moduleTokens[moduleTokens.Length - 1].Contains(".bmp") || moduleTokens[moduleTokens.Length - 1].Contains(".gif") ||
                    moduleTokens[moduleTokens.Length - 1].Contains(".js") || moduleTokens[moduleTokens.Length - 1].Contains(".css"))
                {
                    continue;
                }

                //Ignore all the request that came for the same time - An html page has css, js, and lot of references - they dont count towards an article that was read - round off to 1 event for 1 second.
                if (prevRecord == dtObj)
                {
                    continue;
                }
                else
                {
                    prevRecord = dtObj;
                }

                //If we reached here - we are certainly processing the record - Save any metadata information about the record.
                if (!dateRangeList.ContainsKey(formattedDateValue))
                {
                    //If there is no record for a date - create a record and register start and end time as that of current log entry
                    dateRangeList.Add(formattedDateValue, new DateRange()
                    {
                        StartTime = DateTime.Parse(dtObj.ToString("HH:mm:ss")), EndTime = DateTime.Parse(dtObj.ToString("HH:mm:ss"))
                    });
                }
                else
                {
                    //If there exist a record, update only the end time as we process each record
                    DateRange rangeObj = dateRangeList[formattedDateValue];
                    rangeObj.EndTime = DateTime.Parse(dtObj.ToString("HH:mm:ss"));
                }

                //If there are no records for the current date - check from cloud or local and load it
                if (!existingRecords.ContainsKey(formattedDateValue))
                {
                    //Check if there is a JSON for this date - if exist load the JSON in memory
                    AccessDataDetail existingObj = DeserializeJsonData(deviceId, formattedDateValue);
                    if (existingObj != null)
                    {
                        existingRecords.Add(formattedDateValue, existingObj);
                    }
                    else
                    {
                        existingRecords.Add(formattedDateValue, null);
                    }
                }

                //If there are records for current date - check if there is an overlap in time and ignore
                if (existingRecords[formattedDateValue] != null)
                {
                    DateTime startTime = existingRecords[formattedDateValue].StartTime;
                    DateTime endTime   = existingRecords[formattedDateValue].EndTime;

                    DateTime currrentTime = DateTime.Parse(dtObj.ToString("HH:mm:ss"));

                    //If the current log entry is a overlapping with existing record - ignore
                    if (currrentTime >= startTime && currrentTime <= endTime)
                    {
                        continue;
                    }
                }

                //Check if the request is for Main Module
                if (moduleTokens[3].Contains(".htm") || moduleTokens[3].Contains("html"))
                {
                    isMainModule = true;
                }

                if (!accessDataList.ContainsKey(formattedDateValue))
                {
                    //Add new value object (Dictionary) for each date
                    IDictionary <string, AccessData> accessDataKVPair = new Dictionary <string, AccessData>();
                    accessDataList.Add(formattedDateValue, accessDataKVPair);
                }

                IDictionary <string, AccessData> dictionaryObj = accessDataList[formattedDateValue];

                string moduleName = moduleTokens[2];
                if (!dictionaryObj.ContainsKey(moduleName))
                {
                    //Add new value object (AccessData) for each record in access log
                    AccessData obj = new AccessData()
                    {
                        ModuleName = moduleName, MainModuleCount = 0, SubModuleCount = 0, UpLoadTime = dtObj
                    };
                    dictionaryObj.Add(moduleName, obj);
                }

                AccessData accessDataObj = dictionaryObj[moduleName];
                if (isMainModule)
                {
                    accessDataObj.MainModuleCount = accessDataObj.MainModuleCount + 1;
                }
                else
                {
                    accessDataObj.SubModuleCount = accessDataObj.SubModuleCount + 1;
                }
            }

            //In the loop above - we just skip the overlapping record. Here is where the merge happens
            //Iterate AccessDataDetails for each date
            foreach (string dateValue in accessDataList.Keys)
            {
                AccessDataDetail addObj = new AccessDataDetail();
                addObj.AccessDate = dateValue;
                addObj.DeviceId   = deviceId;

                //Add the Start and End Date we derived from processing the current log
                if (dateRangeList.ContainsKey(dateValue))
                {
                    addObj.StartTime = dateRangeList[dateValue].StartTime;
                    addObj.EndTime   = dateRangeList[dateValue].EndTime;
                }

                //Add the Unmerged record from processing the current log
                IDictionary <string, AccessData> dictionaryObj = accessDataList[dateValue];
                foreach (string moduleName in dictionaryObj.Keys)
                {
                    addObj.AccessDetails.Add(dictionaryObj[moduleName]);
                }

                //Merge existing record with current record
                if (existingRecords.ContainsKey(dateValue))
                {
                    if (existingRecords[dateValue] != null)
                    {
                        if (existingRecords[dateValue].StartTime != null)
                        {
                            //Update Start Time with minimum of both
                            if (existingRecords[dateValue].StartTime <= addObj.StartTime)
                            {
                                addObj.StartTime = existingRecords[dateValue].StartTime;
                            }
                        }

                        if (existingRecords[dateValue].EndTime != null)
                        {
                            //Update End Time with maximum of both
                            if (existingRecords[dateValue].EndTime >= addObj.EndTime)
                            {
                                addObj.EndTime = existingRecords[dateValue].EndTime;
                            }
                        }

                        foreach (AccessData existingaddObj in existingRecords[dateValue].AccessDetails)
                        {
                            string moduleName = existingaddObj.ModuleName;
                            //Check if record exist for this module
                            bool matchFound = false;
                            foreach (AccessData newaddObj in addObj.AccessDetails)
                            {
                                if (moduleName == newaddObj.ModuleName)
                                {
                                    matchFound = true;
                                    newaddObj.MainModuleCount += existingaddObj.MainModuleCount;
                                    newaddObj.SubModuleCount  += existingaddObj.SubModuleCount;
                                    break;
                                }
                            }

                            if (matchFound == false)
                            {
                                addObj.AccessDetails.Add(existingaddObj);
                            }
                        }
                    }
                }

                //Serialize AccessDataDetail into JSON File
                string jsonValue = JsonConvert.SerializeObject(addObj, Formatting.Indented);
                Console.WriteLine(jsonValue);
                Console.WriteLine();

                try
                {
                    string       tempPath = Path.GetTempPath();
                    string       fileName = ComputeJsonName(addObj.DeviceId, addObj.AccessDate);
                    string       filePath = Path.Combine(tempPath, fileName);
                    StreamWriter sw       = new StreamWriter(filePath);
                    sw.WriteLine(jsonValue);
                    sw.Close();
                    result.Add(filePath);
                }
                catch (Exception)
                {
                    return(null);
                }
            }

            return(result);
        }