private static RDD <SerializaType> getVisitsForUsersWithTypeOfUser(RDD <SerializaType> VisitsForUsers) { var Visits = VisitsForUsers.Map <SerializaType>(line => { VisitizationSchema data = VisitizationSchema.Deserialize(line); VisitsForUser_WithTypeOfUser Vfu = new VisitsForUser_WithTypeOfUser(); Vfu.UAIPId = data.UAIPId; Vfu.TagId = data.TagId; Vfu.AnalyticsGuid = data.AnalyticsGuid; Vfu.SAEventConversionFactsRow = data.SAEventConversionFactsRow; if (Vfu.SAEventConversionFactsRow.ANID != null) { Vfu.ANID = Vfu.SAEventConversionFactsRow.ANID.ToSystemGuid(); } if (Vfu.SAEventConversionFactsRow.MUID != null) { Vfu.MUID = Vfu.SAEventConversionFactsRow.MUID.ToSystemGuid(); } if (Vfu.SAEventConversionFactsRow.MUID != null && Vfu.SAEventConversionFactsRow.IsNewMUID != true) { Vfu.TypeOfUser = 2; } else { if (Vfu.SAEventConversionFactsRow.ANID != null) { Vfu.TypeOfUser = 1; } else { if (Vfu.AnalyticsGuid != null) { Vfu.TypeOfUser = 3; } } } return(VisitsForUser_WithTypeOfUser.Serialize(Vfu)); }); return(Visits); }
public IEnumerable <string> GetData(KeyValuePair <string, string> line) { var uetLogsWithSameUAIPTag = line.Value.Split(new char[] { delimeter }, StringSplitOptions.RemoveEmptyEntries); //List<string> output = new List<string>(); var eventGroups = new Dictionary <string, List <UETEvent> >(); var firstRow = true; ClientIPData clientIP = null; var visitization = new VisitizationSchema(); foreach (var uetLogStr in uetLogsWithSameUAIPTag) { var uetLog = UETLogView.Deserialize(uetLogStr); if (uetLog == null) { continue; } if (firstRow) { visitization.UAIPId = uetLog.UAIPId; visitization.TagId = uetLog.TagId; visitization.TagName = uetLog.TagName; //visitization.UserAgent = uetLog.UserAgent; //why UserAgent is needed in the Original VisitizationReducer clientIP = uetLog.ClientIP; firstRow = false; } var anid = uetLog.ANID; var muid = uetLog.MUID; var isNewMuid = uetLog.IsNewMUID; var analyticsGuid = uetLog.AnalyticsGuid; var uetMatchingGuid = uetLog.UETMatchingGuid; var currentUetEvent = new UETEvent(); currentUetEvent.EventDateTime = uetLog.EventDateTime; currentUetEvent.ANID = (!anid.HasValue) ? null : new GUID(anid.Value); currentUetEvent.MUID = (!muid.HasValue) ? null : new GUID(muid.Value); currentUetEvent.IsNewMUID = isNewMuid; currentUetEvent.ReferrerURL = uetLog.ReferrerURL; currentUetEvent.PageTitle = uetLog.PageTitle; currentUetEvent.customEvent = uetLog.customEvent; currentUetEvent.NavigatedFromURL = uetLog.NavigatedFromURL; currentUetEvent.GoalValue = uetLog.GoalValue; currentUetEvent.PageLoad = uetLog.PageLoad; currentUetEvent.Version = uetLog.Version; currentUetEvent.LogServerName = uetLog.LogServerName; currentUetEvent.EventType = uetLog.EventType; currentUetEvent.AnalyticsGuid = (!analyticsGuid.HasValue) ? null : new GUID(analyticsGuid.Value); currentUetEvent.UETMatchingGuid = (!uetMatchingGuid.HasValue) ? null : new GUID(uetMatchingGuid.Value); AddToEventGroups(eventGroups, anid, muid, isNewMuid, currentUetEvent); } List <UETEvent> newMuidsList; if (eventGroups.TryGetValue(GroupKeyNewMuids, out newMuidsList)) { var eventsToBeRemoved = new List <UETEvent>(); foreach (var uetEvent in newMuidsList) { List <UETEvent> events; if (eventGroups.TryGetValue(uetEvent.MUID.ToSystemGuid().ToString(), out events)) { eventsToBeRemoved.Add(uetEvent); events.Insert(0, uetEvent); } } foreach (var ev in eventsToBeRemoved) { newMuidsList.Remove(ev); } } foreach (var eventGroup in eventGroups) { // Build the Visit List from the event groups var events = eventGroup.Value; if (events.Count > 0) { var factObject = BuildVisitEventGroupFromEventsList(sessionTimeoutThreshold, endOfDelta, events, maxEventsPerVisit, maxVisitsPerIPTag); if (factObject.Visits.Count > 0) { factObject.ClientIP = clientIP; var firstEvent = events[0]; factObject.ANID = firstEvent.ANID; factObject.MUID = firstEvent.MUID; factObject.IsNewMUID = firstEvent.IsNewMUID; visitization.AnalyticsGuid = firstEvent.AnalyticsGuid == null ? (Guid?)null : firstEvent.AnalyticsGuid.ToSystemGuid(); FixCustomGoalValues(factObject); visitization.SAEventConversionFactsRow = factObject; yield return(VisitizationSchema.Serialize(visitization)); //output.Add(VisitizationSchema.Serialize(visitization)); } } } //return output; }