public void Trigger(object stateInfo) { try { if (webTrackingInProgress) { return; } webTrackingInProgress = true; var accountsWithWebAnalytics = accountService.GetWebAnalyticsProviders(new GetWebAnalyticsProvidersRequest()).WebAnalyticsProviders; DateTime currentTime = DateTime.Now.ToUniversalTime(); string currentTimeString = string.Format("{0:u}", currentTime).Replace(":", "-"); dynamic UniqueVisitors; APIManager apiManager = new APIManager(); string apiLookupStringPath = ConfigurationManager.AppSettings["WEBANALYTICS_API_RESPONSE_PATH"] + "/APILookUp" + "/APILookUp_UTC_" + currentTimeString; string smartTouchPartnerKey = ConfigurationManager.AppSettings["SMARTTOUCH_PARTNER_KEY"]; JavaScriptSerializer jsonSerializer = new JavaScriptSerializer(); ValidatePhysicalPaths(); apiManager.APICall("http://api.visistat.com/api-lookup.php?hid=" + smartTouchPartnerKey + "&act=2", apiLookupStringPath); var initialSetupProcessingStartHour = ConfigurationManager.AppSettings["INITIAL_SETUP_PROCESSING_START_HOUR"]; var initialSetupProcessingStopHour = ConfigurationManager.AppSettings["INITIAL_SETUP_PROCESSING_STOP_HOUR"]; var splitVisitInterval = ConfigurationManager.AppSettings["SPLIT_VISIT_INTERVAL"]; IList <dynamic> accountTimeZones = new List <dynamic>(); using (StreamReader streamReader = new StreamReader(apiLookupStringPath + ".txt")) { while (streamReader.Peek() >= 0) { var readLine = streamReader.ReadLine(); if (string.IsNullOrEmpty(readLine)) { break; } var accountInfo = readLine.Split('|').ToArray(); accountTimeZones.Add(new { APIKey = accountInfo[11], TimeZoneID = accountInfo[12], TrackingDomain = accountInfo[2] }); } } #region -- Processing -- foreach (var webAnalyticsProvider in accountsWithWebAnalytics) { var accountCurrentTime = currentTime; Logger.Current.Verbose("accountCurrentTime : " + accountCurrentTime.ToString()); IList <WebVisit> WebVisits = new List <WebVisit>(); var isInitialSetup = webAnalyticsProvider.LastAPICallTimeStamp == null; if (!isInitialSetup || (isInitialSetup && accountCurrentTime.Hour > int.Parse(initialSetupProcessingStartHour) && accountCurrentTime.Hour < int.Parse(initialSetupProcessingStopHour))) { IList <string> identities = new List <string>(); Logger.Current.Informational("Analyzing web visits for web analytics provider id: " + webAnalyticsProvider.Id + ". AccountID: " + webAnalyticsProvider.AccountID); var content = ""; try { webAnalyticsProvider.LastAPICallTimeStamp = webAnalyticsProvider.LastAPICallTimeStamp != null ? webAnalyticsProvider.LastAPICallTimeStamp.Value : accountCurrentTime.AddDays(-180); var accountTimeZone = accountTimeZones.Where(tz => tz.APIKey == webAnalyticsProvider.APIKey).Select(tz => tz.TimeZoneID).FirstOrDefault(); var accountUTCTimeSpan = VisiStatTimeZones.TimeZones().Where(tz => tz.TimeZoneId == byte.Parse(accountTimeZone)).Select(tz => tz.TimeSpan).FirstOrDefault(); var accountTimeZoneId = VisiStatTimeZones.TimeZones().Where(tz => tz.TimeZoneId == byte.Parse(accountTimeZone)).Select(tz => tz.GlobalTimeZoneId).FirstOrDefault(); TimeZoneInfo tzInfo = TimeZoneInfo.FindSystemTimeZoneById(accountTimeZoneId); var isDaylightSaving = tzInfo.IsDaylightSavingTime(TimeZoneInfo.ConvertTime(webAnalyticsProvider.LastAPICallTimeStamp.Value, tzInfo)); // http://www.timeanddate.com/time/change/usa/new-york?year=2015 if (isDaylightSaving) { accountUTCTimeSpan = accountUTCTimeSpan.Add(new TimeSpan(1, 0, 0)); } var accountStartDate = webAnalyticsProvider.LastAPICallTimeStamp.Value.AddHours(accountUTCTimeSpan.Hours).AddMinutes(accountUTCTimeSpan.Minutes); var accountEndDate = accountCurrentTime.AddHours(accountUTCTimeSpan.Hours).AddMinutes(accountUTCTimeSpan.Minutes); TimeSpan startEndTimeSpan = (TimeSpan)(accountEndDate - accountStartDate); if (startEndTimeSpan.TotalHours > 3) { accountEndDate = accountStartDate.AddHours(3); accountCurrentTime = webAnalyticsProvider.LastAPICallTimeStamp.Value.AddHours(3); Logger.Current.Verbose("accountEndDate: " + accountEndDate.ToString()); } string accountStartDateString = String.Format("{0:u}", accountStartDate); string accountEndDateString = String.Format("{0:u}", accountEndDate); int year = accountStartDate.Year; int month = accountStartDate.Month; var identityLocation = Path.Combine(ConfigurationManager.AppSettings["WEBANALYTICS_API_RESPONSE_PATH"], "Received", webAnalyticsProvider.AccountID.ToString(), year.ToString(), month.ToString()); if (!Directory.Exists(identityLocation)) { Directory.CreateDirectory(identityLocation); } string uvIdentitiesLocation = Path.Combine(identityLocation, "UVByIdentity_" + accountEndDateString.Replace(":", "-") + "_AccID_" + webAnalyticsProvider.AccountID + "_WAID_" + webAnalyticsProvider.Id); string uvByIddUrl = "http://api.visistat.com/stats-api-v25.php?key=" + webAnalyticsProvider.APIKey + "&qt=idd&d=json&sdate=" + accountStartDateString + "&edate=" + accountEndDateString; Logger.Current.Informational("AccountId: " + webAnalyticsProvider.AccountID + ", uvByIddURL: " + uvByIddUrl); content = apiManager.APICall(uvByIddUrl, uvIdentitiesLocation); UniqueVisitors = (dynamic)jsonSerializer.DeserializeObject(content); foreach (dynamic item in UniqueVisitors) { string identity = item[1]; identities.Add(identity); } IList <string> knownIdentities = contactService .CompareKnownIdentities(new CompareKnownContactIdentitiesRequest() { ReceivedIdentities = identities, AccountId = webAnalyticsProvider.AccountID, } ).KnownIdentities.ToList(); IEnumerable <string> cpByIdentityLocation = new List <string>(); cpByIdentityLocation = apiManager.GetCPByIdentityFiles(webAnalyticsProvider, knownIdentities, accountStartDateString, accountEndDateString); int ipIterator = 0; foreach (string location in cpByIdentityLocation) { string[] readFile = File.ReadAllLines(location + ".txt"); var visits = (dynamic)jsonSerializer.DeserializeObject(readFile[0]); int visitIterator = 0; foreach (var visit in visits) { if (visit[2] != "city") { DateTime currentPageVisitedOn = Convert.ToDateTime(visit[0]); DateTime nextPageVisitedOn = visitIterator > 1 ? Convert.ToDateTime(visits[visitIterator - 1][0]) : currentPageVisitedOn; //if (isDaylightSaving) //{ // currentPageVisitedOn = currentPageVisitedOn.AddHours(-1); // nextPageVisitedOn = nextPageVisitedOn.AddHours(-1); //} WebVisit newWebVisit = new WebVisit() { VisitedOn = currentPageVisitedOn.AddHours(-accountUTCTimeSpan.Hours).AddMinutes(-accountUTCTimeSpan.Minutes), PageVisited = visit[1], City = visit[2], State = visit[3], Country = visit[4], ISPName = visit[7], ContactReference = knownIdentities[ipIterator], IPAddress = visit[6], }; if (visitIterator > 1) { newWebVisit.Duration = (short)nextPageVisitedOn.Subtract(currentPageVisitedOn).TotalSeconds; } WebVisits.Add(newWebVisit); } visitIterator = visitIterator + 1; } ipIterator = ipIterator + 1; } try { Logger.Current.Verbose("After collecting the info the accountCurrentTime : " + accountCurrentTime.ToString()); webAnalyticsService.AddContactWebVisits(new AddContactWebVisitRequest() { ContactWebVisits = WebVisits, LastAPICallTimeStamp = accountCurrentTime, WebAnalyticsProvider = webAnalyticsProvider, SplitVisitInterval = short.Parse(splitVisitInterval) }); } catch (Exception ex) { Logger.Current.Error("Exception occurred while saving the web visits to the database. AccountID: " + webAnalyticsProvider.AccountID, ex); } } catch (Exception ex) { Logger.Current.Error("Exception occurred while making an API call. AccountID: " + webAnalyticsProvider.AccountID, ex); } Logger.Current.Informational("Analyzing web visits for web analytics provider id: " + webAnalyticsProvider.Id + " is completed"); } } #endregion } catch (Exception ex) { Logger.Current.Error("An exception occured with getting web analytics.", ex); } webTrackingInProgress = false; }
protected override void ExecuteInternal(IJobExecutionContext context) { var webAnalyticsProviders = _accountService .GetWebAnalyticsProviders(new GetWebAnalyticsProvidersRequest()) .WebAnalyticsProviders; var utcNow = DateTime.UtcNow; var yesterday = utcNow.Yesterday(); Log.Informational("Query partners info"); var partnerInfos = _kickfireService.GetPartnerInfos().ToArray(); foreach (var webAnalyticsProvider in webAnalyticsProviders) { try { if (webAnalyticsProvider.AccountID == 340) { Debugger.Break(); } var accountInfo = _accountRepository.GetAccountBasicDetails(webAnalyticsProvider.AccountID); //Do not query suspended accounts if (accountInfo.Status == (byte)AccountStatus.Suspend) { continue; } Log.Informational("Analyzing web visits for web analytics provider id: " + webAnalyticsProvider.Id + ". AccountID: " + webAnalyticsProvider.AccountID); var lastUpdatedAt = webAnalyticsProvider.LastAPICallTimeStamp ?? yesterday; var accountTimeZone = int.Parse(partnerInfos .Where(x => x.ApiKey == webAnalyticsProvider.APIKey) .Select(x => x.TimeZoneId) .First()); var kickfireTimeZone = VisiStatTimeZones .TimeZones() .First(x => x.TimeZoneId == accountTimeZone) .GlobalTimeZoneId; Log.Informational($"Account {webAnalyticsProvider.AccountID} CurrentTime: {utcNow.ToTimezone(kickfireTimeZone)}"); var startDate = lastUpdatedAt .SetKind(DateTimeKind.Utc) .ToTimezone(kickfireTimeZone); var endDate = utcNow .ToTimezone(kickfireTimeZone); Log.Informational($"Query unique visitors for AccountID: {webAnalyticsProvider.AccountID}"); var identities = _kickfireService.GetUniqueVisitors(webAnalyticsProvider.APIKey, startDate, endDate) .Select(x => x.Identity) .Take(2000) //Max value that dapper can handle .ToArray(); var knownIdentities = _contactService .CompareKnownIdentities(new CompareKnownContactIdentitiesRequest { ReceivedIdentities = identities, AccountId = webAnalyticsProvider.AccountID, }).KnownIdentities .ToArray(); Log.Informational($"Query page visits for AccountID: {webAnalyticsProvider.AccountID}"); var webVisits = _kickfireService .GetPageVisitors(webAnalyticsProvider.APIKey, knownIdentities, startDate, endDate) .Select(visit => new WebVisit { VisitedOn = visit.VisitedOn .ConvertToTimeZone(kickfireTimeZone, accountInfo.TimeZone) .SetKind(DateTimeKind.Utc), PageVisited = visit.PageName, City = visit.City, State = visit.Region, Country = visit.Country, ISPName = visit.IspName, IPAddress = visit.IpAddress, ContactReference = visit.ContactReference, Duration = visit.DurationInSeconds }).ToList(); _webAnalyticsService.AddContactWebVisits(new AddContactWebVisitRequest { ContactWebVisits = webVisits, LastAPICallTimeStamp = utcNow, WebAnalyticsProvider = webAnalyticsProvider, SplitVisitInterval = _jobConfig.SplitVisitInterval }); } catch (Exception ex) { Log.Error("Exception occurred while making an API call. AccountID: " + webAnalyticsProvider.AccountID, ex); } Log.Informational("Analyzing web visits for web analytics provider id: " + webAnalyticsProvider.Id + " is completed"); } }