Exemplo n.º 1
0
 //====================================================================================================
 //
 public static void housekeepDaily(CoreController core, HouseKeepEnvironmentModel env)
 {
     try {
         //
         LogController.logInfo(core, "HousekeepDaily");
         //
         // -- Download Updates
         SoftwareUpdatesClass.downloadAndInstall(core);
         //
         // -- Addon folder
         AddonFolderClass.housekeep(core);
         //
         // -- metadata
         ContentFieldClass.housekeep(core, env);
         //
         // -- content
         PageContentClass.housekeep(core, env);
         AddonContentFieldTypeRuleClass.housekeep(core, env);
         AddonContentTriggerRuleClass.housekeep(core, env);
         ContentWatchClass.housekeep(core, env);
         EmailDropClass.housekeep(core, env);
         EmailLogClass.housekeep(core, env);
         FieldHelpClass.housekeep(core, env);
         GroupRulesClass.housekeep(core, env);
         MemberRuleClass.housekeep(core, env);
         MetadataClass.housekeep(core, env);
         LinkAliasClass.housekeep(core, env);
         //
         // -- Properties
         UserProperyClass.housekeep(core);
         VisitPropertyClass.housekeep(core);
         VisitorPropertyClass.housekeep(core);
         //
         // -- visits, visitors, viewings
         VisitClass.housekeep(core, env);
         VisitorClass.housekeep(core, env);
         ViewingsClass.housekeep(core, env);
         //
         // -- summary
         VisitSummaryClass.housekeep(core, env);
         ViewingSummaryClass.housekeep(core, env);
         //
         // -- logs
         ActivityLogClass.housekeep(core, env);
         //
         // -- people
         PersonClass.housekeep(core, env);
     } catch (Exception ex) {
         LogController.logError(core, ex);
         throw;
     }
 }
Exemplo n.º 2
0
 //
 //=========================================================================================
 /// <summary>
 /// summarized visits hourly
 /// </summary>
 /// <param name="core"></param>
 /// <param name="env"></param>
 public static void housekeep(CoreController core, HouseKeepEnvironmentModel env)
 {
     try {
         //
         LogController.logInfo(core, "Housekeep, visitsummary");
         //
         bool newHour = (core.dateTimeNowMockable.Hour != env.lastCheckDateTime.Hour);
         if (env.forceHousekeep || newHour)
         {
             //
             // Set NextSummaryStartDate based on the last time we ran hourly summarization
             //
             DateTime LastTimeSummaryWasRun = env.visitArchiveDate;
             core.db.sqlCommandTimeout = 180;
             using (var csData = new CsModel(core)) {
                 if (csData.openSql(core.db.getSQLSelect("ccVisitSummary", "DateAdded", "(timeduration=1)and(Dateadded>" + DbController.encodeSQLDate(env.visitArchiveDate) + ")", "id Desc", "", 1)))
                 {
                     LastTimeSummaryWasRun = csData.getDate("DateAdded");
                     LogController.logInfo(core, "Update hourly visit summary, last time summary was run was [" + LastTimeSummaryWasRun + "]");
                 }
                 else
                 {
                     LogController.logInfo(core, "Update hourly visit summary, no hourly summaries were found, set start to [" + LastTimeSummaryWasRun + "]");
                 }
             }
             DateTime NextSummaryStartDate = LastTimeSummaryWasRun;
             //
             // Each hourly entry includes visits that started during that hour, but we do not know when they finished (maybe during last hour)
             //   Find the oldest starttime of all the visits with endtimes after the LastTimeSummaryWasRun. Resummarize all periods
             //   from then to now
             //
             //   For the past 24 hours, find the oldest visit with the last viewing during the last hour
             //
             DateTime StartOfHour     = (new DateTime(LastTimeSummaryWasRun.Year, LastTimeSummaryWasRun.Month, LastTimeSummaryWasRun.Day, LastTimeSummaryWasRun.Hour, 1, 1)).AddHours(-1); // (Int(24 * LastTimeSummaryWasRun) / 24) - PeriodStep
             DateTime OldestDateAdded = StartOfHour;
             core.db.sqlCommandTimeout = 180;
             using (var csData = new CsModel(core)) {
                 if (csData.openSql(core.db.getSQLSelect("ccVisits", "DateAdded", "LastVisitTime>" + DbController.encodeSQLDate(StartOfHour), "dateadded", "", 1)))
                 {
                     OldestDateAdded = csData.getDate("DateAdded");
                     if (OldestDateAdded < NextSummaryStartDate)
                     {
                         NextSummaryStartDate = OldestDateAdded;
                         LogController.logInfo(core, "Update hourly visit summary, found a visit with the last viewing during the past hour. It started [" + OldestDateAdded + "], before the last summary was run.");
                     }
                 }
             }
             DateTime PeriodStartDate = core.dateTimeNowMockable.Date.AddDays(-90);
             double   PeriodStep      = 1;
             int      HoursPerDay     = 0;
             core.db.sqlCommandTimeout = 180;
             for (double PeriodDatePtr = PeriodStartDate.ToOADate(); PeriodDatePtr <= OldestDateAdded.ToOADate(); PeriodDatePtr += PeriodStep)
             {
                 //
                 // Verify there are 24 hour records for every day back the past 90 days
                 //
                 DateTime DateofMissingSummary = DateTime.MinValue;
                 using (var csData = new CsModel(core)) {
                     if (csData.openSql("select count(id) as HoursPerDay from ccVisitSummary where TimeDuration=1 and DateNumber=" + encodeInteger(PeriodDatePtr) + " group by DateNumber"))
                     {
                         HoursPerDay = csData.getInteger("HoursPerDay");
                     }
                     csData.close();
                     if (HoursPerDay < 24)
                     {
                         DateofMissingSummary = DateTime.FromOADate(PeriodDatePtr);
                         break;
                     }
                 }
                 if ((DateofMissingSummary != DateTime.MinValue) && (DateofMissingSummary < NextSummaryStartDate))
                 {
                     LogController.logInfo(core, "Found a missing hourly period in the visit summary table [" + DateofMissingSummary + "], it only has [" + HoursPerDay + "] hourly summaries.");
                     NextSummaryStartDate = DateofMissingSummary;
                 }
                 //
                 // Now summarize all visits during all hourly periods between OldestDateAdded and the previous Hour
                 //
                 LogController.logInfo(core, "Summaryize visits hourly, starting [" + NextSummaryStartDate + "]");
                 PeriodStep = (double)1 / (double)24;
                 VisitSummaryClass.summarizePeriod(core, env, NextSummaryStartDate, core.dateTimeNowMockable, 1, core.siteProperties.dataBuildVersion, env.oldestVisitSummaryWeCareAbout);
             }
         }
     } catch (Exception ex) {
         LogController.logError(core, ex);
     }
 }
Exemplo n.º 3
0
 //
 //=========================================================================================
 // Summarize the visits
 //   excludes non-cookie visits
 //   excludes administrator and developer visits
 //   excludes authenticated users with ExcludeFromReporting
 //
 // Average time on site
 //
 //   Example data
 //   Pages       TimeToLastHit
 //   1           0           - hit 1 page, start time = last time
 //   10          3510        - hit 10 pages, first hit time - last hit time = 3510
 //   2           30          - hit 2 pages, first hit time - last hit time = 30
 //
 // AveReadTime is the average time spent reading pages
 //   this is calculated from the multi-page visits only
 //   = MultiPageTimeToLastHitSum / ( MultiPageHitCnt - MultiPageVisitCnt )
 //   = ( 3510 + 30 ) / ((10+2) - 2 )
 //   = 354
 //
 // TotalTimeOnSite is the total time people spent reading pages
 //   There are two parts:
 //     1) the TimeToLastHit, which covers all but the last hit of each visit
 //     2) assume the last hit of each visit is the AveReadTime
 //   = MultiPageTimeToLastHitSum + ( AveReadTime * VisitCnt )
 //   = ( 3510 + 30 ) + ( 354 * 3 )
 //   = 4602
 //
 // AveTimeOnSite
 //   = TotalTimeOnSite / TotalHits
 //   = 4602 / 3
 //   = 1534
 //
 //=========================================================================================
 //
 private static void summarizePeriod(CoreController core, HouseKeepEnvironmentModel env, DateTime StartTimeDate, DateTime EndTimeDate, int HourDuration, string BuildVersion, DateTime OldestVisitSummaryWeCareAbout)
 {
     try {
         //
         if (string.CompareOrdinal(BuildVersion, CoreController.codeVersion()) >= 0)
         {
             DateTime PeriodStart = default(DateTime);
             PeriodStart = StartTimeDate;
             if (PeriodStart < OldestVisitSummaryWeCareAbout)
             {
                 PeriodStart = OldestVisitSummaryWeCareAbout;
             }
             double StartTimeHoursSinceMidnight = PeriodStart.TimeOfDay.TotalHours;
             PeriodStart = PeriodStart.Date.AddHours(StartTimeHoursSinceMidnight);
             DateTime PeriodDatePtr = default(DateTime);
             PeriodDatePtr = PeriodStart;
             while (PeriodDatePtr < EndTimeDate)
             {
                 //
                 int      DateNumber = encodeInteger(PeriodDatePtr.AddHours(HourDuration / 2.0).ToOADate());
                 int      TimeNumber = encodeInteger(PeriodDatePtr.TimeOfDay.TotalHours);
                 DateTime DateStart  = default(DateTime);
                 DateStart = PeriodDatePtr.Date;
                 DateTime DateEnd = default(DateTime);
                 DateEnd = PeriodDatePtr.AddHours(HourDuration).Date;
                 //
                 // No Cookie Visits
                 //
                 string SQL = "select count(v.id) as NoCookieVisits"
                              + " from ccvisits v"
                              + " where (v.CookieSupport<>1)"
                              + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                              + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                              + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                              + "";
                 int NoCookieVisits = 0;
                 using (var csData = new CsModel(core)) {
                     core.db.sqlCommandTimeout = 180;
                     csData.openSql(SQL);
                     if (csData.ok())
                     {
                         NoCookieVisits = csData.getInteger("NoCookieVisits");
                     }
                 }
                 //
                 // Total Visits
                 //
                 SQL = "select count(v.id) as VisitCnt ,Sum(v.PageVisits) as HitCnt ,sum(v.TimetoLastHit) as TimeOnSite"
                       + " from ccvisits v"
                       + " where (v.CookieSupport<>0)"
                       + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                       + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                       + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                       + "";
                 //
                 int VisitCnt = 0;
                 int HitCnt   = 0;
                 using (var csData = new CsModel(core)) {
                     core.db.sqlCommandTimeout = 180;
                     csData.openSql(SQL);
                     if (csData.ok())
                     {
                         VisitCnt = csData.getInteger("VisitCnt");
                         HitCnt   = csData.getInteger("HitCnt");
                         double TimeOnSite = csData.getNumber("TimeOnSite");
                     }
                 }
                 //
                 // -- Visits by new visitors
                 int    NewVisitorVisits    = 0;
                 int    SinglePageVisits    = 0;
                 int    AuthenticatedVisits = 0;
                 int    MobileVisits        = 0;
                 int    BotVisits           = 0;
                 double AveTimeOnSite       = 0;
                 if (VisitCnt > 0)
                 {
                     SQL = "select count(v.id) as NewVisitorVisits"
                           + " from ccvisits v"
                           + " where (v.CookieSupport<>0)"
                           + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                           + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                           + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                           + " and(v.VisitorNew<>0)"
                           + "";
                     using (var csData = new CsModel(core)) {
                         core.db.sqlCommandTimeout = 180;
                         csData.openSql(SQL);
                         if (csData.ok())
                         {
                             NewVisitorVisits = csData.getInteger("NewVisitorVisits");
                         }
                     }
                     //
                     // Single Page Visits
                     //
                     SQL = "select count(v.id) as SinglePageVisits"
                           + " from ccvisits v"
                           + " where (v.CookieSupport<>0)"
                           + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                           + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                           + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                           + " and(v.PageVisits=1)"
                           + "";
                     using (var csData = new CsModel(core)) {
                         core.db.sqlCommandTimeout = 180;
                         csData.openSql(SQL);
                         if (csData.ok())
                         {
                             SinglePageVisits = csData.getInteger("SinglePageVisits");
                         }
                     }
                     //
                     // Multipage Visits
                     //
                     SQL = "select count(v.id) as VisitCnt ,sum(v.PageVisits) as HitCnt ,sum(v.TimetoLastHit) as TimetoLastHitSum "
                           + " from ccvisits v"
                           + " where (v.CookieSupport<>0)"
                           + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                           + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                           + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                           + " and(PageVisits>1)"
                           + "";
                     int    MultiPageHitCnt           = 0;
                     int    MultiPageVisitCnt         = 0;
                     double MultiPageTimetoLastHitSum = 0;
                     using (var csData = new CsModel(core)) {
                         core.db.sqlCommandTimeout = 180;
                         csData.openSql(SQL);
                         if (csData.ok())
                         {
                             MultiPageVisitCnt         = csData.getInteger("VisitCnt");
                             MultiPageHitCnt           = csData.getInteger("HitCnt");
                             MultiPageTimetoLastHitSum = csData.getNumber("TimetoLastHitSum");
                         }
                     }
                     //
                     // Authenticated Visits
                     //
                     SQL = "select count(v.id) as AuthenticatedVisits "
                           + " from ccvisits v"
                           + " where (v.CookieSupport<>0)"
                           + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                           + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                           + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                           + " and(VisitAuthenticated<>0)"
                           + "";
                     using (var csData = new CsModel(core)) {
                         core.db.sqlCommandTimeout = 180;
                         csData.openSql(SQL);
                         if (csData.ok())
                         {
                             AuthenticatedVisits = csData.getInteger("AuthenticatedVisits");
                         }
                     }
                     //
                     //
                     // Mobile Visits
                     //
                     SQL = "select count(v.id) as cnt "
                           + " from ccvisits v"
                           + " where (v.CookieSupport<>0)"
                           + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                           + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                           + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                           + " and(Mobile<>0)"
                           + "";
                     using (var csData = new CsModel(core)) {
                         core.db.sqlCommandTimeout = 180;
                         csData.openSql(SQL);
                         if (csData.ok())
                         {
                             MobileVisits = csData.getInteger("cnt");
                         }
                     }
                     //
                     // Bot Visits
                     //
                     SQL = "select count(v.id) as cnt "
                           + " from ccvisits v"
                           + " where (v.CookieSupport<>0)"
                           + " and(v.dateadded>=" + DbController.encodeSQLDate(DateStart) + ")"
                           + " and (v.dateadded<" + DbController.encodeSQLDate(DateEnd) + ")"
                           + " and((v.ExcludeFromAnalytics is null)or(v.ExcludeFromAnalytics=0))"
                           + " and(Bot<>0)"
                           + "";
                     using (var csData = new CsModel(core)) {
                         core.db.sqlCommandTimeout = 180;
                         csData.openSql(SQL);
                         if (csData.ok())
                         {
                             BotVisits = csData.getInteger("cnt");
                         }
                     }
                     //
                     if ((MultiPageHitCnt > MultiPageVisitCnt) && (HitCnt > 0))
                     {
                         int    AveReadTime     = encodeInteger(MultiPageTimetoLastHitSum / (MultiPageHitCnt - MultiPageVisitCnt));
                         double TotalTimeOnSite = MultiPageTimetoLastHitSum + (AveReadTime * VisitCnt);
                         AveTimeOnSite = TotalTimeOnSite / VisitCnt;
                     }
                 }
                 //
                 // Add or update the Visit Summary Record
                 //
                 using (var csData = new CsModel(core)) {
                     core.db.sqlCommandTimeout = 180;
                     csData.open("Visit Summary", "(timeduration=" + HourDuration + ")and(DateNumber=" + DateNumber + ")and(TimeNumber=" + TimeNumber + ")");
                     if (!csData.ok())
                     {
                         csData.close();
                         csData.insert("Visit Summary");
                     }
                     //
                     if (csData.ok())
                     {
                         csData.set("name", HourDuration + " hr summary for " + DateTime.FromOADate(DateNumber).ToShortDateString() + " " + TimeNumber + ":00");
                         csData.set("DateNumber", DateNumber);
                         csData.set("TimeNumber", TimeNumber);
                         csData.set("Visits", VisitCnt);
                         csData.set("PagesViewed", HitCnt);
                         csData.set("TimeDuration", HourDuration);
                         csData.set("NewVisitorVisits", NewVisitorVisits);
                         csData.set("SinglePageVisits", SinglePageVisits);
                         csData.set("AuthenticatedVisits", AuthenticatedVisits);
                         csData.set("NoCookieVisits", NoCookieVisits);
                         csData.set("AveTimeOnSite", AveTimeOnSite);
                         {
                             csData.set("MobileVisits", MobileVisits);
                             csData.set("BotVisits", BotVisits);
                         }
                     }
                 }
                 PeriodDatePtr = PeriodDatePtr.AddHours(HourDuration);
             }
             {
                 //
                 // Delete any daily visit summary duplicates during this period(keep the first)
                 //
                 string SQL = "delete from ccvisitsummary"
                              + " where id in ("
                              + " select d.id from ccvisitsummary d,ccvisitsummary f"
                              + " where f.datenumber=d.datenumber"
                              + " and f.datenumber>" + env.oldestVisitSummaryWeCareAbout.ToOADate() + " and f.datenumber<" + env.yesterday.ToOADate() + " and f.TimeDuration=24"
                              + " and d.TimeDuration=24"
                              + " and f.id<d.id"
                              + ")";
                 core.db.sqlCommandTimeout = 180;
                 core.db.executeNonQuery(SQL);
                 //
                 // Find missing daily summaries, summarize that date
                 //
                 SQL = core.db.getSQLSelect("ccVisitSummary", "DateNumber", "TimeDuration=24 and DateNumber>=" + env.oldestVisitSummaryWeCareAbout.Date.ToOADate(), "DateNumber,TimeNumber");
                 using (var csData = new CsModel(core)) {
                     csData.openSql(SQL);
                     DateTime datePtr = env.oldestVisitSummaryWeCareAbout;
                     while (datePtr <= env.yesterday)
                     {
                         if (!csData.ok())
                         {
                             //
                             // Out of data, start with this DatePtr
                             //
                             VisitSummaryClass.summarizePeriod(core, env, datePtr, datePtr, 24, core.siteProperties.dataBuildVersion, env.oldestVisitSummaryWeCareAbout);
                         }
                         else
                         {
                             DateTime workingDate = DateTime.MinValue.AddDays(csData.getInteger("DateNumber"));
                             if (datePtr < workingDate)
                             {
                                 //
                                 // There are missing dates, update them
                                 //
                                 VisitSummaryClass.summarizePeriod(core, env, datePtr, workingDate.AddDays(-1), 24, core.siteProperties.dataBuildVersion, env.oldestVisitSummaryWeCareAbout);
                             }
                         }
                         if (csData.ok())
                         {
                             //
                             // if there is more data, go to the next record
                             //
                             csData.goNext();
                         }
                         datePtr = datePtr.AddDays(1).Date;
                     }
                     csData.close();
                 }
             }
         }
         //
         return;
     } catch (Exception ex) {
         LogController.logError(core, ex);
     }
 }