public static void Update()
{
string message;
try
{
VotePage.LogInfo("UpdateOfficesAlternateOfficeLevel", "Started");
var alternateOfficeLevelsUpdated = 0;
foreach (var row in Offices.GetAllAlternateOfficeLevelUpdateData())
{
var newAlternateOfficeLevel = ComputeAlternateOfficeClass(row)
.ToInt();
if (newAlternateOfficeLevel == row.AlternateOfficeLevel)
{
continue;
}
Offices.UpdateAlternateOfficeLevel(newAlternateOfficeLevel, row.OfficeKey);
alternateOfficeLevelsUpdated++;
}
message = $"{alternateOfficeLevelsUpdated} AlternateOfficeLevels updated";
}
catch (Exception ex)
{
VotePage.LogException("UpdateOfficesAlternateOfficeLevel", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("UpdateOfficesAlternateOfficeLevel", message);
}
private static void ProcessTransaction(CacheInvalidationRowGlobal row, bool debugging = false)
{
if (debugging)
{
Debug($"row.TransactionType.ToLowerInvariant() = {row.TransactionType.ToLowerInvariant()}");
}
switch (row.TransactionType.ToLowerInvariant())
{
case "all":
VotePage.LogInfo("LocalCacheInvalidation", "Begin InvalidateAll");
InvalidateAll();
break;
case "nagsall":
InvalidateNagsAll(debugging);
break;
case "politicianimage":
InvalidatePoliticianImage(row.CacheKey);
break;
default:
throw new VoteException(
"Unidentified invalidation transaction type" + row.TransactionType);
}
}
// This method is only used by the CommonCacheCleanup which is run via the
// Windows task scheduler
public static void CleanUpCacheInvalidation()
{
string message;
try
{
VotePage.LogInfo("CleanUpCacheInvalidation", "Started");
// Get the number of days to retain, default to 7
var daysString =
ConfigurationManager.AppSettings["VoteCommonCacheInvalidationRetentionDays"];
if (!int.TryParse(daysString, out var days))
{
days = 7;
}
// Convert to a past DateTime
var expiration = DateTime.UtcNow - new TimeSpan(days, 0, 0, 0);
// Do it
var deleted = CacheInvalidation.DeleteExpiredTransactions(expiration);
message =
$"{deleted} CacheInvalidation rows deleted";
}
catch (Exception ex)
{
VotePage.LogException("CleanUpCacheInvalidation", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("CleanUpCacheInvalidation", message);
}
public static int UpdateSitemapVirtualPage()
{
string message;
var pages = 0;
try
{
VotePage.LogInfo("UpdateSitemapVirtualPage", "Started");
//var table = Sitemap.GetAllKeyData(0);
//foreach (var row in table)
// new SitemapManager().UpdateVirtualPage(row.DomainDataCode);
pages = new SitemapManager().UpdateVirtualPage();
message = "Completed";
}
catch (Exception ex)
{
VotePage.LogException("UpdateSitemapVirtualPage", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("UpdateSitemapVirtualPage", message);
return(pages);
}
private static void InvalidatePageCacheAll()
{
var count = CachePagesLocal.CountTable(0);
VotePage.LogInfo("LocalCacheInvalidation", "Begin InvalidatePageCacheAll:" +
count);
CachePagesLocal.TruncateTable();
VotePage.LogInfo("LocalCacheInvalidation", "End InvalidatePageCacheAll:" +
count);
}
public static void Update()
{
string message;
try
{
VotePage.LogInfo("UpdatePoliticianSearchKeys", "Started");
var alphaNamesUpdated = 0;
var vowelStrippedNamesUpdated = 0;
var table = Politicians.GetAllSearchKeyUpdateData(0);
foreach (var row in table)
{
var newAlphaName = row.LastName.StripAccents();
if (newAlphaName != row.AlphaName)
{
row.AlphaName = newAlphaName;
alphaNamesUpdated++;
}
var newVowelStrippedName = row.LastName.StripVowels();
if (newVowelStrippedName != row.VowelStrippedName)
{
row.VowelStrippedName = newVowelStrippedName;
vowelStrippedNamesUpdated++;
}
}
Politicians.UpdateTable(table, PoliticiansTable.ColumnSet.SearchKeyUpdate, 0);
message =
$"{alphaNamesUpdated} AlphaNames updated, {vowelStrippedNamesUpdated} VowelStrippedNames updated";
}
catch (Exception ex)
{
VotePage.LogException("UpdatePoliticianSearchKeys", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("UpdatePoliticianSearchKeys", message);
}
public static void Update()
{
string message;
try
{
VotePage.LogInfo("UpdateSingleCandidateContestWinners", "Started");
message =
$"{ElectionsPoliticians.MarkWinnersForSingleCandidatePastContests(0)} winners updated";
}
catch (Exception ex)
{
VotePage.LogException("UpdateSingleCandidateContestWinners", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("UpdateSingleCandidateContestWinners", message);
}
// This method is only used by the CommonCacheInvalidation which is run via the
// Windows task scheduler
public static void ProcessPendingTransactions()
{
string message;
try
{
VotePage.LogInfo("CommonCacheInvalidation", "Started");
var table = CacheInvalidation.GetUnprocessedData();
var count = table.Sum(ProcessTransaction);
message =
$"{table.Count()} CacheInvalidation rows processed, {count} Page rows deleted";
}
catch (Exception ex)
{
VotePage.LogException("CommonCacheInvalidation", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("CommonCacheInvalidation", message);
}
public static void UpdateAllSitemapVirtualPages()
{
string message;
try
{
VotePage.LogInfo("UpdateAllSitemapVirtualPages", "Started");
var table = Sitemap.GetAllKeyData(0);
foreach (var row in table)
{
new SitemapManager().UpdateVirtualPage(row.DomainDataCode);
}
message = "Completed";
}
catch (Exception ex)
{
VotePage.LogException("UpdateAllSitemapVirtualPages", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("UpdateAllSitemapVirtualPages", message);
}
private static void ProcessTransaction(CacheInvalidationRow row)
{
switch (row.TransactionType.ToLowerInvariant())
{
case "all":
VotePage.LogInfo("LocalCacheInvalidation", "Begin InvalidateAll");
InvalidateAll();
break;
//case "ballotall":
// InvalidateBallotAll();
// break;
//case "ballotbydomaindesigncodeelectionkey":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 2)
// InvalidateBallotByDomainDesignCodeElectionKey(keys[0], keys[1]);
// }
// break;
//case "ballotbyelectionkey":
// InvalidateBallotByElectionKey(row.CacheKey);
// break;
//case "electedall":
// InvalidateElectedAll();
// break;
//case "electedbystatecode":
// InvalidateElectedByStateCode(row.CacheKey);
// break;
//case "electionall":
// InvalidateElectionAll();
// break;
//case "electionbyelectionkey":
// InvalidateElectionByElectionKey(row.CacheKey);
// break;
//case "introall":
// InvalidateIntroAll();
// break;
//case "introbypoliticiankey":
// InvalidateIntroByPoliticianKey(row.CacheKey);
// break;
//case "issueall":
// InvalidateIssueAll();
// break;
//case "issuebyissuekey":
// InvalidateIssueByIssueKey(row.CacheKey);
// break;
//case "issuebyofficekey":
// InvalidateIssueByOfficeKey(row.CacheKey);
// break;
//case "issuebyelectionkeyofficekey":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 2)
// InvalidateIssueByElectionKeyOfficeKey(keys[0], keys[1]);
// }
// break;
//case "issuebyofficekeyissuekey":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 2)
// InvalidateIssueByOfficeKeyIssueKey(keys[0], keys[1]);
// }
// break;
//case "issuebyelectionkeyofficekeyissuekey":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 3)
// InvalidateIssueByElectionKeyOfficeKeyIssueKey(
// keys[0], keys[1], keys[2]);
// }
// break;
case "nagsall":
InvalidateNagsAll();
break;
//case "officialsall":
// InvalidateOfficialsAll();
// break;
//case "officialsbystatecode":
// InvalidateOfficialsByStateCode(row.CacheKey);
// break;
//case "officialsbystatecodecountycode":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 2)
// InvalidateOfficialsByStateCodeCountyCode(keys[0], keys[1]);
// }
// break;
//case "officialsbystatecodecountycodelocalcode":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 3)
// InvalidateOfficialsByStateCodeCountyCodeLocalCode(
// keys[0], keys[1], keys[2]);
// }
// break;
case "politicianimage":
InvalidatePoliticianImage(row.CacheKey);
break;
//case "politicianissueall":
// InvalidatePoliticianIssueAll();
// break;
//case "politicianissuebyissuekey":
// InvalidatePoliticianIssueByIssuekey(row.CacheKey);
// break;
//case "politicianissuebypoliticiankey":
// InvalidatePoliticianIssueByPoliticianKey(row.CacheKey);
// break;
//case "politicianissuebypoliticiankeyissuekey":
// {
// var keys = row.CacheKey.Split('|');
// if (keys.Length == 2)
// InvalidatePoliticianIssueByPoliticianKeyIssueKey(keys[0], keys[1]);
// }
// break;
//case "referendumall":
// InvalidateReferendumAll();
// break;
//case "referendumbyelectionkey":
// InvalidateReferendumByElectionkey(row.CacheKey);
// break;
default:
throw new VoteException(
"Unidentified invalidation transaction type" + row.TransactionType);
}
}
public static void Update()
{
string message;
try
{
VotePage.LogInfo("UpdatePoliticiansLiveOfficeKey", "Started");
var officeKeysUpdated = 0;
var officeStatusesUpdated = 0;
var electionKeysUpdated = 0;
var dictionary = PoliticiansLiveOfficeKeyView.GetAllData(commandTimeout: 3600)
.ToDictionary(row => row.PoliticianKey, row => row);
var politiciansTable = Politicians.GetAllLiveOfficeData();
foreach (var politiciansRow in politiciansTable)
{
if (!dictionary.TryGetValue(politiciansRow.PoliticianKey, out var viewRow))
{
continue;
}
var keyAndStatus =
PoliticianOfficeStatus.FromLiveOfficeKeyAndStatus(
viewRow.LiveOfficeKeyAndStatus);
if (keyAndStatus.OfficeKey != politiciansRow.LiveOfficeKey)
{
Politicians.UpdateLiveOfficeKey(keyAndStatus.OfficeKey,
politiciansRow.PoliticianKey);
officeKeysUpdated++;
}
if (keyAndStatus.PoliticianStatus.ToString() !=
politiciansRow.LiveOfficeStatus)
{
Politicians.UpdateLiveOfficeStatus(
keyAndStatus.PoliticianStatus.ToString(), politiciansRow.PoliticianKey);
officeStatusesUpdated++;
}
// ReSharper disable InvertIf
if (keyAndStatus.ElectionKey != politiciansRow.LiveElectionKey)
// ReSharper restore InvertIf
{
Politicians.UpdateLiveElectionKey(
keyAndStatus.ElectionKey, politiciansRow.PoliticianKey);
electionKeysUpdated++;
}
}
message =
$"{officeKeysUpdated} LiveOfficeKeys updated," +
$" {officeStatusesUpdated} LiveOfficeStatuses updated," +
$" {electionKeysUpdated} LiveElectionKeys updated";
}
catch (Exception ex)
{
VotePage.LogException("UpdatePoliticiansLiveOfficeKey", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("UpdatePoliticiansLiveOfficeKey", message);
}
private static void ServeImageContent(string politicianKey, string column,
string defaultColumn, DateTime lastModDate, bool isModified, bool noCache)
{
var response = HttpContext.Current.Response;
//var maxAge = new TimeSpan(24, 0, 0); // 24 hours -- used in headers
var maxAge = new TimeSpan(0, 0, 0); // force a server query always
var expiration = DateTime.UtcNow + maxAge;
var isProfileOriginal =
column.IsEqIgnoreCase(
PoliticiansImagesBlobs.GetColumnName(
PoliticiansImagesBlobs.Column.ProfileOriginal));
if (!isModified)
{
response.StatusCode = 304;
response.StatusDescription = "Not Modified";
}
response.Cache.SetCacheability(HttpCacheability.Public);
response.Cache.SetETag('"' +
lastModDate.Ticks.ToString(CultureInfo.InvariantCulture) + '"');
response.Cache.SetLastModified(lastModDate);
response.Cache.SetMaxAge(maxAge);
response.Cache.SetExpires(expiration);
response.Cache.SetSlidingExpiration(false);
response.Cache.SetRevalidation(HttpCacheRevalidation.AllCaches);
if (isModified) // serve actual image
{
var cache = HttpContext.Current.Cache;
var cacheKey = "politicianImage." + politicianKey + "." + column;
ApplicationCacheEntry cacheEntry = null;
// We don't use memory cache for profile originals
if (!noCache && !isProfileOriginal)
{
cacheEntry = cache.Get(cacheKey) as ApplicationCacheEntry;
}
if (cacheEntry != null && cacheEntry.Expiration > DateTime.UtcNow)
{
response.ContentType = cacheEntry.ContentType;
response.BinaryWrite(cacheEntry.Blob);
Interlocked.Increment(ref CurrentStatistics.ImagesServedFromMemoryCache);
}
else
{
var blob = GetPoliticianImage(politicianKey, column, defaultColumn,
noCache);
if (blob != null)
{
// we get the content type from the actual image
var contentType = GetContentType(blob);
response.ContentType = contentType;
response.BinaryWrite(blob);
if (!isProfileOriginal)
{
var cacheExpiration =
DateTime.UtcNow.AddMinutes(MemCache.CacheExpiration);
cacheEntry = new ApplicationCacheEntry
{
Expiration = cacheExpiration,
Blob = blob,
ContentType = contentType
};
cache.Add(cacheKey, cacheEntry, null, cacheExpiration,
Cache.NoSlidingExpiration, CacheItemPriority.Normal, null);
}
Interlocked.Increment(ref CurrentStatistics.ImagesServedFromDisc);
}
}
}
else // tell browser to use cached version
{
response.AddHeader("Content-Length", "0");
Interlocked.Increment(ref CurrentStatistics.ImagesServedFromBrowserCache);
}
if (_LastStatisticsSnapshot.SnapshotTime + StatisticsSnapshotInterval <
DateTime.UtcNow)
{
// May need to snapshot statistics, but make sure only one thread does it
if (Interlocked.Exchange(ref _SnapshottingStatistics, 1) == 0)
// it's our job
{
var previousSnapshot = _LastStatisticsSnapshot;
_LastStatisticsSnapshot = CurrentStatistics.Clone();
_LastStatisticsSnapshot.SnapshotTime = DateTime.UtcNow;
VotePage.LogInfo("ImageCaching",
$"Image caching from {previousSnapshot.SnapshotTime} to {_LastStatisticsSnapshot.SnapshotTime}:" +
$" {_LastStatisticsSnapshot.ImagesServedFromBrowserCache - previousSnapshot.ImagesServedFromBrowserCache} from browser cache," +
$" {_LastStatisticsSnapshot.ImagesServedFromMemoryCache - previousSnapshot.ImagesServedFromMemoryCache} from memory cache," +
$" {_LastStatisticsSnapshot.ImagesServedFromDisc - previousSnapshot.ImagesServedFromDisc} from disc");
Interlocked.Exchange(ref _SnapshottingStatistics, 0);
}
}
response.End();
}
public static void RefreshYouTubePoliticians()
{
// this is so we only do it every three days
if (DateTime.UtcNow.Day % 3 != 0)
{
return;
}
string message;
try
{
VotePage.LogInfo("RefreshYouTubePoliticians", "Started");
var politiciansUpdated = 0;
var table = Politicians.GetYouTubeRefreshData();
foreach (var row in table)
{
YouTubeInfo info = null;
var url = row.YouTubeWebAddress;
var videoType = string.Empty;
if (url.IsValidYouTubeVideoUrl())
{
var id = url.GetYouTubeVideoId();
info = GetVideoInfo(id, false, 1);
videoType = "video";
}
else if (url.IsValidYouTubePlaylistUrl())
{
var id = url.GetYouTubePlaylistId();
info = GetPlaylistInfo(id, false, 1);
videoType = "playlist";
}
else if (url.IsValidYouTubeChannelUrl() || url.IsValidYouTubeCustomChannelUrl() ||
url.IsValidYouTubeUserChannelUrl())
{
var id = LookupChannelId(url, 1);
if (!string.IsNullOrWhiteSpace(id))
{
info = GetChannelInfo(id, false, 1);
}
videoType = "channel";
}
var error = string.Empty;
if (info == null || !info.IsValid)
{
switch (videoType)
{
case "video":
error = YouTubeInfo.VideoIdNotFoundMessage;
break;
case "playlist":
error = YouTubeInfo.PlaylistIdNotFoundMessage;
break;
case "channel":
error = YouTubeInfo.ChannelIdNotFoundMessage;
break;
default:
error = YouTubeInfo.InvalidVideoUrlMessage;
break;
}
if (row.YouTubeAutoDisable != error)
{
row.YouTubeAutoDisable = error;
politiciansUpdated++;
}
continue;
}
if (!info.IsPublic)
{
switch (videoType)
{
case "video":
error = YouTubeInfo.VideoNotPublicMessage;
break;
case "playlist":
error = YouTubeInfo.PlaylistNotPublicMessage;
break;
case "channel":
error = YouTubeInfo.ChannelNotPublicMessage;
break;
}
if (row.YouTubeAutoDisable != error)
{
row.YouTubeAutoDisable = error;
politiciansUpdated++;
}
continue;
}
if (info.ShortDescription != row.YouTubeDescription ||
info.Duration != row.YouTubeRunningTime ||
info.PublishedAt != row.YouTubeDate ||
row.YouTubeAutoDisable != null)
{
row.YouTubeDescription = info.ShortDescription;
row.YouTubeRunningTime = info.Duration;
row.YouTubeAutoDisable = null;
row.YouTubeDate = info.PublishedAt;
politiciansUpdated++;
}
}
Politicians.UpdateTable(table, PoliticiansTable.ColumnSet.YouTubeRefresh);
message =
$"{table.Count} Expired YouTube Politicians found, {politiciansUpdated} YouTube Politicians updated";
}
catch (Exception ex)
{
VotePage.LogException("RefreshYouTubePoliticians", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("RefreshYouTubePoliticians", message);
}
public static void RefreshYouTubeAnswers()
{
string message;
var expiration = new TimeSpan(3, 0, 0, 0); // 3 days
// fudge is to account for small differences in run time
var fudge = new TimeSpan(1, 0, 0); // 1 hour
var now = DateTime.UtcNow;
try
{
VotePage.LogInfo("UpdatePoliticiansLiveOfficeKey", "Started");
var answersUpdated = 0;
var table = Answers.GetDataForYouTubeRefresh(now - expiration + fudge);
foreach (var row in table)
{
row.YouTubeRefreshTime = now;
var youTubeId = row.YouTubeUrl.GetYouTubeVideoId();
if (string.IsNullOrWhiteSpace(youTubeId))
{
if (row.YouTubeAutoDisable != YouTubeInfo.InvalidVideoUrlMessage)
{
row.YouTubeAutoDisable = YouTubeInfo.InvalidVideoUrlMessage;
answersUpdated++;
}
continue;
}
var videoInfo = GetVideoInfo(youTubeId, false, 1);
if (!videoInfo.IsValid)
{
if (row.YouTubeAutoDisable != YouTubeInfo.VideoIdNotFoundMessage)
{
row.YouTubeAutoDisable = YouTubeInfo.VideoIdNotFoundMessage;
answersUpdated++;
}
continue;
}
if (!videoInfo.IsPublic)
{
if (row.YouTubeAutoDisable != YouTubeInfo.VideoNotPublicMessage)
{
row.YouTubeAutoDisable = YouTubeInfo.VideoNotPublicMessage;
answersUpdated++;
}
continue;
}
var description = videoInfo.Description.Trim();
if (string.IsNullOrWhiteSpace(description) || description.Length > YouTubeInfo.MaxVideoDescriptionLength)
{
description = videoInfo.Title.Trim();
}
if (description != row.YouTubeDescription ||
videoInfo.Duration != row.YouTubeRunningTime ||
row.YouTubeAutoDisable != null ||
row.YouTubeSource == YouTubeInfo.VideoUploadedByCandidateMessage &&
videoInfo.PublishedAt != row.YouTubeDate)
{
row.YouTubeDescription = description;
row.YouTubeRunningTime = videoInfo.Duration;
row.YouTubeAutoDisable = null;
if (row.YouTubeSource == YouTubeInfo.VideoUploadedByCandidateMessage)
{
row.YouTubeDate = videoInfo.PublishedAt;
}
answersUpdated++;
}
}
Answers.UpdateTable(table);
message =
$"{table.Count} Expired YouTube answers found, {answersUpdated} YouTube Answers updated";
}
catch (Exception ex)
{
VotePage.LogException("RefreshYouTubeAnswers", ex);
message = $"Exception: {ex.Message} [see exception log for details]";
}
VotePage.LogInfo("RefreshYouTubeAnswers", message);
}