private int GetTotalEntity(TotalEntity toGet)
{
// don't want to just return the in-memory one since it could be out of date
TotalEntity result = (TotalEntity)statsTable.Execute(TableOperation.Retrieve <TotalEntity>(toGet.PartitionKey, toGet.RowKey)).Result;
toGet.ItemCount = result.ItemCount;
return(result.ItemCount);
}
public void IncrementUrlTableLength()
{
TotalEntity temp = (TotalEntity)statsTable.Execute(TableOperation
.Retrieve <TotalEntity>(urlTotalCount.PartitionKey, urlTotalCount.RowKey)).Result;
if (Math.Abs(temp.ItemCount - urlTotalCount.ItemCount) > 10)
{
// de-sync noticed, update our local to the remote
urlTotalCount = temp;
}
urlTotalCount.IncrementCount();
statsTable.Execute(TableOperation.InsertOrReplace(urlTotalCount));
}
public void IncrementTotalUrlsCrawled()
{
TotalEntity temp = (TotalEntity)statsTable.Execute(TableOperation
.Retrieve <TotalEntity>(allUrlsCrawled.PartitionKey, allUrlsCrawled.RowKey)).Result;
// allow for some wiggle room, in case the request takes a while
if (Math.Abs(temp.ItemCount - allUrlsCrawled.ItemCount) > 10)
{
// de-sync noticed, update our local to the remote
allUrlsCrawled = temp;
}
allUrlsCrawled.IncrementCount();
statsTable.Execute(TableOperation.InsertOrReplace(allUrlsCrawled));
}
public void ClearTables()
{
urlTable.DeleteIfExists();
urlTotalCount = new TotalEntity("urlsIndexed", 0);
statsTable.Execute(TableOperation.InsertOrReplace(urlTotalCount));
allUrlsCrawled = new TotalEntity("allUrls", 0);
statsTable.Execute(TableOperation.InsertOrReplace(allUrlsCrawled));
command = new WorkerCommandEntity(false);
statsTable.Execute(TableOperation.InsertOrReplace(command));
recentUrls = new RecentUrlEntity();
statsTable.Execute(TableOperation.InsertOrReplace(recentUrls));
TableQuery query = new TableQuery().Where(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, ErrorEntity.ERROR_PARTITION));
foreach (var row in statsTable.ExecuteQuery(query))
{
statsTable.Execute(TableOperation.Delete(row));
}
}
private void InitEntities()
{
// all of the following create a new local entity, try to get the remote one
// only assigning to the new one if the remote doesn't exist
// Tried making a generic function for this, doesn't work well
TotalEntity newUrlTotal = new TotalEntity("urlsIndexed", 0);
TotalEntity remoteUrlTotal = (TotalEntity)statsTable.Execute(TableOperation
.Retrieve <TotalEntity>(newUrlTotal.PartitionKey, newUrlTotal.RowKey)).Result;
if (remoteUrlTotal != null)
{
urlTotalCount = remoteUrlTotal;
}
else
{
urlTotalCount = newUrlTotal;
statsTable.Execute(TableOperation.InsertOrReplace(urlTotalCount));
}
TotalEntity newAllUrl = new TotalEntity("allUrls", 0);
TotalEntity remoteAllUrl = (TotalEntity)statsTable.Execute(TableOperation
.Retrieve <TotalEntity>(newAllUrl.PartitionKey, newAllUrl.RowKey)).Result;
if (remoteAllUrl != null)
{
allUrlsCrawled = remoteAllUrl;
}
else
{
allUrlsCrawled = newAllUrl;
statsTable.Execute(TableOperation.InsertOrReplace(allUrlsCrawled));
}
WorkerCommandEntity newCommand = new WorkerCommandEntity(false);
WorkerCommandEntity remoteCommand = (WorkerCommandEntity)statsTable.Execute(TableOperation
.Retrieve <WorkerCommandEntity>(newCommand.PartitionKey, newCommand.RowKey)).Result;
if (remoteCommand != null)
{
command = remoteCommand;
}
else
{
command = newCommand;
statsTable.Execute(TableOperation.InsertOrReplace(command));
}
RecentUrlEntity newRecents = new RecentUrlEntity();
RecentUrlEntity remoteRecents = (RecentUrlEntity)statsTable.Execute(TableOperation
.Retrieve <RecentUrlEntity>(newRecents.PartitionKey, newRecents.RowKey)).Result;
if (remoteRecents != null)
{
recentUrls = remoteRecents;
}
else
{
recentUrls = newRecents;
statsTable.Execute(TableOperation.InsertOrReplace(recentUrls));
}
}