private static string retrieveStat(string statName)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]
);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudTable statsTable = tableClient.GetTableReference(StatEntity.TABLE_STATS);
statsTable.CreateIfNotExists();
TableOperation retrieveOperation = TableOperation.Retrieve <StatEntity>(statName, statName);
TableResult retrievedResult = statsTable.Execute(retrieveOperation);
if (retrievedResult.Result != null)
{
StatEntity statEntity = (StatEntity)retrievedResult.Result;
return(statEntity.Value);
}
else
{
return("");
}
}
public string getTimer()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable statsTable = tableClient.GetTableReference("stattable");
try
{
TableQuery <StatEntity> counterquery = new TableQuery <StatEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "TomBot"));
List <StatEntity> bean = statsTable.ExecuteQuery(counterquery).ToList();
if (bean.Count != 0)
{
StatEntity getem = bean.ElementAt(0);
return("Finding links took " + getem.timer + " ms...");
}
}
catch (Exception e)
{
return("Could not access stats: " + e.Message);
}
return("Could not retrieve timer...");
}
public string getTrienhardt()
{
string stats = "Trienhardt is not here...";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable statsTable = tableClient.GetTableReference("stattable");
try
{
TableQuery <StatEntity> counterquery = new TableQuery <StatEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "Trienhardt"));
List <StatEntity> bean = statsTable.ExecuteQuery(counterquery).ToList();
if (bean.Count != 0)
{
StatEntity getem = bean.ElementAt(0);
stats = "Last Word Added: '" + getem.lastTen + "', Lines: " + getem.visitcount;
}
}
catch (Exception e)
{
return("Could not retrieve stats: " + e.Message);
}
return(stats);
}
public string buildTrie()
{
trie = new Trie();
int MAX = 1000;
int count = 0;
string term = "";
try
{
if (!File.Exists(path))
{
downloadWiki();
}
using (StreamReader sr = File.OpenText(path))
{
while (!sr.EndOfStream)
{
term = sr.ReadLine();
count++;
trie.insert(term);
if (count % MAX == 0)
{
//perform memory check to ensure not running out of memory
if (getAvailableMBytes() <= _maxMem)
{
break;
}
}
}
}
try
{
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable statsTable = tableClient.GetTableReference(Operation._STATS_TABLE);
statsTable.CreateIfNotExists();
StatEntity stat = new StatEntity();
stat.PartitionKey = Operation._TRIE_PKEY;
stat.RowKey = Operation._TRIE_RKEY;
stat.lastTenString = term;
stat.indexSize = count;
statsTable.ExecuteAsync(TableOperation.InsertOrReplace(stat));
}
catch (Exception e)
{
return("Failed to insert trie stats into table");
}
return("Successfully built trie with " + count + " words. The last word was " + term);
}
catch (Exception e)
{
return("Error building trie");
}
}
public string BuildTrie()
{
RemoveTrie();
triehard = new Trie();
string filepath = System.Web.HttpContext.Current.Server.MapPath(@"/wikititles.txt");
if (!File.Exists(filepath))
{
return("Please download the wiki titles...");
}
StreamReader reader = new StreamReader(filepath);
float startMem = GetAvailableMBytes();
float memUsed = 0;
int countLines = 0;
string currentWord = "";
while (!reader.EndOfStream)
{
if (countLines % 1000 == 0)
{
float mem = GetAvailableMBytes();
memUsed = startMem - mem;
if (mem < 50)
{
break;
}
}
currentWord = reader.ReadLine();
countLines++;
triehard.Add(currentWord.Trim().ToLower());
if (currentWord.Trim().ToLower().StartsWith("c"))
{
break;
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable statsTable = tableClient.GetTableReference("stattable");
statsTable.CreateIfNotExists();
StatEntity stats = new StatEntity();
stats.PartitionKey = "Trienhardt";
stats.RowKey = "beepboopbastion";
stats.lastTen = currentWord;
stats.visitcount = countLines;
statsTable.ExecuteAsync(TableOperation.InsertOrReplace(stats));
return("Last insert: " + currentWord + ", Lines: " + countLines + ", MemUsed: " + memUsed);
}
public long Add()
{
using (MyDbContext dbc = new MyDbContext())
{
if (dbc.GetAll <StatEntity>().AsNoTracking().Any())
{
return(-1);
}
var typeList = MyEnumHelper.GetEnumList <TypeEnum>();
var itemList = MyEnumHelper.GetEnumList <ItemEnum>();
StatEntity entity;
foreach (var type in typeList)
{
if (type.Id <= 3)
{
var items = itemList.Where(i => i.Id < 10);
foreach (var item in items)
{
entity = new StatEntity();
entity.ItemId = item.Id;
entity.TypeId = type.Id;
dbc.Stats.Add(entity);
}
}
if (type.Id == 4)
{
var items = itemList.Where(i => i.Id >= 10 && i.Id < 20);
foreach (var item in items)
{
entity = new StatEntity();
entity.ItemId = item.Id;
entity.TypeId = type.Id;
dbc.Stats.Add(entity);
}
}
if (type.Id == 5)
{
var items = itemList.Where(i => i.Id > 20);
foreach (var item in items)
{
entity = new StatEntity();
entity.ItemId = item.Id;
entity.TypeId = type.Id;
dbc.Stats.Add(entity);
}
}
}
dbc.SaveChanges();
return(1);
}
}
public bool Del(long id)
{
using (MyDbContext dbc = new MyDbContext())
{
StatEntity entity = dbc.GetAll <StatEntity>().SingleOrDefault(g => g.Id == id);
if (entity == null)
{
return(false);
}
entity.IsDeleted = true;
dbc.SaveChanges();
return(true);
}
}
public StatDTO ToDTO(StatEntity entity)
{
StatDTO dto = new StatDTO();
dto.ItemName = entity.ItemId.GetEnumName <ItemEnum>().Replace("0", "、").Replace("1", "（").Replace("2", "）");
dto.Id = entity.Id;
dto.TypeId = entity.TypeId;
dto.ItemId = entity.ItemId;
dto.TypeName = entity.TypeId.GetEnumName <TypeEnum>().Replace("0", "、").Replace("1", "（").Replace("2", "）");
dto.QuarterNumber = entity.QuarterNumber;
dto.QuarterAmount = entity.QuarterAmount;
dto.YearNumber = entity.YearNumber;
dto.YearAmount = entity.YearAmount;
dto.TotalNumber = entity.TotalNumber;
dto.TotalAmount = entity.TotalAmount;
return(dto);
}
public string getTrieStats()
{
statsTable = setTable(Operation._STATS_TABLE);
try
{
TableQuery <StatEntity> statQuery = new TableQuery <StatEntity>()
.Where(TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, Operation._TRIE_PKEY),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, Operation._TRIE_RKEY)));
StatEntity stat = statsTable.ExecuteQuery(statQuery).First();
return(new JavaScriptSerializer().Serialize(stat));
}
catch (Exception e)
{
return("Error retrieving trie stats");
}
}
public void updateStat(string statName, string statValue)
{
TableOperation retrieveOperation = TableOperation.Retrieve <StatEntity>(statName, statName);
TableResult retrievedResult = statsTable.Execute(retrieveOperation);
if (retrievedResult.Result == null)
{
StatEntity statEntity = new StatEntity(statName, statValue);
TableOperation insertOperation = TableOperation.Insert(statEntity);
statsTable.Execute(insertOperation);
}
else
{
StatEntity statEntity = (StatEntity)retrievedResult.Result;
statEntity.Value = statValue;
TableOperation updateOperation = TableOperation.Replace(statEntity);
statsTable.Execute(updateOperation);
}
}
public string getStatus()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable statsTable = tableClient.GetTableReference("stattable");
try
{
TableQuery <StatEntity> counterquery = new TableQuery <StatEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "TomBot"));
List <StatEntity> bean = statsTable.ExecuteQuery(counterquery).ToList();
if (bean.Count != 0)
{
StatEntity getem = bean.ElementAt(0);
if (getem.status == Robotom.STATUS_STOP)
{
return("TomBot is stopped...");
}
else if (getem.status == Robotom.STATUS_IDLE)
{
return("TomBot is idle...");
}
else if (getem.status == Robotom.STATUS_LOADING)
{
return("TomBot is loading...");
}
else if (getem.status == Robotom.STATUS_CRAWLING)
{
return("TomBot is crawling...");
}
}
}
catch (Exception e)
{
return("Could not access stats: " + e.Message);
}
return("Could not retrieve status...");
}
public List <int> getStats()
{
List <int> stats = new List <int>();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable statsTable = tableClient.GetTableReference("stattable");
try
{
TableQuery <StatEntity> counterquery = new TableQuery <StatEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "TomBot"));
List <StatEntity> bean = statsTable.ExecuteQuery(counterquery).ToList();
if (bean.Count != 0)
{
StatEntity getem = bean.ElementAt(0);
stats.Add(getem.memUsage);
stats.Add(getem.cpuUsage);
stats.Add(getem.queuesize);
stats.Add(getem.tablesize);
stats.Add(getem.visitcount);
}
}
catch (Exception e)
{
stats.Add(-1);
stats.Add(-1);
stats.Add(-1);
stats.Add(-1);
stats.Add(-1);
return(stats);
}
return(stats);
}
private void updateStats(StatEntity stats, CloudTable statTable)
{
stats.memUsage = unchecked ((int)memprocess.NextValue());
stats.cpuUsage = unchecked ((int)cpuprocess.NextValue());
stats.queuesize = crawler.queueCount;
stats.tablesize = crawler.tableCount;
stats.visitcount = crawler.parsed.Count();
System.Text.StringBuilder last = new System.Text.StringBuilder("");
foreach (string u in crawler.lastTen)
{
last.Append(u + ";");
}
stats.lastTen = last.ToString();
stats.timer = unchecked ((int)crawler.timer);
stats.status = status;
try
{
statTable.ExecuteAsync(TableOperation.InsertOrReplace(stats));
}
catch (Exception e)
{
Trace.TraceInformation(e.Message);
}
}
private async Task RunAsync(CancellationToken cancellationToken)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue opQueue = queueClient.GetQueueReference(Operation._OP_QUEUE);
opQueue.CreateIfNotExists();
CloudQueue robotQueue = queueClient.GetQueueReference(Operation._ROBOTS_QUEUE);
robotQueue.CreateIfNotExists();
CloudQueue urlQueue = queueClient.GetQueueReference(Operation._URLS_QUEUE);
urlQueue.CreateIfNotExists();
//Create table client
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
//Retrieve reference to pages table
CloudTable pagesTable = tableClient.GetTableReference(Operation._PAGES_TABLE);
pagesTable.CreateIfNotExists();
//retrieve reference to stats table
CloudTable statsTable = tableClient.GetTableReference(Operation._STATS_TABLE);
statsTable.CreateIfNotExists();
//retrieve reference to errors table
CloudTable errorsTable = tableClient.GetTableReference(Operation._ERRORS_TABLE);
errorsTable.CreateIfNotExists();
crawler = new Crawler(robotQueue, urlQueue, pagesTable, statsTable, errorsTable);
memCounter = new PerformanceCounter("Memory", "Available MBytes");
cpuCounter = new PerformanceCounter("Processor", "% Processor Time", "_Total");
stat = new StatEntity(cpuCounter.NextValue(), memCounter.NextValue());
updateStatsTable(statsTable);
stat.updateRunning();
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(50);
//check op queue for any commands
CloudQueueMessage command = opQueue.GetMessage(TimeSpan.FromMinutes(5));
while (command != null)
{
string commandString = command.AsString;
//update status
stat.updateStatus(Operation._IDLE);
updateStatsTable(statsTable);
if (commandString == Operation._START)
{
if (!stat.getRunning())
{
stat.updateRunning();
pagesTable = tableClient.GetTableReference(Operation._PAGES_TABLE);
pagesTable.CreateIfNotExists();
errorsTable = tableClient.GetTableReference(Operation._ERRORS_TABLE);
errorsTable.CreateIfNotExists();
}
}
else if (commandString == Operation._CLEAR)
{
stat.updateRunning();
opQueue.Clear();
robotQueue.Clear();
urlQueue.Clear();
pagesTable.DeleteIfExists();
errorsTable.DeleteIfExists();
crawler = new Crawler(robotQueue, urlQueue, pagesTable, statsTable, errorsTable);
stat = new StatEntity(cpuCounter.NextValue(), memCounter.NextValue());
updateStatsTable(statsTable);
Thread.Sleep(40000);
}
updateStatsTable(statsTable);
opQueue.DeleteMessage(command);
command = opQueue.GetMessage(TimeSpan.FromMinutes(5));
}
if (stat.getRunning())
{
//check robot queue for any robots to parse
CloudQueueMessage robotUrl = robotQueue.GetMessage(TimeSpan.FromMinutes(5));
while (robotUrl != null)
{
//update status
stat.updateStatus(Operation._LOADING);
updateStatsTable(statsTable);
int queueSizeUpdate = crawler.parseRobots(robotUrl.AsString);
//stat.updateQueue(queueSizeUpdate);
updateStatsTable(statsTable);
robotQueue.DeleteMessage(robotUrl);
robotUrl = robotQueue.GetMessage(TimeSpan.FromMinutes(5));
}
//check url queue for any urls
CloudQueueMessage queueUrl = urlQueue.GetMessage(TimeSpan.FromMinutes(5));
if (queueUrl != null)
{
//update status
stat.updateStatus(Operation._CRAWLING);
updateStatsTable(statsTable);
Tuple <int, int> crawled = crawler.crawlSite(queueUrl.AsString);
stat.updateStats(new Uri(queueUrl.AsString), crawled.Item1, crawled.Item2);
updateStatsTable(statsTable);
urlQueue.DeleteMessage(queueUrl);
}
}
else
{
stat.updateStatus(Operation._IDLE);
}
updateStatsTable(statsTable);
//Trace.TraceInformation("Working");
}
}
public override void Run()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
ConfigurationManager.AppSettings["StorageConnectionString"]);
// set up queues
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue commandQ = queueClient.GetQueueReference("commandq");
CloudQueue htmlQ = queueClient.GetQueueReference("htmlq");
htmlQ.CreateIfNotExists();
commandQ.CreateIfNotExists();
// set up tables
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable resultTable = tableClient.GetTableReference("crawltable");
CloudTable errorTable = tableClient.GetTableReference("errortable");
CloudTable statTable = tableClient.GetTableReference("stattable");
resultTable.CreateIfNotExists();
errorTable.CreateIfNotExists();
statTable.CreateIfNotExists();
this.memprocess = new PerformanceCounter("Memory", "Available MBytes");
this.cpuprocess = new PerformanceCounter("Processor", "% Processor Time", "_Total");
memprocess.NextValue();
cpuprocess.NextValue();
int tablesize = 0;
TableQuery <StatEntity> counterquery = new TableQuery <StatEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "TomBot"));
List <StatEntity> bean = resultTable.ExecuteQuery(counterquery).ToList();
if (bean.Count != 0)
{
tablesize = bean.ElementAt(0).tablesize;
}
crawler = new TomBot(htmlQ, resultTable, errorTable, tablesize);
StatEntity stats = new StatEntity();
stats.PartitionKey = "TomBot";
stats.RowKey = "beepboopstats";
status = Robotom.STATUS_IDLE;
while (true)
{
updateStats(stats, statTable);
//check commands
CloudQueueMessage command = commandQ.GetMessage();
if (command != null)
{
string commandString = command.AsString;
if (commandString.Equals(Robotom.COMMAND_STOP))
{
status = Robotom.STATUS_STOP;
updateStats(stats, statTable);
}
else if (commandString.StartsWith(Robotom.COMMAND_START))
{
status = Robotom.STATUS_LOADING;
updateStats(stats, statTable);
string robotLink = commandString.Split(' ')[1] + "/robots.txt";
Uri thing = new Uri(robotLink);
List <Uri> sitemaps = crawler.ParseRobot(thing);
if (thing.Host.Equals("cnn.com"))
{
sitemaps.AddRange(crawler.ParseRobot(new Uri("http://bleacherreport.com/robots.txt")));
}
updateStats(stats, statTable);
while (sitemaps.Count > 0)
{
Uri next = sitemaps.ElementAt(0);
sitemaps.AddRange(crawler.ParseXml(next));
sitemaps.Remove(next);
updateStats(stats, statTable);
}
}
commandQ.DeleteMessage(command);
}
if (status != Robotom.STATUS_STOP)
{
//check htmlq
CloudQueueMessage nextHtml = htmlQ.GetMessage();
if (nextHtml != null)
{
status = Robotom.STATUS_CRAWLING;
updateStats(stats, statTable);
crawler.ParseHtml(new Uri(nextHtml.AsString));
htmlQ.DeleteMessageAsync(nextHtml);
}
else
{
status = Robotom.STATUS_IDLE;
updateStats(stats, statTable);
}
}
else
{
htmlQ.Clear();
crawler.queueCount = 0;
}
Thread.Sleep(10);
}
}
public long Calc()
{
using (MyDbContext dbc = new MyDbContext())
{
if (!dbc.GetAll <StatEntity>().AsNoTracking().Any())
{
return(-1);
}
var typeList = MyEnumHelper.GetEnumList <TypeEnum>();
var itemList = MyEnumHelper.GetEnumList <ItemEnum>();
DateTime time = DateTime.Now;
StatEntity stat;
IQueryable <CheckWorkEntity> result;
foreach (var type in typeList)
{
if (type.Id <= 3)
{
var items = itemList.Where(i => i.Id < 10);
foreach (var item in items)
{
result = dbc.GetAll <CheckWorkEntity>().AsNoTracking().Where(c => c.TypeId == type.Id && c.ItemId == item.Id);
if (result.Count() <= 0)
{
continue;
}
stat = dbc.GetAll <StatEntity>().SingleOrDefault(s => s.TypeId == type.Id && s.ItemId == item.Id);
//if(time.Month)
//stat.QuarterNumber = result.Where(c=>c.CreateTime);
stat.QuarterAmount = 0;
stat.YearNumber = 0;
stat.YearAmount = 0;
stat.TotalNumber = 0;
stat.TotalAmount = 0;
dbc.SaveChanges();
}
}
if (type.Id == 4)
{
var items = itemList.Where(i => i.Id >= 10 && i.Id < 20);
foreach (var item in items)
{
stat = new StatEntity();
stat.ItemId = item.Id;
stat.TypeId = type.Id;
dbc.Stats.Add(stat);
}
}
if (type.Id == 5)
{
var items = itemList.Where(i => i.Id > 20);
foreach (var item in items)
{
stat = new StatEntity();
stat.ItemId = item.Id;
stat.TypeId = type.Id;
dbc.Stats.Add(stat);
}
}
}
dbc.SaveChanges();
return(1);
}
}
public static int RunAddAndReturnExitCode(CollectIssuesStatOptions options)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(options.Connection);
var tableClient = storageAccount.CreateCloudTableClient();
var contractTable = tableClient.GetTableReference(options.ContractTableName);
var counter = 0;
var stat = new ConcurrentDictionary <string, long>();
TableContinuationToken token = null;
do
{
var query = new TableQuery <ContractEntity> {
TakeCount = options.BatchSize
};
var segment = contractTable.ExecuteQuerySegmentedAsync(query, token).Result;
Parallel.ForEach(segment.Results, entry =>
{
if (entry.AnalysisStatus != "Finished")
{
return;
}
var content = Blob.ReadAsync(options.Connection, options.AnalysisBlobContainerName, entry.AnalysisId).Result;
if (string.IsNullOrEmpty(content))
{
return;
}
var list = JsonConvert.DeserializeObject <List <AnalysisResult> >(content);
foreach (var issue in list.SelectMany(x => x.Issues))
{
stat.AddOrUpdate(issue.SwcID, 1, (key, oldValue) => oldValue + 1);
}
});
counter += segment.Results.Count;
Console.WriteLine($"{DateTime.Now.ToString("MM/dd/yyyy HH:mm:ss.fffffff ")} Handled {counter} records");
token = segment.ContinuationToken;
} while (token != null);
var batchOperation = new TableBatchOperation();
foreach (var key in stat.Keys)
{
var entry = new StatEntity
{
PartitionKey = "IssueStat",
RowKey = key,
Count = stat[key]
};
batchOperation.InsertOrReplace(entry);
}
var statTable = tableClient.GetTableReference(options.StatTableName);
statTable.CreateIfNotExistsAsync().Wait();
statTable.ExecuteBatchAsync(batchOperation).Wait();
return(0);
}
public static int RunAddAndReturnExitCode(CollectProcessingStatOptions options)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(options.Connection);
var tableClient = storageAccount.CreateCloudTableClient();
var sourceTable = tableClient.GetTableReference(options.TableName);
var stat = new Dictionary <ProcessingStatFields, long>
{
{ ProcessingStatFields.Processed, 0 },
{ ProcessingStatFields.Failed, 0 },
{ ProcessingStatFields.Finished, 0 },
{ ProcessingStatFields.LowSeverity, 0 },
{ ProcessingStatFields.MediumSeverity, 0 },
{ ProcessingStatFields.HighSeverity, 0 },
{ ProcessingStatFields.NoIssues, 0 },
};
var counter = 0;
TableContinuationToken token = null;
do
{
var query = new TableQuery <ContractEntity> {
TakeCount = options.BatchSize
};
var segment = sourceTable.ExecuteQuerySegmentedAsync(query, token).Result;
foreach (var entry in segment.Results)
{
stat[ProcessingStatFields.Processed] += 1;
if (entry.AnalysisStatus == "Error")
{
stat[ProcessingStatFields.Failed] += 1;
}
else
{
stat[ProcessingStatFields.Finished] += 1;
}
switch (entry.Severity)
{
case "Low":
stat[ProcessingStatFields.LowSeverity] += 1;
break;
case "Medium":
stat[ProcessingStatFields.MediumSeverity] += 1;
break;
case "High":
stat[ProcessingStatFields.HighSeverity] += 1;
break;
default:
stat[ProcessingStatFields.NoIssues] += 1;
break;
}
}
counter += segment.Results.Count;
Console.WriteLine($"Handled {counter} records");
token = segment.ContinuationToken;
} while (token != null);
var batchOperation = new TableBatchOperation();
foreach (var key in stat.Keys)
{
var entry = new StatEntity
{
PartitionKey = "ProcessingStat",
RowKey = key.ToString(),
Count = stat[key]
};
batchOperation.InsertOrReplace(entry);
}
var statTable = tableClient.GetTableReference(options.StatTableName);
statTable.CreateIfNotExistsAsync().Wait();
statTable.ExecuteBatchAsync(batchOperation).Wait();
return(0);
}
public static async Task Run(
[TimerTrigger("0 0 0 * * *")] TimerInfo timer,
[Table("%Storage:ContractTable%", Connection = "Storage:Connection")] CloudTable contractTable,
[Table("%Storage:StatTable%", Connection = "Storage:Connection")] CloudTable statTable,
ILogger log,
ExecutionContext context)
{
IConfigurationRoot config = new ConfigurationBuilder()
.SetBasePath(context.FunctionAppDirectory)
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
var stat = new Dictionary <ProcessingStatFields, long>
{
{ ProcessingStatFields.Processed, 0 },
{ ProcessingStatFields.Failed, 0 },
{ ProcessingStatFields.Finished, 0 },
{ ProcessingStatFields.LowSeverity, 0 },
{ ProcessingStatFields.MediumSeverity, 0 },
{ ProcessingStatFields.HighSeverity, 0 },
{ ProcessingStatFields.NoIssues, 0 },
};
TableContinuationToken token = null;
do
{
var query = new TableQuery <ContractEntity> {
TakeCount = 100
};
var segment = contractTable.ExecuteQuerySegmentedAsync(query, token).Result;
foreach (var entry in segment.Results)
{
stat[ProcessingStatFields.Processed] += 1;
if (entry.AnalysisStatus == "Error")
{
stat[ProcessingStatFields.Failed] += 1;
}
else
{
stat[ProcessingStatFields.Finished] += 1;
}
switch (entry.Severity)
{
case "Low":
stat[ProcessingStatFields.LowSeverity] += 1;
break;
case "Medium":
stat[ProcessingStatFields.MediumSeverity] += 1;
break;
case "High":
stat[ProcessingStatFields.HighSeverity] += 1;
break;
default:
stat[ProcessingStatFields.NoIssues] += 1;
break;
}
}
token = segment.ContinuationToken;
} while (token != null);
var batchOperation = new TableBatchOperation();
foreach (var key in stat.Keys)
{
var entry = new StatEntity
{
PartitionKey = "ProcessingStat",
RowKey = key.ToString(),
Count = stat[key]
};
batchOperation.InsertOrReplace(entry);
}
await statTable.ExecuteBatchAsync(batchOperation);
}
