public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]
HttpRequestMessage req, TraceWriter log)
{
Startup.Init();
TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter();
tokenAllocationTableAdapter.Init();
List <FrequencyEntity> entities = tokenAllocationTableAdapter.GetAllCountGte(TokenAllocationTableAdapter.PartitionLabel, 8)
.Where(x => x.Ignore == false).OrderByDescending(x => x.Count).ToList();
int nextIndex = 1;
if (entities.Any(x => x.AllocatedIndex.HasValue))
{
nextIndex = entities.Max(x => x.AllocatedIndex.Value) + 1;
}
List <FrequencyEntity> updatedEntities = new List <FrequencyEntity>();
int labelAllocatedCount = 0;
foreach (FrequencyEntity frequencyEntity in entities)
{
if (!frequencyEntity.AllocatedIndex.HasValue)
{
frequencyEntity.AllocatedIndex = nextIndex++;
labelAllocatedCount++;
updatedEntities.Add(frequencyEntity);
}
}
tokenAllocationTableAdapter.Update(updatedEntities);
entities = tokenAllocationTableAdapter.GetAllCountGte(TokenAllocationTableAdapter.PartitionDigram, 7)
.Where(x => x.Ignore == false).OrderByDescending(x => x.Count).ToList();
nextIndex = 1;
if (entities.Any(x => x.AllocatedIndex.HasValue))
{
nextIndex = entities.Max(x => x.AllocatedIndex.Value) + 1;
}
updatedEntities = new List <FrequencyEntity>();
int digramAllocatedCount = 0;
foreach (FrequencyEntity frequencyEntity in entities)
{
if (!frequencyEntity.AllocatedIndex.HasValue)
{
frequencyEntity.AllocatedIndex = nextIndex++;
digramAllocatedCount++;
updatedEntities.Add(frequencyEntity);
}
}
tokenAllocationTableAdapter.Update(updatedEntities);
return(req.CreateResponse(HttpStatusCode.OK, $"Allocated {labelAllocatedCount} label indexes and {digramAllocatedCount} digram indexes"));
}
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getfrequencies")]
HttpRequestMessage req, TraceWriter log)
{
Startup.Init();
ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();
imageAnalysisTableAdapter.Init();
TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter();
tokenAllocationTableAdapter.Init();
List <ImageAnalysisEntity> analyses = imageAnalysisTableAdapter.GetAllCanonical();
Dictionary <string, int> digramFrequencies = new Dictionary <string, int>();
Dictionary <string, int> labelFrequencies = new Dictionary <string, int>();
int processedCount = 0;
foreach (ImageAnalysisEntity entity in analyses)
{
ImageAnalysis canonicalAnalysis = JsonConvert.DeserializeObject <ImageAnalysis>(entity.CanonicalJson);
UpdateCounts(StringTokenizer.GetDigrams(canonicalAnalysis.TokenizedText), digramFrequencies);
UpdateCounts(canonicalAnalysis.Labels.Keys.Distinct(), labelFrequencies);
processedCount++;
if (processedCount % 100 == 0)
{
log.Info($"Processed frequencies for {processedCount} image analyses");
}
}
log.Info($"Inserting {digramFrequencies.Count} digrams");
tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionDigram, digramFrequencies);
log.Info($"Inserting {labelFrequencies.Count} labels");
tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionLabel, labelFrequencies);
return(req.CreateResponse(HttpStatusCode.OK, $"Processed {analyses.Count} analyses"));
}
