public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequestMessage req, TraceWriter log) { Startup.Init(); TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter(); tokenAllocationTableAdapter.Init(); List <FrequencyEntity> entities = tokenAllocationTableAdapter.GetAllCountGte(TokenAllocationTableAdapter.PartitionLabel, 8) .Where(x => x.Ignore == false).OrderByDescending(x => x.Count).ToList(); int nextIndex = 1; if (entities.Any(x => x.AllocatedIndex.HasValue)) { nextIndex = entities.Max(x => x.AllocatedIndex.Value) + 1; } List <FrequencyEntity> updatedEntities = new List <FrequencyEntity>(); int labelAllocatedCount = 0; foreach (FrequencyEntity frequencyEntity in entities) { if (!frequencyEntity.AllocatedIndex.HasValue) { frequencyEntity.AllocatedIndex = nextIndex++; labelAllocatedCount++; updatedEntities.Add(frequencyEntity); } } tokenAllocationTableAdapter.Update(updatedEntities); entities = tokenAllocationTableAdapter.GetAllCountGte(TokenAllocationTableAdapter.PartitionDigram, 7) .Where(x => x.Ignore == false).OrderByDescending(x => x.Count).ToList(); nextIndex = 1; if (entities.Any(x => x.AllocatedIndex.HasValue)) { nextIndex = entities.Max(x => x.AllocatedIndex.Value) + 1; } updatedEntities = new List <FrequencyEntity>(); int digramAllocatedCount = 0; foreach (FrequencyEntity frequencyEntity in entities) { if (!frequencyEntity.AllocatedIndex.HasValue) { frequencyEntity.AllocatedIndex = nextIndex++; digramAllocatedCount++; updatedEntities.Add(frequencyEntity); } } tokenAllocationTableAdapter.Update(updatedEntities); return(req.CreateResponse(HttpStatusCode.OK, $"Allocated {labelAllocatedCount} label indexes and {digramAllocatedCount} digram indexes")); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getfrequencies")] HttpRequestMessage req, TraceWriter log) { Startup.Init(); ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter(); tokenAllocationTableAdapter.Init(); List <ImageAnalysisEntity> analyses = imageAnalysisTableAdapter.GetAllCanonical(); Dictionary <string, int> digramFrequencies = new Dictionary <string, int>(); Dictionary <string, int> labelFrequencies = new Dictionary <string, int>(); int processedCount = 0; foreach (ImageAnalysisEntity entity in analyses) { ImageAnalysis canonicalAnalysis = JsonConvert.DeserializeObject <ImageAnalysis>(entity.CanonicalJson); UpdateCounts(StringTokenizer.GetDigrams(canonicalAnalysis.TokenizedText), digramFrequencies); UpdateCounts(canonicalAnalysis.Labels.Keys.Distinct(), labelFrequencies); processedCount++; if (processedCount % 100 == 0) { log.Info($"Processed frequencies for {processedCount} image analyses"); } } log.Info($"Inserting {digramFrequencies.Count} digrams"); tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionDigram, digramFrequencies); log.Info($"Inserting {labelFrequencies.Count} labels"); tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionLabel, labelFrequencies); return(req.CreateResponse(HttpStatusCode.OK, $"Processed {analyses.Count} analyses")); }