Esempio n. 1
0
        public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]
                                              HttpRequestMessage req, TraceWriter log)
        {
            Startup.Init();

            TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter();

            tokenAllocationTableAdapter.Init();

            List <FrequencyEntity> entities = tokenAllocationTableAdapter.GetAllCountGte(TokenAllocationTableAdapter.PartitionLabel, 8)
                                              .Where(x => x.Ignore == false).OrderByDescending(x => x.Count).ToList();

            int nextIndex = 1;

            if (entities.Any(x => x.AllocatedIndex.HasValue))
            {
                nextIndex = entities.Max(x => x.AllocatedIndex.Value) + 1;
            }

            List <FrequencyEntity> updatedEntities = new List <FrequencyEntity>();
            int labelAllocatedCount = 0;

            foreach (FrequencyEntity frequencyEntity in entities)
            {
                if (!frequencyEntity.AllocatedIndex.HasValue)
                {
                    frequencyEntity.AllocatedIndex = nextIndex++;
                    labelAllocatedCount++;
                    updatedEntities.Add(frequencyEntity);
                }
            }

            tokenAllocationTableAdapter.Update(updatedEntities);

            entities = tokenAllocationTableAdapter.GetAllCountGte(TokenAllocationTableAdapter.PartitionDigram, 7)
                       .Where(x => x.Ignore == false).OrderByDescending(x => x.Count).ToList();

            nextIndex = 1;
            if (entities.Any(x => x.AllocatedIndex.HasValue))
            {
                nextIndex = entities.Max(x => x.AllocatedIndex.Value) + 1;
            }

            updatedEntities = new List <FrequencyEntity>();
            int digramAllocatedCount = 0;

            foreach (FrequencyEntity frequencyEntity in entities)
            {
                if (!frequencyEntity.AllocatedIndex.HasValue)
                {
                    frequencyEntity.AllocatedIndex = nextIndex++;
                    digramAllocatedCount++;
                    updatedEntities.Add(frequencyEntity);
                }
            }

            tokenAllocationTableAdapter.Update(updatedEntities);

            return(req.CreateResponse(HttpStatusCode.OK, $"Allocated {labelAllocatedCount} label indexes and {digramAllocatedCount} digram indexes"));
        }
Esempio n. 2
0
        public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getfrequencies")]
                                              HttpRequestMessage req, TraceWriter log)
        {
            Startup.Init();

            ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();

            imageAnalysisTableAdapter.Init();

            TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter();

            tokenAllocationTableAdapter.Init();

            List <ImageAnalysisEntity> analyses = imageAnalysisTableAdapter.GetAllCanonical();

            Dictionary <string, int> digramFrequencies = new Dictionary <string, int>();
            Dictionary <string, int> labelFrequencies  = new Dictionary <string, int>();

            int processedCount = 0;

            foreach (ImageAnalysisEntity entity in analyses)
            {
                ImageAnalysis canonicalAnalysis = JsonConvert.DeserializeObject <ImageAnalysis>(entity.CanonicalJson);

                UpdateCounts(StringTokenizer.GetDigrams(canonicalAnalysis.TokenizedText), digramFrequencies);
                UpdateCounts(canonicalAnalysis.Labels.Keys.Distinct(), labelFrequencies);
                processedCount++;
                if (processedCount % 100 == 0)
                {
                    log.Info($"Processed frequencies for {processedCount} image analyses");
                }
            }

            log.Info($"Inserting {digramFrequencies.Count} digrams");
            tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionDigram, digramFrequencies);

            log.Info($"Inserting {labelFrequencies.Count} labels");
            tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionLabel, labelFrequencies);

            return(req.CreateResponse(HttpStatusCode.OK, $"Processed {analyses.Count} analyses"));
        }