コード例 #1
0
 public void AddAzureIndexField(AzureField indexField)
 {
     if (!string.IsNullOrEmpty(indexField.Name) && !this.AzureIndexFields.Any(f => f.Name == indexField.Name))
     {
         this.AzureIndexFields.Enqueue(indexField);
     }
 }
コード例 #2
0
 public void AddAzureIndexField(AzureField indexField)
 {
     if (!string.IsNullOrEmpty(indexField.Name) && !this.AzureIndexFields.Any(f => f.Name == indexField.Name))
     {
         this.AzureIndexFields.Enqueue(indexField);
     }
 }
コード例 #3
0
        public void BuildAzureIndexSchema(AzureField keyField, AzureField idField)
        {
            if (!this.AzureSchemaBuilt)
            {
                try
                {
                    //this.AzureIndexFields = this.AzureIndexFields.Where(f => f.Name != keyField.Name).ToList();
                    AddAzureIndexField(keyField);
                    AddAzureIndexField(idField);

                    var indexName = index.Name;
                    var fields = AzureIndexFields
                        .GroupBy(f => f.Name)
                        .Select(f => f.First().Field).ToList();

                    var definition = new Index()
                    {
                        Name = indexName,
                        Fields = fields
                    };

                    var boostFields = AzureIndexFields.Where(f => f.Boost > 0 && f.Field.IsSearchable);
                    if (boostFields.Any())
                    {
                        var scoringProfile = new ScoringProfile();
                        scoringProfile.Name = index.AzureConfiguration.AzureDefaultScoringProfileName;
                        scoringProfile.TextWeights = new TextWeights(new Dictionary<string, double>());

                        foreach (var boostField in boostFields)
                        {
                            if (!scoringProfile.TextWeights.Weights.Any(w => w.Key == boostField.Name))
                            {
                                scoringProfile.TextWeights.Weights.Add(boostField.Name, boostField.Boost);
                            }
                        }

                        if (scoringProfile.TextWeights.Weights.Any())
                        {
                            definition.ScoringProfiles = new List<ScoringProfile>();
                            definition.ScoringProfiles.Add(scoringProfile);
                            definition.DefaultScoringProfile = index.AzureConfiguration.AzureDefaultScoringProfileName;
                        }
                    }

                    AzureIndex = index.AzureServiceClient.Indexes.Create(definition);
                    this.AzureSchemaBuilt = true;
                }
                catch (Exception ex)
                {
                    CrawlingLog.Log.Fatal("Error creating index" + index.Name, ex);
                }
            }
        }
コード例 #4
0
        public void BuildAzureIndexSchema(AzureField keyField, AzureField idField)
        {
            if (!this.AzureSchemaBuilt)
            {
                try
                {
                    //this.AzureIndexFields = this.AzureIndexFields.Where(f => f.Name != keyField.Name).ToList();
                    AddAzureIndexField(keyField);
                    AddAzureIndexField(idField);

                    var indexName = index.Name;
                    var fields    = AzureIndexFields
                                    .GroupBy(f => f.Name)
                                    .Select(f => f.First().Field).ToList();

                    var definition = new Index()
                    {
                        Name   = indexName,
                        Fields = fields
                    };

                    var boostFields = AzureIndexFields.Where(f => f.Boost > 0 && f.Field.IsSearchable);
                    if (boostFields.Any())
                    {
                        var scoringProfile = new ScoringProfile();
                        scoringProfile.Name        = index.AzureConfiguration.AzureDefaultScoringProfileName;
                        scoringProfile.TextWeights = new TextWeights(new Dictionary <string, double>());

                        foreach (var boostField in boostFields)
                        {
                            if (!scoringProfile.TextWeights.Weights.Any(w => w.Key == boostField.Name))
                            {
                                scoringProfile.TextWeights.Weights.Add(boostField.Name, boostField.Boost);
                            }
                        }

                        if (scoringProfile.TextWeights.Weights.Any())
                        {
                            definition.ScoringProfiles = new List <ScoringProfile>();
                            definition.ScoringProfiles.Add(scoringProfile);
                            definition.DefaultScoringProfile = index.AzureConfiguration.AzureDefaultScoringProfileName;
                        }
                    }

                    AzureIndex            = index.AzureServiceClient.Indexes.Create(definition);
                    this.AzureSchemaBuilt = true;
                }
                catch (Exception ex)
                {
                    CrawlingLog.Log.Fatal("Error creating index" + index.Name, ex);
                }
            }
        }
コード例 #5
0
        public bool ReconcileAzureIndexSchema(Document document, int retryCount = 0)
        {
            try
            {
                var fieldCount = AzureIndex.Fields.Count;
                if (document != null)
                {
                    //Look for fields that are different from the standards:
                    foreach (var key in document.Keys)
                    {
                        if (!AzureIndexFields.Any(f => f.Name == key))
                        {
                            object objVal;
                            document.TryGetValue(key, out objVal);
                            var field      = AzureFieldBuilder.BuildField(key, objVal, index);
                            var azureField = new AzureField(key, objVal, field);
                            AddAzureIndexField(azureField);
                        }
                    }
                }

                if (AzureIndexFields.Count > fieldCount)
                {
                    var indexName = index.Name;
                    var fields    = AzureIndexFields
                                    .GroupBy(f => f.Name)
                                    .Select(f => f.First().Field).ToList();

                    AzureIndex.Fields = fields;
                    AzureIndex        = index.AzureServiceClient.Indexes.CreateOrUpdate(AzureIndex);
                }

                return(true);
            }
            catch (Exception ex)
            {
                if (!ReconcileAzureIndexSchema(null))
                {
                    Thread.Sleep(50);
                    if (retryCount < 6)
                    {
                        return(ReconcileAzureIndexSchema(null, retryCount++));
                    }
                    else
                    {
                        CrawlingLog.Log.Warn("Error updating index" + index.Name);
                    }
                }

                return(false);
            }
        }
コード例 #6
0
        public bool ReconcileAzureIndexSchema(Document document, int retryCount = 0)
        {
            try
            {
                var fieldCount = AzureIndex.Fields.Count;
                if (document != null)
                {
                    //Look for fields that are different from the standards:
                    foreach (var key in document.Keys)
                    {
                        if (!AzureIndexFields.Any(f => f.Name == key))
                        {
                            object objVal;
                            document.TryGetValue(key, out objVal);
                            var field = AzureFieldBuilder.BuildField(key, objVal, index);
                            var azureField = new AzureField(key, objVal, field);
                            AddAzureIndexField(azureField);
                        }
                    }
                }

                if (AzureIndexFields.Count > fieldCount)
                {
                    var indexName = index.Name;
                    var fields = AzureIndexFields
                        .GroupBy(f => f.Name)
                        .Select(f => f.First().Field).ToList();

                    AzureIndex.Fields = fields;
                    AzureIndex = index.AzureServiceClient.Indexes.CreateOrUpdate(AzureIndex);
                }

                return true;
            }
            catch (Exception ex)
            {
                if (!ReconcileAzureIndexSchema(null))
                {
                    Thread.Sleep(50);
                    if (retryCount < 6)
                    {
                        return ReconcileAzureIndexSchema(null, retryCount++);
                    }
                    else
                    {
                        CrawlingLog.Log.Warn("Error updating index" + index.Name);
                    }
                }
                
                return false;
            }
        }