private Field BuildKeyField(string name)
{
var field = AzureFieldBuilder.BuildField(name, "", this.AzureIndex);
field.IsKey = true;
return(field);
}
public override void AddField(string fieldName, object fieldValue, bool append = false)
{
var fieldConfiguration = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName);
var fieldName1 = fieldName;
fieldName = this.Index.FieldNameTranslator.GetIndexFieldName(fieldName);
var fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName) as AzureSearchFieldConfiguration;
if (fieldSettings != null)
{
if (fieldConfiguration != null)
{
fieldValue = fieldConfiguration.FormatForWriting(fieldValue);
}
this.AddField(fieldName, fieldValue, fieldSettings, 0.0f);
}
else
{
if (VerboseLogging.Enabled)
{
var stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Field: {0} (Adding field with no field configuration)" + Environment.NewLine, (object)fieldName);
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue != null ? (object)fieldValue.GetType().ToString() : (object)"NULL");
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue);
VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString));
}
var enumerable = fieldValue as IEnumerable;
if (enumerable != null && !(fieldValue is string))
{
foreach (object obj in enumerable)
{
//var valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(obj, fieldName1);
//if (fieldConfiguration != null)
// fieldName1 = fieldConfiguration.FormatForWriting(fieldName1);
if (fieldName1 != null)
{
if (QueueContainsName(fieldName))
{
var field = this.fields.FirstOrDefault(f => f.Name == fieldName);
}
else
{
this.fields.Enqueue(new AzureField(fieldName, fieldName1, AzureFieldBuilder.BuildField(fieldName, fieldName1, this.Index as IAzureProviderIndex)));
}
}
}
}
else
{
//var valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(fieldValue, fieldName1);
//if (fieldConfiguration != null)
// valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex);
if (fieldName1 != null)
{
this.fields.Enqueue(new AzureField(fieldName, fieldValue, AzureFieldBuilder.BuildField(fieldName, fieldValue, this.Index as IAzureProviderIndex)));
}
}
}
}
protected void AddField(string name, object value, AzureSearchFieldConfiguration fieldSettings, float boost = 0.0f)
{
Assert.IsNotNull((object)fieldSettings, "fieldSettings");
name = this.Index.FieldNameTranslator.GetIndexFieldName(name);
boost += fieldSettings.Boost;
IEnumerable enumerable = value as IEnumerable;
if (enumerable != null && !(value is string))
{
foreach (object valueToIndex in enumerable)
{
object obj = fieldSettings.FormatForWriting(valueToIndex);
var field = AzureFieldBuilder.CreateField(name, obj, fieldSettings, this.Index.Configuration.IndexFieldStorageValueFormatter);
if (field != null)
{
field.Boost = boost;
this.fields.Enqueue(field);
}
}
}
else
{
value = fieldSettings.FormatForWriting(value);
var field = AzureFieldBuilder.CreateField(name, value, fieldSettings, this.Index.Configuration.IndexFieldStorageValueFormatter);
if (field == null)
{
return;
}
field.Boost = boost;
this.fields.Enqueue(field);
}
}
private int GetSortFieldType(OrderByMethod orderByMethod)
{
if (orderByMethod.FieldType == typeof(string))
{
return(3);
}
if (AzureFieldBuilder.IsNumericField(orderByMethod.FieldType))
{
return(6);
}
return(AzureFieldBuilder.IsFloatingPointField(orderByMethod.FieldType) ? 7 : 3);
}
public bool ReconcileAzureIndexSchema(Document document, int retryCount = 0)
{
try
{
var fieldCount = AzureIndex.Fields.Count;
if (document != null)
{
//Look for fields that are different from the standards:
foreach (var key in document.Keys)
{
if (!AzureIndexFields.Any(f => f.Name == key))
{
object objVal;
document.TryGetValue(key, out objVal);
var field = AzureFieldBuilder.BuildField(key, objVal, index);
var azureField = new AzureField(key, objVal, field);
AddAzureIndexField(azureField);
}
}
}
if (AzureIndexFields.Count > fieldCount)
{
var indexName = index.Name;
var fields = AzureIndexFields
.GroupBy(f => f.Name)
.Select(f => f.First().Field).ToList();
AzureIndex.Fields = fields;
AzureIndex = index.AzureServiceClient.Indexes.CreateOrUpdate(AzureIndex);
}
return(true);
}
catch (Exception ex)
{
if (!ReconcileAzureIndexSchema(null))
{
Thread.Sleep(50);
if (retryCount < 6)
{
return(ReconcileAzureIndexSchema(null, retryCount++));
}
else
{
CrawlingLog.Log.Warn("Error updating index" + index.Name);
}
}
return(false);
}
}
public IndexData(ISearchIndex index, IIndexable indexable, Document document, ConcurrentQueue <AzureField> fields)
{
this.Document = document;
this.Fields = fields;
this.AzureIndex = (IAzureProviderIndex)index;
this.UpdateTerm = new AzureField("s_key", index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(indexable.UniqueId.Value, "s_key"), BuildKeyField("s_key"));
this.FullUpdateTerm = new AzureField("s_uniqueid", index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(indexable.UniqueId.Value, "s_uniqueid"), AzureFieldBuilder.BuildField("s_uniqueid", index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(indexable.UniqueId.Value, "s_uniqueid"), this.AzureIndex));
this.Culture = indexable.Culture;
}
