public static AzureField CreateField(string name, object value, AzureSearchFieldConfiguration fieldConfiguration, IIndexFieldStorageValueFormatter indexFieldStorageValueFormatter)
{
if (value == null)
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value null", (object)name)));
return(null);
}
if (fieldConfiguration == null)
{
throw new ArgumentNullException("fieldConfiguration");
}
if (VerboseLogging.Enabled)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Field: {0}" + Environment.NewLine, name);
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, value.GetType());
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, value);
//stringBuilder.AppendFormat(" - fieldConfiguration analyzer: {0}" + Environment.NewLine, fieldConfiguration.Analyzer != null ? (object)fieldConfiguration.Analyzer.GetType().ToString() : (object)"NULL");
stringBuilder.AppendFormat(" - fieldConfiguration boost: {0}" + Environment.NewLine, fieldConfiguration.Boost);
stringBuilder.AppendFormat(" - fieldConfiguration fieldID: {0}" + Environment.NewLine, fieldConfiguration.FieldID);
stringBuilder.AppendFormat(" - fieldConfiguration FieldName: {0}" + Environment.NewLine, fieldConfiguration.FieldName);
stringBuilder.AppendFormat(" - fieldConfiguration FieldTypeName: {0}" + Environment.NewLine, fieldConfiguration.FieldTypeName);
//stringBuilder.AppendFormat(" - fieldConfiguration IndexType: {0}" + Environment.NewLine, (object)fieldConfiguration.IndexType);
//stringBuilder.AppendFormat(" - fieldConfiguration StorageType: {0}" + Environment.NewLine, (object)fieldConfiguration.StorageType);
//stringBuilder.AppendFormat(" - fieldConfiguration VectorType: {0}" + Environment.NewLine, (object)fieldConfiguration.VectorType);
stringBuilder.AppendFormat(" - fieldConfiguration Type: {0}" + Environment.NewLine, fieldConfiguration.Type);
VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString));
}
return(new AzureField(name, value, BuildField(name, fieldConfiguration), fieldConfiguration.Type));
}
public override void AddField(string fieldName, object fieldValue, bool append = false)
{
var fieldConfiguration = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName);
var fieldName1 = fieldName;
fieldName = this.Index.FieldNameTranslator.GetIndexFieldName(fieldName);
var fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName) as AzureSearchFieldConfiguration;
if (fieldSettings != null)
{
if (fieldConfiguration != null)
{
fieldValue = fieldConfiguration.FormatForWriting(fieldValue);
}
this.AddField(fieldName, fieldValue, fieldSettings, 0.0f);
}
else
{
if (VerboseLogging.Enabled)
{
var stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Field: {0} (Adding field with no field configuration)" + Environment.NewLine, (object)fieldName);
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue != null ? (object)fieldValue.GetType().ToString() : (object)"NULL");
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue);
VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString));
}
var enumerable = fieldValue as IEnumerable;
if (enumerable != null && !(fieldValue is string))
{
foreach (object obj in enumerable)
{
//var valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(obj, fieldName1);
//if (fieldConfiguration != null)
// fieldName1 = fieldConfiguration.FormatForWriting(fieldName1);
if (fieldName1 != null)
{
if (QueueContainsName(fieldName))
{
var field = this.fields.FirstOrDefault(f => f.Name == fieldName);
}
else
{
this.fields.Enqueue(new AzureField(fieldName, fieldName1, AzureFieldBuilder.BuildField(fieldName, fieldName1, this.Index as IAzureProviderIndex)));
}
}
}
}
else
{
//var valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(fieldValue, fieldName1);
//if (fieldConfiguration != null)
// valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex);
if (fieldName1 != null)
{
this.fields.Enqueue(new AzureField(fieldName, fieldValue, AzureFieldBuilder.BuildField(fieldName, fieldValue, this.Index as IAzureProviderIndex)));
}
}
}
}
public override void AddField(string fieldName, object fieldValue, bool append = false)
{
AbstractSearchFieldConfiguration fieldConfiguration = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName);
string fieldName1 = fieldName;
fieldName = this.Index.FieldNameTranslator.GetIndexFieldName(fieldName);
AzureSearchFieldConfiguration fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName) as AzureSearchFieldConfiguration;
if (fieldSettings != null)
{
if (fieldConfiguration != null)
{
fieldValue = fieldConfiguration.FormatForWriting(fieldValue);
}
this.AddField(fieldName, fieldValue, fieldSettings, 0.0f);
}
else
{
if (VerboseLogging.Enabled)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Field: {0} (Adding field with no field configuration)" + Environment.NewLine, (object)fieldName);
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue != null ? (object)fieldValue.GetType().ToString() : (object)"NULL");
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue);
VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString));
}
IEnumerable enumerable = fieldValue as IEnumerable;
if (enumerable != null && !(fieldValue is string))
{
foreach (object obj in enumerable)
{
object valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(obj, fieldName1);
if (fieldConfiguration != null)
{
valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex);
}
if (valueToIndex != null)
{
this.fields.Enqueue((Field) new Field(fieldName, valueToIndex.ToString()));
}
}
}
else
{
object valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(fieldValue, fieldName1);
if (fieldConfiguration != null)
{
valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex);
}
if (valueToIndex == null)
{
return;
}
//TODO: How to figure out setting field value
this.fields.Enqueue((Field) new Field(fieldName, DataType.String /*, valueToIndex.ToString()*/));
}
}
}
private void CheckAndAddField(IIndexable indexable, IIndexableDataField field)
{
var name = field.Name;
if (IsTemplate && Options.HasExcludedTemplateFields && (Options.ExcludedTemplateFields.Contains(name) || Options.ExcludedTemplateFields.Contains(field.Id.ToString())))
{
VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Field was excluded.", field.Id, field.Name, field.TypeKey));
}
else if (IsMedia && Options.HasExcludedMediaFields && Options.ExcludedMediaFields.Contains(field.Name))
{
VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Media field was excluded.", field.Id, field.Name, field.TypeKey));
}
else
{
if (!Options.ExcludedFields.Contains(field.Id.ToString()))
{
if (!Options.ExcludedFields.Contains(name))
{
try
{
if (Options.IndexAllFields)
{
using (new LanguageFallbackFieldSwitcher(new bool?(Index.EnableFieldLanguageFallback)))
{
AddField(field);
return;
}
}
else if (Options.IncludedFields.Contains(name) || Options.IncludedFields.Contains(field.Id.ToString()))
{
using (new LanguageFallbackFieldSwitcher(new bool?(Index.EnableFieldLanguageFallback)))
{
AddField(field);
return;
}
}
else
{
VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Field was not included.", field.Id, field.Name, field.TypeKey));
return;
}
}
catch (Exception ex)
{
if (!Settings.StopOnCrawlFieldError())
{
CrawlingLog.Log.Fatal(string.Format("Could not add field {1} : {2} for indexable {0}", indexable.UniqueId, field.Id, field.Name), ex);
return;
}
throw;
}
}
}
VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Field was excluded.", field.Id, field.Name, field.TypeKey));
}
}
private static void LogIndexOperation(Func <string> logOperation, IndexData data, Document document)
{
VerboseLogging.CrawlingLogDebug(() =>
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendLine(logOperation());
stringBuilder.AppendLine(string.Format(" - UpdateTerm Field:{0}, Text:{1}", data.UpdateTerm.Name, data.UpdateTerm.Value));
if (VerboseLogging.Enabled)
{
foreach (var key in document.Keys)
{
stringBuilder.AppendLine(string.Format(" - {0}: {1}", key, document[key]));
}
}
return(stringBuilder.ToString());
});
}
public void Delete(IIndexable indexable, IProviderUpdateContext context)
{
Assert.ArgumentNotNull(indexable, "indexable");
VerboseLogging.CrawlingLogDebug(() => string.Format("Deleting indexable UniqueId:{0}, Index:{1}", indexable.UniqueId, context.Index.Name));
var providerUpdateContextEx = context as IProviderUpdateContextEx;
if (providerUpdateContextEx != null)
{
providerUpdateContextEx.Delete(indexable.UniqueId, new IExecutionContext[1]
{
indexable.Culture != null ? new CultureExecutionContext(indexable.Culture) : null
});
}
else
{
context.Delete(indexable.UniqueId);
}
}
public virtual void AddItemFields()
{
try
{
VerboseLogging.CrawlingLogDebug(() => "AddItemFields start");
if (Options.IndexAllFields)
{
Indexable.LoadAllFields();
}
if (IsParallel)
{
var exceptions = new ConcurrentQueue <Exception>();
Parallel.ForEach(Indexable.Fields, ParallelOptions, f =>
{
try
{
CheckAndAddField(Indexable, f);
}
catch (Exception ex)
{
exceptions.Enqueue(ex);
}
});
if (exceptions.Count > 0)
{
throw new AggregateException(exceptions);
}
}
else
{
foreach (var field in Indexable.Fields)
{
CheckAndAddField(Indexable, field);
}
}
}
finally
{
VerboseLogging.CrawlingLogDebug(() => "AddItemFields End");
}
}
public override void AddField(IIndexableDataField field)
{
AbstractSearchFieldConfiguration fieldConfiguration1 = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(field);
object fieldValue = this.Index.Configuration.FieldReaders.GetFieldValue(field);
string name = field.Name;
AzureSearchFieldConfiguration fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(field) as AzureSearchFieldConfiguration;
if (fieldSettings == null)
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Cannot resolve field settings for field id:{0}, name:{1}, typeKey:{2} - The field will not be added to the index.", field.Id, (object)field.Name, (object)field.TypeKey)));
}
else
{
object obj = fieldConfiguration1.FormatForWriting(fieldValue);
float boost = BoostingManager.ResolveFieldBoosting(field);
if (IndexOperationsHelper.IsTextField(field))
{
AzureSearchFieldConfiguration fieldConfiguration2 = this.Index.Configuration.FieldMap.GetFieldConfiguration("_content") as AzureSearchFieldConfiguration;
this.AddField("_content", obj, fieldConfiguration2 ?? this.defaultTextField, 0.0f);
}
this.AddField(name, obj, fieldSettings, boost);
}
}
public static Field CreateField(string name, object value, AzureSearchFieldConfiguration fieldConfiguration, IIndexFieldStorageValueFormatter indexFieldStorageValueFormatter)
{
if (value == null)
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value null", (object)name)));
return((Field)null);
}
if (fieldConfiguration == null)
{
throw new ArgumentNullException("fieldConfiguration");
}
if (VerboseLogging.Enabled)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Field: {0}" + Environment.NewLine, (object)name);
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, (object)value.GetType());
stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, value);
//stringBuilder.AppendFormat(" - fieldConfiguration analyzer: {0}" + Environment.NewLine, fieldConfiguration.Analyzer != null ? (object)fieldConfiguration.Analyzer.GetType().ToString() : (object)"NULL");
stringBuilder.AppendFormat(" - fieldConfiguration boost: {0}" + Environment.NewLine, (object)fieldConfiguration.Boost);
stringBuilder.AppendFormat(" - fieldConfiguration fieldID: {0}" + Environment.NewLine, (object)fieldConfiguration.FieldID);
stringBuilder.AppendFormat(" - fieldConfiguration FieldName: {0}" + Environment.NewLine, (object)fieldConfiguration.FieldName);
stringBuilder.AppendFormat(" - fieldConfiguration FieldTypeName: {0}" + Environment.NewLine, (object)fieldConfiguration.FieldTypeName);
//stringBuilder.AppendFormat(" - fieldConfiguration IndexType: {0}" + Environment.NewLine, (object)fieldConfiguration.IndexType);
//stringBuilder.AppendFormat(" - fieldConfiguration StorageType: {0}" + Environment.NewLine, (object)fieldConfiguration.StorageType);
//stringBuilder.AppendFormat(" - fieldConfiguration VectorType: {0}" + Environment.NewLine, (object)fieldConfiguration.VectorType);
stringBuilder.AppendFormat(" - fieldConfiguration Type: {0}" + Environment.NewLine, (object)fieldConfiguration.Type);
VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString));
}
if (AzureFieldBuilder.IsNumericField(fieldConfiguration.Type))
{
if (value is string && string.IsNullOrEmpty((string)value))
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value or empty null", (object)name)));
return((Field)null);
}
long result;
if (long.TryParse(value.ToString(), out result))
{
var numericField = new Field(name, DataType.Int64);
//TODO: How to set value?
//numericField.((long)Convert.ChangeType(value, typeof(long)));
return(numericField);
}
}
if (AzureFieldBuilder.IsFloatingPointField(fieldConfiguration.Type))
{
if (value is string && string.IsNullOrEmpty((string)value))
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value or empty null", (object)name)));
return((Field)null);
}
var numericField = new Field(name, DataType.Double);
//numericField.SetDoubleValue((double)Convert.ChangeType(value, typeof(double), (IFormatProvider)LanguageUtil.GetCultureInfo()));
return((Field)numericField);
}
string value_Renamed = indexFieldStorageValueFormatter.FormatValueForIndexStorage(value, name).ToString();
if (VerboseLogging.Enabled)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Field: {0}" + Environment.NewLine, (object)name);
stringBuilder.AppendFormat(" - formattedValue: {0}" + Environment.NewLine, (object)value_Renamed);
VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString));
}
//TODO: How to set field value?
return((Field) new Field(name, DataType.String /*, value_Renamed*/));
}
public virtual void AddSpecialFields()
{
try
{
VerboseLogging.CrawlingLogDebug(() => "AddSpecialFields Start");
AddSpecialField("s_key", Key, false);
AddSpecialField("s_uniqueid", Indexable.UniqueId.ToString(), false);
AddSpecialField("s_datasource", Indexable.DataSource.ToLowerInvariant(), false);
AddSpecialField("s_indexname", Index.Name.ToLowerInvariant(), false);
var hashedIndexable = Indexable as IHashedIndexable;
if (hashedIndexable != null)
{
AddSpecialField("s_hash", hashedIndexable.GetIndexableHashCode(), false);
}
var documentTypedIndexable = Indexable as IDocumentTypedIndexable;
if (documentTypedIndexable != null)
{
AddSpecialField("s_documenttype", documentTypedIndexable.DocumentType, false);
}
var indexableBuiltinFields = Indexable as IIndexableBuiltinFields;
if (indexableBuiltinFields == null)
{
return;
}
AddSpecialField("s_database", indexableBuiltinFields.Database, false);
AddSpecialField("s_language", indexableBuiltinFields.Language, false);
AddSpecialField("s_template", indexableBuiltinFields.TemplateId, false);
AddSpecialField("s_parent", indexableBuiltinFields.Parent, false);
if (indexableBuiltinFields.IsLatestVersion)
{
AddSpecialField("s_latestversion", true, false);
}
AddSpecialField("s_version", indexableBuiltinFields.Version, false);
AddSpecialField("s_group", indexableBuiltinFields.Group, false);
if (indexableBuiltinFields.IsClone)
{
AddSpecialField("s_isclone", true, false);
}
AddSpecialField("s_fullpath", indexableBuiltinFields.FullPath, false);
if (Options.ExcludeAllSpecialFields)
{
return;
}
AddSpecialField("s_name", indexableBuiltinFields.Name, false);
AddSpecialField("s_displayname", indexableBuiltinFields.DisplayName, false);
AddSpecialField("s_creator", indexableBuiltinFields.CreatedBy, false);
AddSpecialField("s_editor", indexableBuiltinFields.UpdatedBy, false);
AddSpecialField("s_templatename", indexableBuiltinFields.TemplateName, false);
AddSpecialField("s_created", indexableBuiltinFields.CreatedDate, false);
AddSpecialField("s_updated", indexableBuiltinFields.UpdatedDate, false);
AddSpecialField("s_path", indexableBuiltinFields.Paths, false);
AddSpecialField("s_content", indexableBuiltinFields.Name + " " + indexableBuiltinFields.DisplayName, false);
if (Options.Tags == null || Options.Tags.Length <= 0)
{
return;
}
AddSpecialField("s_tags", Options.Tags, false);
}
finally
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => "AddSpecialFields End"));
}
}
public void Delete(IIndexableUniqueId indexableUniqueId, IProviderUpdateContext context)
{
Assert.ArgumentNotNull(indexableUniqueId, "indexableUniqueId");
VerboseLogging.CrawlingLogDebug(() => string.Format("Deleting indexable UniqueId:{0}, Index:{1}", indexableUniqueId, context.Index.Name));
context.Delete(indexableUniqueId);
}
public override void AddComputedIndexFields()
{
try
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => "AddComputedIndexFields Start"));
if (this.IsParallel)
{
ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>();
Parallel.ForEach <IComputedIndexField>((IEnumerable <IComputedIndexField>) this.Options.ComputedIndexFields, this.ParallelOptions, (Action <IComputedIndexField, ParallelLoopState>)((computedIndexField, parallelLoopState) =>
{
object fieldValue;
try
{
fieldValue = computedIndexField.ComputeFieldValue(this.Indexable);
}
catch (Exception ex)
{
CrawlingLog.Log.Warn(string.Format("Could not compute value for ComputedIndexField: {0} for indexable: {1}", (object)computedIndexField.FieldName, (object)this.Indexable.UniqueId), ex);
if (!this.Settings.StopOnCrawlFieldError())
{
return;
}
exceptions.Enqueue(ex);
parallelLoopState.Stop();
return;
}
this.AddComputedIndexField(computedIndexField, fieldValue);
}));
if (exceptions.Count > 0)
{
throw new AggregateException((IEnumerable <Exception>)exceptions);
}
}
else
{
foreach (IComputedIndexField computedIndexField in this.Options.ComputedIndexFields)
{
object fieldValue;
try
{
fieldValue = computedIndexField.ComputeFieldValue(this.Indexable);
}
catch (Exception ex)
{
CrawlingLog.Log.Warn(string.Format("Could not compute value for ComputedIndexField: {0} for indexable: {1}", (object)computedIndexField.FieldName, (object)this.Indexable.UniqueId), ex);
if (this.Settings.StopOnCrawlFieldError())
{
throw;
}
else
{
continue;
}
}
this.AddComputedIndexField(computedIndexField, fieldValue);
}
}
}
finally
{
VerboseLogging.CrawlingLogDebug((Func <string>)(() => "AddComputedIndexFields End"));
}
}
public override void AddComputedIndexFields()
{
var cachedComputedValues = new Dictionary <string, object>();
object obj;
try
{
VerboseLogging.CrawlingLogDebug(() => "AddComputedIndexFields Start");
foreach (IComputedIndexField computedIndexField in base.Options.ComputedIndexFields)
{
try
{
var type = computedIndexField.GetType().FullName;
if (cachedComputedValues.ContainsKey(type))
{
obj = cachedComputedValues[type];
}
else
{
obj = computedIndexField.ComputeFieldValue(base.Indexable);
cachedComputedValues[type] = obj;
}
}
catch (Exception exception1)
{
Exception exception = exception1;
if (base.Settings.StopOnCrawlFieldError())
{
throw;
}
else
{
CrawlingLog.Log.Error(string.Format("Could not compute value for ComputedIndexField: {0} for indexable: {1}", computedIndexField.FieldName, base.Indexable.UniqueId), exception);
continue;
}
}
LuceneSearchFieldConfiguration fieldConfiguration = base.Index.Configuration.FieldMap.GetFieldConfiguration(computedIndexField.FieldName) as LuceneSearchFieldConfiguration;
if (obj is IEnumerable && !(obj is string))
{
foreach (object obj1 in obj as IEnumerable)
{
if (fieldConfiguration == null)
{
this.AddField(computedIndexField.FieldName, obj1, false);
}
else
{
this.AddField(computedIndexField.FieldName, obj1, fieldConfiguration, 0f);
}
}
}
else if (fieldConfiguration == null)
{
this.AddField(computedIndexField.FieldName, obj, false);
}
else
{
this.AddField(computedIndexField.FieldName, obj, fieldConfiguration, 0f);
}
}
}
finally
{
VerboseLogging.CrawlingLogDebug(() => "AddComputedIndexFields End");
}
}