public static AzureField CreateField(string name, object value, AzureSearchFieldConfiguration fieldConfiguration, IIndexFieldStorageValueFormatter indexFieldStorageValueFormatter) { if (value == null) { VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value null", (object)name))); return(null); } if (fieldConfiguration == null) { throw new ArgumentNullException("fieldConfiguration"); } if (VerboseLogging.Enabled) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendFormat("Field: {0}" + Environment.NewLine, name); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, value.GetType()); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, value); //stringBuilder.AppendFormat(" - fieldConfiguration analyzer: {0}" + Environment.NewLine, fieldConfiguration.Analyzer != null ? (object)fieldConfiguration.Analyzer.GetType().ToString() : (object)"NULL"); stringBuilder.AppendFormat(" - fieldConfiguration boost: {0}" + Environment.NewLine, fieldConfiguration.Boost); stringBuilder.AppendFormat(" - fieldConfiguration fieldID: {0}" + Environment.NewLine, fieldConfiguration.FieldID); stringBuilder.AppendFormat(" - fieldConfiguration FieldName: {0}" + Environment.NewLine, fieldConfiguration.FieldName); stringBuilder.AppendFormat(" - fieldConfiguration FieldTypeName: {0}" + Environment.NewLine, fieldConfiguration.FieldTypeName); //stringBuilder.AppendFormat(" - fieldConfiguration IndexType: {0}" + Environment.NewLine, (object)fieldConfiguration.IndexType); //stringBuilder.AppendFormat(" - fieldConfiguration StorageType: {0}" + Environment.NewLine, (object)fieldConfiguration.StorageType); //stringBuilder.AppendFormat(" - fieldConfiguration VectorType: {0}" + Environment.NewLine, (object)fieldConfiguration.VectorType); stringBuilder.AppendFormat(" - fieldConfiguration Type: {0}" + Environment.NewLine, fieldConfiguration.Type); VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString)); } return(new AzureField(name, value, BuildField(name, fieldConfiguration), fieldConfiguration.Type)); }
public override void AddField(string fieldName, object fieldValue, bool append = false) { var fieldConfiguration = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName); var fieldName1 = fieldName; fieldName = this.Index.FieldNameTranslator.GetIndexFieldName(fieldName); var fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName) as AzureSearchFieldConfiguration; if (fieldSettings != null) { if (fieldConfiguration != null) { fieldValue = fieldConfiguration.FormatForWriting(fieldValue); } this.AddField(fieldName, fieldValue, fieldSettings, 0.0f); } else { if (VerboseLogging.Enabled) { var stringBuilder = new StringBuilder(); stringBuilder.AppendFormat("Field: {0} (Adding field with no field configuration)" + Environment.NewLine, (object)fieldName); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue != null ? (object)fieldValue.GetType().ToString() : (object)"NULL"); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue); VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString)); } var enumerable = fieldValue as IEnumerable; if (enumerable != null && !(fieldValue is string)) { foreach (object obj in enumerable) { //var valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(obj, fieldName1); //if (fieldConfiguration != null) // fieldName1 = fieldConfiguration.FormatForWriting(fieldName1); if (fieldName1 != null) { if (QueueContainsName(fieldName)) { var field = this.fields.FirstOrDefault(f => f.Name == fieldName); } else { this.fields.Enqueue(new AzureField(fieldName, fieldName1, AzureFieldBuilder.BuildField(fieldName, fieldName1, this.Index as IAzureProviderIndex))); } } } } else { //var valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(fieldValue, fieldName1); //if (fieldConfiguration != null) // valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex); if (fieldName1 != null) { this.fields.Enqueue(new AzureField(fieldName, fieldValue, AzureFieldBuilder.BuildField(fieldName, fieldValue, this.Index as IAzureProviderIndex))); } } } }
public override void AddField(string fieldName, object fieldValue, bool append = false) { AbstractSearchFieldConfiguration fieldConfiguration = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName); string fieldName1 = fieldName; fieldName = this.Index.FieldNameTranslator.GetIndexFieldName(fieldName); AzureSearchFieldConfiguration fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(fieldName) as AzureSearchFieldConfiguration; if (fieldSettings != null) { if (fieldConfiguration != null) { fieldValue = fieldConfiguration.FormatForWriting(fieldValue); } this.AddField(fieldName, fieldValue, fieldSettings, 0.0f); } else { if (VerboseLogging.Enabled) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendFormat("Field: {0} (Adding field with no field configuration)" + Environment.NewLine, (object)fieldName); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue != null ? (object)fieldValue.GetType().ToString() : (object)"NULL"); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, fieldValue); VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString)); } IEnumerable enumerable = fieldValue as IEnumerable; if (enumerable != null && !(fieldValue is string)) { foreach (object obj in enumerable) { object valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(obj, fieldName1); if (fieldConfiguration != null) { valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex); } if (valueToIndex != null) { this.fields.Enqueue((Field) new Field(fieldName, valueToIndex.ToString())); } } } else { object valueToIndex = this.Index.Configuration.IndexFieldStorageValueFormatter.FormatValueForIndexStorage(fieldValue, fieldName1); if (fieldConfiguration != null) { valueToIndex = fieldConfiguration.FormatForWriting(valueToIndex); } if (valueToIndex == null) { return; } //TODO: How to figure out setting field value this.fields.Enqueue((Field) new Field(fieldName, DataType.String /*, valueToIndex.ToString()*/)); } } }
private void CheckAndAddField(IIndexable indexable, IIndexableDataField field) { var name = field.Name; if (IsTemplate && Options.HasExcludedTemplateFields && (Options.ExcludedTemplateFields.Contains(name) || Options.ExcludedTemplateFields.Contains(field.Id.ToString()))) { VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Field was excluded.", field.Id, field.Name, field.TypeKey)); } else if (IsMedia && Options.HasExcludedMediaFields && Options.ExcludedMediaFields.Contains(field.Name)) { VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Media field was excluded.", field.Id, field.Name, field.TypeKey)); } else { if (!Options.ExcludedFields.Contains(field.Id.ToString())) { if (!Options.ExcludedFields.Contains(name)) { try { if (Options.IndexAllFields) { using (new LanguageFallbackFieldSwitcher(new bool?(Index.EnableFieldLanguageFallback))) { AddField(field); return; } } else if (Options.IncludedFields.Contains(name) || Options.IncludedFields.Contains(field.Id.ToString())) { using (new LanguageFallbackFieldSwitcher(new bool?(Index.EnableFieldLanguageFallback))) { AddField(field); return; } } else { VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Field was not included.", field.Id, field.Name, field.TypeKey)); return; } } catch (Exception ex) { if (!Settings.StopOnCrawlFieldError()) { CrawlingLog.Log.Fatal(string.Format("Could not add field {1} : {2} for indexable {0}", indexable.UniqueId, field.Id, field.Name), ex); return; } throw; } } } VerboseLogging.CrawlingLogDebug(() => string.Format("Skipping field id:{0}, name:{1}, typeKey:{2} - Field was excluded.", field.Id, field.Name, field.TypeKey)); } }
private static void LogIndexOperation(Func <string> logOperation, IndexData data, Document document) { VerboseLogging.CrawlingLogDebug(() => { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendLine(logOperation()); stringBuilder.AppendLine(string.Format(" - UpdateTerm Field:{0}, Text:{1}", data.UpdateTerm.Name, data.UpdateTerm.Value)); if (VerboseLogging.Enabled) { foreach (var key in document.Keys) { stringBuilder.AppendLine(string.Format(" - {0}: {1}", key, document[key])); } } return(stringBuilder.ToString()); }); }
public void Delete(IIndexable indexable, IProviderUpdateContext context) { Assert.ArgumentNotNull(indexable, "indexable"); VerboseLogging.CrawlingLogDebug(() => string.Format("Deleting indexable UniqueId:{0}, Index:{1}", indexable.UniqueId, context.Index.Name)); var providerUpdateContextEx = context as IProviderUpdateContextEx; if (providerUpdateContextEx != null) { providerUpdateContextEx.Delete(indexable.UniqueId, new IExecutionContext[1] { indexable.Culture != null ? new CultureExecutionContext(indexable.Culture) : null }); } else { context.Delete(indexable.UniqueId); } }
public virtual void AddItemFields() { try { VerboseLogging.CrawlingLogDebug(() => "AddItemFields start"); if (Options.IndexAllFields) { Indexable.LoadAllFields(); } if (IsParallel) { var exceptions = new ConcurrentQueue <Exception>(); Parallel.ForEach(Indexable.Fields, ParallelOptions, f => { try { CheckAndAddField(Indexable, f); } catch (Exception ex) { exceptions.Enqueue(ex); } }); if (exceptions.Count > 0) { throw new AggregateException(exceptions); } } else { foreach (var field in Indexable.Fields) { CheckAndAddField(Indexable, field); } } } finally { VerboseLogging.CrawlingLogDebug(() => "AddItemFields End"); } }
public override void AddField(IIndexableDataField field) { AbstractSearchFieldConfiguration fieldConfiguration1 = this.Context.Index.Configuration.FieldMap.GetFieldConfiguration(field); object fieldValue = this.Index.Configuration.FieldReaders.GetFieldValue(field); string name = field.Name; AzureSearchFieldConfiguration fieldSettings = this.Index.Configuration.FieldMap.GetFieldConfiguration(field) as AzureSearchFieldConfiguration; if (fieldSettings == null) { VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Cannot resolve field settings for field id:{0}, name:{1}, typeKey:{2} - The field will not be added to the index.", field.Id, (object)field.Name, (object)field.TypeKey))); } else { object obj = fieldConfiguration1.FormatForWriting(fieldValue); float boost = BoostingManager.ResolveFieldBoosting(field); if (IndexOperationsHelper.IsTextField(field)) { AzureSearchFieldConfiguration fieldConfiguration2 = this.Index.Configuration.FieldMap.GetFieldConfiguration("_content") as AzureSearchFieldConfiguration; this.AddField("_content", obj, fieldConfiguration2 ?? this.defaultTextField, 0.0f); } this.AddField(name, obj, fieldSettings, boost); } }
public static Field CreateField(string name, object value, AzureSearchFieldConfiguration fieldConfiguration, IIndexFieldStorageValueFormatter indexFieldStorageValueFormatter) { if (value == null) { VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value null", (object)name))); return((Field)null); } if (fieldConfiguration == null) { throw new ArgumentNullException("fieldConfiguration"); } if (VerboseLogging.Enabled) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendFormat("Field: {0}" + Environment.NewLine, (object)name); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, (object)value.GetType()); stringBuilder.AppendFormat(" - value: {0}" + Environment.NewLine, value); //stringBuilder.AppendFormat(" - fieldConfiguration analyzer: {0}" + Environment.NewLine, fieldConfiguration.Analyzer != null ? (object)fieldConfiguration.Analyzer.GetType().ToString() : (object)"NULL"); stringBuilder.AppendFormat(" - fieldConfiguration boost: {0}" + Environment.NewLine, (object)fieldConfiguration.Boost); stringBuilder.AppendFormat(" - fieldConfiguration fieldID: {0}" + Environment.NewLine, (object)fieldConfiguration.FieldID); stringBuilder.AppendFormat(" - fieldConfiguration FieldName: {0}" + Environment.NewLine, (object)fieldConfiguration.FieldName); stringBuilder.AppendFormat(" - fieldConfiguration FieldTypeName: {0}" + Environment.NewLine, (object)fieldConfiguration.FieldTypeName); //stringBuilder.AppendFormat(" - fieldConfiguration IndexType: {0}" + Environment.NewLine, (object)fieldConfiguration.IndexType); //stringBuilder.AppendFormat(" - fieldConfiguration StorageType: {0}" + Environment.NewLine, (object)fieldConfiguration.StorageType); //stringBuilder.AppendFormat(" - fieldConfiguration VectorType: {0}" + Environment.NewLine, (object)fieldConfiguration.VectorType); stringBuilder.AppendFormat(" - fieldConfiguration Type: {0}" + Environment.NewLine, (object)fieldConfiguration.Type); VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString)); } if (AzureFieldBuilder.IsNumericField(fieldConfiguration.Type)) { if (value is string && string.IsNullOrEmpty((string)value)) { VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value or empty null", (object)name))); return((Field)null); } long result; if (long.TryParse(value.ToString(), out result)) { var numericField = new Field(name, DataType.Int64); //TODO: How to set value? //numericField.((long)Convert.ChangeType(value, typeof(long))); return(numericField); } } if (AzureFieldBuilder.IsFloatingPointField(fieldConfiguration.Type)) { if (value is string && string.IsNullOrEmpty((string)value)) { VerboseLogging.CrawlingLogDebug((Func <string>)(() => string.Format("Skipping field {0} - value or empty null", (object)name))); return((Field)null); } var numericField = new Field(name, DataType.Double); //numericField.SetDoubleValue((double)Convert.ChangeType(value, typeof(double), (IFormatProvider)LanguageUtil.GetCultureInfo())); return((Field)numericField); } string value_Renamed = indexFieldStorageValueFormatter.FormatValueForIndexStorage(value, name).ToString(); if (VerboseLogging.Enabled) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendFormat("Field: {0}" + Environment.NewLine, (object)name); stringBuilder.AppendFormat(" - formattedValue: {0}" + Environment.NewLine, (object)value_Renamed); VerboseLogging.CrawlingLogDebug(new Func <string>(((object)stringBuilder).ToString)); } //TODO: How to set field value? return((Field) new Field(name, DataType.String /*, value_Renamed*/)); }
public virtual void AddSpecialFields() { try { VerboseLogging.CrawlingLogDebug(() => "AddSpecialFields Start"); AddSpecialField("s_key", Key, false); AddSpecialField("s_uniqueid", Indexable.UniqueId.ToString(), false); AddSpecialField("s_datasource", Indexable.DataSource.ToLowerInvariant(), false); AddSpecialField("s_indexname", Index.Name.ToLowerInvariant(), false); var hashedIndexable = Indexable as IHashedIndexable; if (hashedIndexable != null) { AddSpecialField("s_hash", hashedIndexable.GetIndexableHashCode(), false); } var documentTypedIndexable = Indexable as IDocumentTypedIndexable; if (documentTypedIndexable != null) { AddSpecialField("s_documenttype", documentTypedIndexable.DocumentType, false); } var indexableBuiltinFields = Indexable as IIndexableBuiltinFields; if (indexableBuiltinFields == null) { return; } AddSpecialField("s_database", indexableBuiltinFields.Database, false); AddSpecialField("s_language", indexableBuiltinFields.Language, false); AddSpecialField("s_template", indexableBuiltinFields.TemplateId, false); AddSpecialField("s_parent", indexableBuiltinFields.Parent, false); if (indexableBuiltinFields.IsLatestVersion) { AddSpecialField("s_latestversion", true, false); } AddSpecialField("s_version", indexableBuiltinFields.Version, false); AddSpecialField("s_group", indexableBuiltinFields.Group, false); if (indexableBuiltinFields.IsClone) { AddSpecialField("s_isclone", true, false); } AddSpecialField("s_fullpath", indexableBuiltinFields.FullPath, false); if (Options.ExcludeAllSpecialFields) { return; } AddSpecialField("s_name", indexableBuiltinFields.Name, false); AddSpecialField("s_displayname", indexableBuiltinFields.DisplayName, false); AddSpecialField("s_creator", indexableBuiltinFields.CreatedBy, false); AddSpecialField("s_editor", indexableBuiltinFields.UpdatedBy, false); AddSpecialField("s_templatename", indexableBuiltinFields.TemplateName, false); AddSpecialField("s_created", indexableBuiltinFields.CreatedDate, false); AddSpecialField("s_updated", indexableBuiltinFields.UpdatedDate, false); AddSpecialField("s_path", indexableBuiltinFields.Paths, false); AddSpecialField("s_content", indexableBuiltinFields.Name + " " + indexableBuiltinFields.DisplayName, false); if (Options.Tags == null || Options.Tags.Length <= 0) { return; } AddSpecialField("s_tags", Options.Tags, false); } finally { VerboseLogging.CrawlingLogDebug((Func <string>)(() => "AddSpecialFields End")); } }
public void Delete(IIndexableUniqueId indexableUniqueId, IProviderUpdateContext context) { Assert.ArgumentNotNull(indexableUniqueId, "indexableUniqueId"); VerboseLogging.CrawlingLogDebug(() => string.Format("Deleting indexable UniqueId:{0}, Index:{1}", indexableUniqueId, context.Index.Name)); context.Delete(indexableUniqueId); }
public override void AddComputedIndexFields() { try { VerboseLogging.CrawlingLogDebug((Func <string>)(() => "AddComputedIndexFields Start")); if (this.IsParallel) { ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>(); Parallel.ForEach <IComputedIndexField>((IEnumerable <IComputedIndexField>) this.Options.ComputedIndexFields, this.ParallelOptions, (Action <IComputedIndexField, ParallelLoopState>)((computedIndexField, parallelLoopState) => { object fieldValue; try { fieldValue = computedIndexField.ComputeFieldValue(this.Indexable); } catch (Exception ex) { CrawlingLog.Log.Warn(string.Format("Could not compute value for ComputedIndexField: {0} for indexable: {1}", (object)computedIndexField.FieldName, (object)this.Indexable.UniqueId), ex); if (!this.Settings.StopOnCrawlFieldError()) { return; } exceptions.Enqueue(ex); parallelLoopState.Stop(); return; } this.AddComputedIndexField(computedIndexField, fieldValue); })); if (exceptions.Count > 0) { throw new AggregateException((IEnumerable <Exception>)exceptions); } } else { foreach (IComputedIndexField computedIndexField in this.Options.ComputedIndexFields) { object fieldValue; try { fieldValue = computedIndexField.ComputeFieldValue(this.Indexable); } catch (Exception ex) { CrawlingLog.Log.Warn(string.Format("Could not compute value for ComputedIndexField: {0} for indexable: {1}", (object)computedIndexField.FieldName, (object)this.Indexable.UniqueId), ex); if (this.Settings.StopOnCrawlFieldError()) { throw; } else { continue; } } this.AddComputedIndexField(computedIndexField, fieldValue); } } } finally { VerboseLogging.CrawlingLogDebug((Func <string>)(() => "AddComputedIndexFields End")); } }
public override void AddComputedIndexFields() { var cachedComputedValues = new Dictionary <string, object>(); object obj; try { VerboseLogging.CrawlingLogDebug(() => "AddComputedIndexFields Start"); foreach (IComputedIndexField computedIndexField in base.Options.ComputedIndexFields) { try { var type = computedIndexField.GetType().FullName; if (cachedComputedValues.ContainsKey(type)) { obj = cachedComputedValues[type]; } else { obj = computedIndexField.ComputeFieldValue(base.Indexable); cachedComputedValues[type] = obj; } } catch (Exception exception1) { Exception exception = exception1; if (base.Settings.StopOnCrawlFieldError()) { throw; } else { CrawlingLog.Log.Error(string.Format("Could not compute value for ComputedIndexField: {0} for indexable: {1}", computedIndexField.FieldName, base.Indexable.UniqueId), exception); continue; } } LuceneSearchFieldConfiguration fieldConfiguration = base.Index.Configuration.FieldMap.GetFieldConfiguration(computedIndexField.FieldName) as LuceneSearchFieldConfiguration; if (obj is IEnumerable && !(obj is string)) { foreach (object obj1 in obj as IEnumerable) { if (fieldConfiguration == null) { this.AddField(computedIndexField.FieldName, obj1, false); } else { this.AddField(computedIndexField.FieldName, obj1, fieldConfiguration, 0f); } } } else if (fieldConfiguration == null) { this.AddField(computedIndexField.FieldName, obj, false); } else { this.AddField(computedIndexField.FieldName, obj, fieldConfiguration, 0f); } } } finally { VerboseLogging.CrawlingLogDebug(() => "AddComputedIndexFields End"); } }