public void PostDocuments(ICloudBatch batch)
{
Parallel.ForEach(_innerServices, searchService =>
{
IProvideAvailabilityManager availabilityManager = searchService as IProvideAvailabilityManager;
int num;
if (availabilityManager == null)
{
num = 0;
}
else
{
bool?canWrite = availabilityManager.AvailabilityManager?.CanWrite;
bool flag = false;
num = canWrite.GetValueOrDefault() == flag ? (canWrite.HasValue ? 1 : 0) : 0;
}
if (num != 0)
{
SearchLog.Log.Warn($"Service ${searchService.Name} is not available. Data from the commit can be lost on this search service.", null);
}
else
{
searchService.PostDocuments(batch);
}
});
}
public void PostDocuments(ICloudBatch batch)
{
if (batch is TransparentCloudBatch transparentBatch)
{
PostedBatches.Add(transparentBatch.Documents.ToList());
}
}
public void PostDocuments(ICloudBatch batch)
{
try
{
PostDocumentsImpl(batch);
}
catch (NotFoundException ex)
{
SchemaSynchronizer.RefreshLocalSchema();
PostDocumentsImpl(batch);
}
}
public void PostDocuments(ICloudBatch batch)
{
if (batch is TransparentCloudBatch mutableBatch)
{
foreach (var document in mutableBatch.Documents)
{
var documentId = GetDocumentId(document);
var keys = document.Keys.ToList();
foreach (var key in keys)
{
var fieldName = key;
var fieldSchema = Schema.GetFieldByCloudName(fieldName);
if (!IsFieldSizeLimited(fieldSchema))
{
continue;
}
var estimatedSize = FieldDataUtil.EstimateSize(document[key]);
if (estimatedSize >= MaxFieldSize)
{
document[key] = FieldDataUtil.Truncate(document[key], estimatedSize - MaxFieldSize);
CrawlingLog.Log.Info($"Field '{key}' on document '{documentId}' is too large and was truncated. Disable the Filterable, Sortable, and Facetable flags on this field to prevent truncation.");
}
}
}
var subBatches = SplitBatch(mutableBatch).ToList();
if (subBatches.Count > 1)
{
CrawlingLog.Log.Info($"Batch was too large and was split into {subBatches.Count} smaller batches");
}
foreach (var subBatch in subBatches)
{
innerService.PostDocuments(subBatch);
}
return;
}
// TODO: Configuration warning
innerService.PostDocuments(batch);
}
private void PostDocumentsImpl(ICloudBatch batch)
{
ICloudSearchIndexSchema schema = _searchIndex.SchemaBuilder?.GetSchema();
if (schema != null)
{
SchemaSynchronizer.EnsureIsInSync(schema.AllFields);
Schema = new CloudSearchIndexSchema(SchemaSynchronizer.LocalSchemaSnapshot);
OnSchemaSynced(EventArgs.Empty);
}
if (!AvailabilityManager.CanWrite)
{
throw new SearchServiceIsUnavailableException(_searchIndex.CloudIndexName, $"The service ${Name} is not available for write operations", null);
}
string json = batch.GetJson();
try
{
DocumentOperations.PostDocuments(json);
}
catch (PostFailedForSomeDocumentsException ex)
{
StringBuilder stringBuilder = new StringBuilder("Post failed for some documents");
foreach (MultiStatusResponseDocument document1 in ex.Documents)
{
MultiStatusResponseDocument document = document1;
Dictionary <string, object> dictionary = batch.Documents.FirstOrDefault(d => string.Equals((string)d[CloudSearchConfig.VirtualFields.CloudUniqueId], document.Key, StringComparison.OrdinalIgnoreCase));
CloudSearchFieldConfiguration fieldConfiguration = _searchIndex.Configuration.FieldMap.GetFieldConfiguration(IDFieldName) as CloudSearchFieldConfiguration;
string str = fieldConfiguration == null ? document.Key : (dictionary == null || !dictionary.ContainsKey(fieldConfiguration.CloudFieldName) ? document.Key : dictionary[fieldConfiguration.CloudFieldName].ToString());
stringBuilder.AppendLine($"Document id: {str}, message: {document.Message}");
}
CrawlingLog.Log.Warn(stringBuilder.ToString(), null);
CrawlingLog.Log.Debug(ex.RawResponse, null);
}
}
