public ElasticBulkIndexResponse BulkIndex <T>(IEnumerable <T> entities) where T : class
{
int retryCount = 0;
Exception lastException = null;
while (retryCount < MaxRetry)
{
try
{
IBulkResponse indexResponse = _client.IndexMany(entities);
if (indexResponse != null && indexResponse.IsValid)
{
throw new ElasticSearchServerException(indexResponse.ServerError.Error);
}
ElasticBulkIndexResponse response = new ElasticBulkIndexResponse();
return(response);
}
catch (WebException wex)
{
lastException = wex;
}
catch (Exception ex)
{
lastException = ex;
}
retryCount++;
Thread.Sleep(500);
}
throw new ElasticSearchException("There was an error occured while indexing", lastException);
}
private void ScrollToCompletion(IObserver <IReindexResponse <T> > observer, string fromIndex, string toIndex, Time scroll, ISearchResponse <T> searchResult)
{
if (searchResult == null || !searchResult.IsValid)
{
throw Throw($"Reindexing to {toIndex} failed unexpectedly during searching index {fromIndex}.", searchResult?.ApiCall);
}
IBulkResponse indexResult = null;
int page = 0;
while (searchResult.IsValid && searchResult.Documents.HasAny())
{
indexResult = this.IndexSearchResults(searchResult, observer, toIndex, page);
if (indexResult == null || !indexResult.IsValid)
{
throw Throw($"Reindexing to {toIndex} failed unexpectedly during bulk indexing.", indexResult?.ApiCall);
}
observer.OnNext(new ReindexResponse <T>()
{
BulkResponse = indexResult,
SearchResponse = searchResult,
Scroll = page
});
page++;
searchResult = this._client.Scroll <T>(scroll, searchResult.ScrollId);
if (searchResult == null || !searchResult.IsValid)
{
throw Throw($"Reindexing to {toIndex} failed unexpectedly during searching index {fromIndex}.", searchResult?.ApiCall);
}
}
}
/// <summary>
/// 添加房源到ES（新增或修改）
/// </summary>
/// <param name="listData">要添加的对象</param>
/// <returns></returns>
public void SaveHousesToES(List <BaseHouseInfo> houses)
{
var connSettings = new ConnectionSettings(new Uri(configuration.ESURL))
.BasicAuthentication(configuration.ESUserName, configuration.ESPassword);
var elasticClient = new ElasticClient(connSettings);
if (houses == null)
{
return;
}
foreach (var groupitem in houses.GroupBy(h => h.PubDate))
{
var houseIndex = $"house-{groupitem.Key.ToString("yyyy-MM-dd")}";
var index = elasticClient.IndexExists(houseIndex);
if (!index.Exists && index.IsValid)//判断索引是否存在和有效
{
//创建索引
elasticClient.CreateIndex(houseIndex, i => i
.Settings(s => s.NumberOfShards(2).NumberOfReplicas(0))// 2是常量，阿里云只买了两个片
.Mappings(m => m.Map <BaseHouseInfo>(mm => mm.AutoMap()))
.Mappings(map => map.Map <BaseHouseInfo>(mm => mm)));
}
//批量创建索引和文档
IBulkResponse bulkRs = elasticClient.IndexMany(groupitem, houseIndex);
if (bulkRs.Errors)//如果异常
{
LogHelper.Info("SaveHouses finish,index:" + houseIndex + ",DebugInformation:" + bulkRs.DebugInformation);
//TODO
}
}
;
}
public void SaveHouses(List <DBHouse> houses)
{
LogHelper.RunActionTaskNotThrowEx(() =>
{
var connSettings = new ConnectionSettings(new Uri(configuration.ESURL));
var elasticClient = new ElasticClient(connSettings);
if (houses == null || !houses.Any())
{
return;
}
var houseIndex = $"house-data-{DateTime.Now.ToString("yyyy-MM-dd")}";
var index = elasticClient.IndexExists(houseIndex);
if (!index.Exists && index.IsValid)//判断索引是否存在和有效
{
CreateIndex(houseIndex);
CreateMapping(houseIndex);
}
//批量创建索引和文档
IBulkResponse bulkRs = elasticClient.IndexMany(houses, houseIndex);
if (bulkRs.Errors)//如果异常
{
LogHelper.Info("SaveHouses error,index:" + houseIndex + ",DebugInformation:" + bulkRs.DebugInformation);
}
}, "SaveHouses");
}
protected async Task PublishElasticSearch(IEnumerable <object> evts)
{
try
{
List <object> objectList = evts.ToList();
if (!objectList.Any())
{
return;
}
string indexName = $"{_elasticSearchConfig.IndexNameBase}-{DateTime.UtcNow.ToString("MMddyyyy")}";
//string type = objectList.First().GetType().Name;
string type = _elasticSearchConfig.TypeName;
IBulkResponse rsp = await _elasticClient.IndexManyAsync(objectList, indexName, type);
if (rsp.Errors)
{
_logger?.LogError("Bulk Errors {0}", rsp.Errors.ToString());
}
}
catch (Exception e)
{
_logger?.LogError(0, e, "ElasticSearch PublishEvents error");
}
_logger?.LogDebug("Published {ElasticSearchEventCount} events to ElasticSearch", evts.Count());
}
public void SaveHouses(List <DBHouse> houses)
{
LogHelper.RunActionTaskNotThrowEx(() =>
{
if (houses == null || !houses.Any())
{
return;
}
// Elasticsearch 跨index 无法插入自动去重更新,所以此处做按照发布时间分组
foreach (var group in houses.GroupBy(h => h.PubTime.ToString("yyyy-MM-dd")))
{
var houseIndex = $"house-data-{group.Key}";
var index = _elasticClient.IndexExists(houseIndex);
if (!index.Exists && index.IsValid)//判断索引是否存在和有效
{
CreateIndex(houseIndex);
CreateMapping(houseIndex);
}
//批量创建索引和文档
IBulkResponse bulkRs = _elasticClient.IndexMany(group.ToList(), houseIndex);
if (bulkRs.Errors)//如果异常
{
LogHelper.Info("SaveHouses error,index:" + houseIndex + ",DebugInformation:" + bulkRs.DebugInformation);
}
}
}, "SaveHouses");
}
public async Task <bool> IndexAsync(List <Employee> employees, string langCode)
{
string indexName = $"employees_{langCode}";
IBulkResponse response = await _elasticClient.IndexManyAsync(employees, indexName);
return(response.IsValid);
}
public void InsertToElastic(List <MusicDocument> collection)
{
IBulkResponse bulkIdx = _elasticConn.Client
.Bulk(b => b
.Index("music")
.IndexMany(collection)
);
}
public async Task <bool> IndexAsync(List <Product> products, string langCode)
{
string indexName = $"products_{langCode}";
IBulkResponse response = await _elasticClient.IndexManyAsync(products, indexName);
return(response.IsValid);
}
private void Reindex(IObserver <IReindexResponse <T> > observer)
{
var fromIndex = this._reindexDescriptor._FromIndexName;
var toIndex = this._reindexDescriptor._ToIndexName;
var scroll = this._reindexDescriptor._Scroll ?? "2m";
fromIndex.ThrowIfNullOrEmpty("fromIndex");
toIndex.ThrowIfNullOrEmpty("toIndex");
var indexSettings = this.CurrentClient.GetIndexSettings(this._reindexDescriptor._FromIndexName);
var createSettings = new CreateIndexDescriptor(this._connectionSettings)
.InitializeUsing(indexSettings.Settings);
var createIndexResponse = this.CurrentClient
.CreateIndex(toIndex, (c) =>
{
if (this._reindexDescriptor._CreateIndexSelector != null)
{
return(this._reindexDescriptor._CreateIndexSelector(createSettings));
}
return(c);
});
if (!createIndexResponse.IsValid)
{
throw new ReindexException(createIndexResponse.ConnectionStatus);
}
var page = 0;
var searchResult = this.CurrentClient.Search <T>(
s => s
.Index(fromIndex)
.AllTypes()
.From(0)
.Take(100)
.Query(this._reindexDescriptor._QuerySelector ?? (q => q.MatchAll()))
.SearchType(SearchType.Scan)
.Scroll(scroll)
);
if (searchResult.Total <= 0)
{
throw new ReindexException(searchResult.ConnectionStatus, "index " + fromIndex + " has no documents!");
}
IBulkResponse indexResult = null;
do
{
searchResult = this.CurrentClient.Scroll <T>(scroll, searchResult.ScrollId);
if (searchResult.Documents.HasAny())
{
indexResult = this.IndexSearchResults(searchResult, observer, toIndex, page);
}
page++;
} while (searchResult.IsValid && indexResult != null && indexResult.IsValid && searchResult.Documents.HasAny());
observer.OnCompleted();
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (this.connectionData == null || events == null || events.Count == 0)
{
return;
}
try
{
string currentIndexName = this.GetIndexName(this.connectionData);
if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal))
{
await this.EnsureIndexExists(currentIndexName, this.connectionData.Client).ConfigureAwait(false);
this.connectionData.LastIndexName = currentIndexName;
}
BulkRequest request = new BulkRequest();
List <IBulkOperation> operations = new List <IBulkOperation>();
string documentTypeName = this.connectionData.Configuration.EventDocumentTypeName;
foreach (EventData eventData in events)
{
operations.AddRange(GetCreateOperationsForEvent(eventData, currentIndexName, documentTypeName));
}
request.Operations = operations;
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance
// between different SendEventsAsync callbacks.
// Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3
IBulkResponse response = await this.connectionData.Client.BulkAsync(request).ConfigureAwait(false);
if (!response.IsValid)
{
this.ReportEsRequestError(response, "Bulk upload");
}
else
{
this.healthReporter.ReportHealthy();
}
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(ElasticSearchOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
private void Reindex(IObserver <IReindexResponse <T> > observer)
{
var fromIndex = this._reindexRequest.From;
var toIndex = this._reindexRequest.To;
var scroll = this._reindexRequest.Scroll ?? TimeSpan.FromMinutes(2);
var size = this._reindexRequest.Size ?? 100;
var resolvedFrom = fromIndex.Resolve(this._connectionSettings);
resolvedFrom.ThrowIfNullOrEmpty(nameof(fromIndex));
var resolvedTo = toIndex.Resolve(this._connectionSettings);
resolvedTo.ThrowIfNullOrEmpty(nameof(toIndex));
var indexSettings = this._client.GetIndexSettings(i => i.Index(fromIndex));
Func <CreateIndexDescriptor, ICreateIndexRequest> settings = (ci) => this._reindexRequest.CreateIndexRequest ?? ci;
var createIndexResponse = this._client.CreateIndex(
toIndex, (c) => settings(c.InitializeUsing(indexSettings.Indices[resolvedFrom])));
if (!createIndexResponse.IsValid)
{
throw new ElasticsearchClientException(PipelineFailure.BadResponse, $"Failed to create destination index {toIndex}.", createIndexResponse.ApiCall);
}
var page = 0;
var searchResult = this._client.Search <T>(
s => s
.Index(fromIndex)
.Type(Types.All)
.From(0)
.Size(size)
.Query(q => this._reindexRequest.Query)
.SearchType(SearchType.Scan)
.Scroll(scroll.ToTimeSpan())
);
if (searchResult.Total <= 0)
{
throw new ElasticsearchClientException(PipelineFailure.BadResponse, $"Source index {fromIndex} doesn't contain any documents.", searchResult.ApiCall);
}
IBulkResponse indexResult = null;
do
{
var result = searchResult;
searchResult = this._client.Scroll <T>(scroll, result.ScrollId);
if (searchResult.Documents.HasAny())
{
indexResult = this.IndexSearchResults(searchResult, observer, toIndex, page);
}
page++;
} while (searchResult.IsValid && indexResult != null && indexResult.IsValid && searchResult.Documents.HasAny());
observer.OnCompleted();
}
private void Reindex(IObserver <IReindexResponse <T> > observer)
{
var fromIndex = this._reindexRequest.From;
var toIndex = this._reindexRequest.To;
var scroll = this._reindexRequest.Scroll ?? "2m";
var size = this._reindexRequest.Size ?? 100;
var resolvedFrom = fromIndex.Resolve(this._connectionSettings);
resolvedFrom.ThrowIfNullOrEmpty(nameof(fromIndex));
var resolvedTo = toIndex.Resolve(this._connectionSettings);
resolvedTo.ThrowIfNullOrEmpty(nameof(toIndex));
var indexSettings = this._client.GetIndexSettings(i => i.Index(fromIndex));
Func <CreateIndexDescriptor, ICreateIndexRequest> settings = (ci) => this._reindexRequest.CreateIndexRequest ?? ci;
var createIndexResponse = this._client.CreateIndex(
toIndex, (c) => settings(c.InitializeUsing(indexSettings.Indices[resolvedFrom])));
if (!createIndexResponse.IsValid)
{
throw new ReindexException(createIndexResponse.ApiCall);
}
var page = 0;
var searchResult = this._client.Search <T>(
s => s
.Index(fromIndex)
.Type(Types.All)
.From(0)
.Size(size)
.Query(this._reindexRequest.Query)
.SearchType(SearchType.Scan)
.Scroll(scroll)
);
if (searchResult.Total <= 0)
{
throw new ReindexException(searchResult.ApiCall, "index " + fromIndex + " has no documents!");
}
IBulkResponse indexResult = null;
do
{
var result = searchResult;
searchResult = this._client.Scroll <T>(scroll, result.ScrollId);
if (searchResult.Documents.HasAny())
{
indexResult = this.IndexSearchResults(searchResult, observer, toIndex, page);
}
page++;
} while (searchResult.IsValid && indexResult != null && indexResult.IsValid && searchResult.Documents.HasAny());
observer.OnCompleted();
}
private void ReportErrors(IBulkResponse result, string documentType)
{
foreach (var item in result.ItemsWithErrors)
{
var properties = new Dictionary <string, object> {
{ "DocumentType", documentType }, { "Index", item.Index }, { "Reason", item.Error.Reason }, { "Id", item.Id }
};
Log.Warn($"Error indexing entry with id {item.Id}", properties);
}
}
public bool UpdateViaBulk(dynamic partial)
{
IBulkResponse response = Client.Bulk(b => b
.Update <Load, dynamic>(u => u
.Id(partial.Id)
.Doc(partial)
));
return(response.IsValid);
}
public async Task UpdateAsync(IEnumerable <T> entities)
{
IBulkResponse result = await this._esClient.IndexManyAsync <T>(entities);
if (!result.IsValid)
{
this._logger.LogError(result.OriginalException, $"Error in updating entities of type {this._entityType} to index {this._entityIndex}", new[] { result });
throw result.OriginalException;
}
this._logger.LogTrace($"Updated {result.Items.Count} entities of type {this._entityType} to index {this._entityIndex}", new[] { result });
}
public async Task AddAsync(List <T> entities)
{
IBulkResponse result = await this._esClient.IndexManyAsync <T>(entities, this._entityIndex, this._entityType);
if (!result.IsValid)
{
this._logger.LogError(result.OriginalException, $"Error in adding entities of type {this._entityType} to index {this._entityIndex}", new[] { result });
throw result.OriginalException;
}
this._logger.LogTrace($"Added {result.Items.Count} entity of type {this._entityType} to index {this._entityIndex}. Time taken: {result.Took}", new[] { result });
}
private async Task SendEventsAsync(IEnumerable <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (events == null)
{
return;
}
try
{
string currentIndexName = this.GetIndexName(this.connectionData);
if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal))
{
await this.EnsureIndexExists(currentIndexName, this.connectionData.Client);
this.connectionData.LastIndexName = currentIndexName;
}
BulkRequest request = new BulkRequest();
request.Refresh = true;
List <IBulkOperation> operations = new List <IBulkOperation>();
foreach (EventData eventData in events)
{
BulkCreateOperation <EventData> operation = new BulkCreateOperation <EventData>(eventData);
operation.Index = currentIndexName;
operation.Type = EventDocumentTypeName;
operations.Add(operation);
}
request.Operations = operations;
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance
// between different SendEventsAsync callbacks.
// Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3
IBulkResponse response = await this.connectionData.Client.BulkAsync(request);
if (!response.IsValid)
{
this.ReportEsRequestError(response, "Bulk upload");
}
this.ReportListenerHealthy();
}
catch (Exception e)
{
this.ReportListenerProblem("Diagnostics data upload has failed." + Environment.NewLine + e.ToString());
}
}
public IBulkResponse Bulk(List <IBulkOperation> operations)
{
var request = new BulkRequest()
{
Refresh = true,
Consistency = Consistency.One,
Operations = operations
};
IBulkResponse response = GetClient().Bulk(request);
return(response);
}
/// <summary>
/// 批量添加或者更新文档
/// </summary>
/// <typeparam name="T">文档类型</typeparam>
/// <param name="documents">将要添加或者更新的文档集合</param>
/// <param name="index">文档所在库</param>
/// <returns>返回更新的条数</returns>
public int IndexMany <T>(IEnumerable <T> documents, string index = null) where T : class
{
IBulkResponse response = _builder?.Client.IndexMany <T>(documents, index ?? _defaultIndex, typeof(T).SearchName());
if (response.Errors)
{
if (_logger != null)
{
_logger.LogInformation(response.DebugInformation);
//_logger.Error(response.ItemsWithErrors)
}
}
return(response.Items.Count);
}
private static string BuildMessage(IBulkResponse response)
{
try
{
// DebugInformation can throw. ... .......
var builder = new StringBuilder();
builder.AppendLine(response.DebugInformation);
foreach (var item in response.ItemsWithErrors)
builder.AppendLine(item.Error.Reason);
return builder.ToString();
}
catch { }
return "";
}
public static void Bulk_Index_Basic(string url)
{
var uri = new Uri(url);
var config = new ConnectionSettings(uri);
var client = new ElasticClient(config);
Album a500 = new Album {
Title = "Aquemini",
Artist = "OutKast",
Rank = 500,
//...
};
Album a499 = new Album {
Title = "Live in Cook County Jail",
Artist = "B. B. King",
Rank = 499,
//...
};
Album a498 = new Album {
Title = "The Stone Roses",
Artist = "The Stone Roses",
Rank = 498,
//...
};
IBulkResponse response = client.Bulk(bs => bs
.Index <Album>(bis => bis
.Index("rolling-stone-500")
.Id(a500.Rank)
.Document(a500)
)
.Index <Album>(bis => bis
.Index("rolling-stone-500")
.Id(a499.Rank)
.Document(a499)
)
.Index <Album>(bis => bis
.Index("rolling-stone-500")
.Id(a498.Rank)
.Document(a498)
)
// ... etc ...
);
ExamineResponse(response);
}
public bool IngestCargos(IEnumerable <Cargo> cargoList, out IEnumerable <Cargo> failures)
{
failures = new List <Cargo>();
IBulkResponse response = client.Bulk(b => b.IndexMany(cargoList, (op, c) => op.Document(c)));
if (response.Errors)
{
failures = response.Items
.Select((Item, Index) => new { Item, Index })
.Where(e => !e.Item.IsValid)
.Select(e => cargoList.ElementAt(e.Index));
}
return(!response.Errors);
}
private static string BuildMessage(IBulkResponse response)
{
try
{
// DebugInformation can throw. ... .......
var builder = new StringBuilder();
builder.AppendLine(response.DebugInformation);
foreach (var item in response.ItemsWithErrors)
{
builder.AppendLine(item.Error.Reason);
}
return(builder.ToString());
}
catch { }
return("");
}
private static void ExamineResponse(IBulkResponse response)
{
Console.WriteLine("Errors: {0}", response.Errors);
Console.WriteLine("ItemsWithErrors:");
foreach (BulkOperationResponseItem item in response.ItemsWithErrors)
{
Console.WriteLine("\tId: {0}", item.Id);
Console.WriteLine("\t\tIndex: {0}", item.Index);
Console.WriteLine("\t\tType: {0}", item.Type);
Console.WriteLine("\t\tStatus: {0}", item.Status);
Console.WriteLine("\t\tError: {0}", item.Error);
}
Console.WriteLine();
}
protected void StartPublishTimer()
{
int thirtySeconds = 1000 * 30;
this.publishTimer = new Timer((g) => {
BaseDocument document;
IBulkRequest request = new BulkRequest("active-homeseer-index")
{
Operations = new List <IBulkOperation>()
};
while (!this.cache.IsEmpty)
{
if (this.cache.TryTake(out document))
{
request.Operations.Add(new BulkIndexOperation <BaseDocument>(document));
}
}
if (request.Operations.Count == 0)
{
pluginInstance.LogDebug("WriteToCluster: no documents to write");
this.publishTimer.Change(thirtySeconds, Timeout.Infinite);
return;
}
try
{
pluginInstance.LogDebug(string.Format("Writing {0} documents to Elasticsearch...", request.Operations.Count));
ElasticClient client = GetESClient(Config);
IBulkResponse r = client.Bulk(request);
if (r.IsValid)
{
pluginInstance.LogDebug("Document publish Successful!");
}
else
{
pluginInstance.LogError(string.Format("Document published failed: {0}", r.DebugInformation));
}
}
catch (Exception e)
{
pluginInstance.LogError(string.Format("Error writing documents to Elasticsearch: {0}", e.Message));
}
this.publishTimer.Change(thirtySeconds, Timeout.Infinite);
}, null, 0, Timeout.Infinite);
}
/// <summary>
/// 批量添加或者更新文档
/// </summary>
/// <typeparam name="T">文档类型</typeparam>
/// <param name="documents">将要添加或者更新的文档集合</param>
/// <param name="index">文档所在库</param>
/// <returns>返回更新的条数</returns>
public int Bulk <T>(IEnumerable <T> objects, string index = null) where T : class
{
BulkDescriptor descriptor = new BulkDescriptor();
descriptor.Index(index ?? _defaultIndex).Type(typeof(T).SearchName()).IndexMany(objects);
IBulkResponse response = _builder?.Client.Bulk(descriptor);
if (response.Errors)
{
if (_logger != null)
{
_logger.LogInformation(response.DebugInformation);
//_logger.Error(response.ItemsWithErrors)
}
}
return(response.Items.Count);
}
private (bool success, bool retry) retryOnResponse(IBulkResponse response, DispatcherQueueItem <T> queued)
{
// Elasticsearch bulk thread pool is full.
if (response.ItemsWithErrors.Any(item => item.Status == 429 || item.Error.Type == "es_rejected_execution_exception"))
{
Console.WriteLine($"Server returned 429, re-queued chunk with lastId {queued.ItemsToIndex.Last().CursorValue}");
return(success : false, retry : true);
}
// Index was closed, possibly because it was switched. Flag for bailout.
if (response.ItemsWithErrors.Any(item => item.Error.Type == "index_closed_exception"))
{
Console.Error.WriteLine($"{index} was closed.");
readBuffer.CompleteAdding(); // FIXME: should cancel instead.
return(success : false, retry : false);
}
// TODO: other errors should do some kind of notification.
return(success : true, retry : false);
}
public virtual async Task UpsertAsync(IEnumerable <T> source, CancellationToken cancellationToken)
{
// Validate parameters.
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
// The request.
IBulkRequest request = new BulkDescriptor().
Index(Index.Name).
Type <T>().
IndexMany(source, (d, t) => d.Document(t));
// Send the request.
IBulkResponse response = await ElasticClient.BulkAsync(request, cancellationToken).ConfigureAwait(false);
// Throw if there is an error.
response.ThrowIfError();
}
private void SavePack(string index, Type type, string parentId, List <object> bulkPack)
{
IBulkResponse br = null;
Exception exception = null;
for (int i = 0; i < 3; i++)
{
try
{
br = Client.Bulk(b => b.Index(index).Type(type).IndexMany(bulkPack, (a, o) =>
{
var prntId = parentId ?? (o as IParent)?.getparentid();
if (prntId != null)
{
return(a.Parent(new Id(prntId)));
}
else
{
return(a);
}
}).Refresh(Refresh.False));
}
catch (Exception e)
{
//произошла ошибка? попробуй еще раз .. возможно это интернет лагнул
exception = e;
continue;
}
break;
}
if (br == null)
{
throw exception ?? new Exception("не удалось получить ответ от сервера");
}
if (br.Errors)
{
throw new Exception(br.DebugInformation.Substring(0, 10000));
}
}
public async Task DeleteAsync(IList <long> ids)
{
var descriptor = new BulkDescriptor();
foreach (var id in ids)
{
descriptor.Delete <T>(x => x
.Id(id))
.Refresh(Refresh.WaitFor);
}
IBulkResponse result = await this._esClient.BulkAsync(descriptor);
if (!result.IsValid)
{
this._logger.LogError(result.OriginalException, $"Error deleting entities of type {this._entityType} from index {this._entityIndex}.", new[] { result });
throw result.OriginalException;
}
this._logger.LogTrace($"Deleted {result.Items.Count} entities of type {this._entityType} from index {this._entityIndex}.", new[] { result });
}
private ImportFileResult BuildFileResultFromClientResponse(string file, IBulkResponse response)
{
if (response.ConnectionStatus.Success)
return new ImportFileResult(file, response.Items.Count());
return new ImportFileResult(file, response.ConnectionStatus.Error.ExceptionMessage);
}
public void InitialiseImporter(IBulkResponse mockedBulkResponse = null)
{
_mockedFileSystem.Setup(fs => fs.DirectoryExists(TestSourceDirectory)).Returns(true);
_mockedClientProvider.Setup(cp => cp.GetClient(_mockedConnectionSettings.Object))
.Returns(_mockedClient.Object);
_mockedFileParser.Setup(fp => fp.GetTweets(It.IsAny<string>())).Returns(_testTweets);
if (mockedBulkResponse == null)
_mockedClient.Setup(c => c.IndexMany(It.IsAny<IEnumerable<Tweet>>())).Returns(_mockedClientSuccessResponse.Object);
else
_mockedClient.Setup(c => c.IndexMany(It.IsAny<IEnumerable<Tweet>>())).Returns(mockedBulkResponse);
_mockedClient.Setup(c => c.IsValid).Returns(true);
_importer = new Importer(_mockedFileSystem.Object, _mockedFileParser.Object, _mockedClientProvider.Object, _mockedConnectionSettings.Object, TestSourceDirectory);
}
public StorageException(IBulkResponse response) : base(BuildMessage(response)) {
}