public Artist[] Init(TestEnvironment environment) { if (_artists != null && _artists.Any()) { return(_artists); } IntegrationTestsRuntime.EnsureCleanEnvironment(); _artists = ClientTestData.Artists.CreateArtists(10); using (var client = IntegrationTestsRuntime.CreateDbClient()) { var bulk = new BulkRequest(); bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var artist = _artists.Single(i => i.ArtistId == row.Id); client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev); } client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Wait(); var queryRequests = ClientTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter))); var queries = queryRequests.Select(q => client.Views.QueryAsync(q) as Task).ToArray(); Task.WaitAll(queries); } return(_artists); }
/// <summary> /// 推送ES数据列表 /// </summary> /// <typeparam name="T">数据模型</typeparam> /// <param name="index">索引名称 [db]</param> /// <param name="type">索引类型 [table]</param> /// <param name="primarykeyName">推送的主键名称</param> /// <param name="data">推送数据</param> /// <returns></returns> public static bool Pulk <T>(string index, string type, string primarykeyName, List <T> data) where T : class { var bulkRequest = new BulkRequest(index, type) { Operations = new List <IBulkOperation>() }; var indexs = data.Select(p => new BulkIndexOperation <T>(p) { Id = p.GetType().GetProperty(primarykeyName).GetValue(p, null).ToString() }) .Cast <IBulkOperation>() .ToList(); bulkRequest.Operations = indexs; var response = client.Bulk(bulkRequest); if (response.OriginalException != null) { throw new ArgumentException(response.OriginalException.Message, response.OriginalException); } if (DebuggerMode && response.ApiCall != null) { var url = response.ApiCall.Uri.AbsoluteUri; var query = System.Text.Encoding.UTF8.GetString(response.ApiCall.RequestBodyInBytes); var responseText = System.Text.Encoding.UTF8.GetString(response.ApiCall.ResponseBodyInBytes); Debug.WriteLine(string.Format("url:{0}\t\nquery:{1}\t\n response:{2}", url, query, responseText)); } return(response.IsValid); }
public async Task BulkRequest_SerialisationTest() { var operations = new BulkOperationsCollection { new BulkIndexOperation <Project>(FixedProject) { Index = "project" }, new BulkIndexOperation <Project>(FixedProject) }; var request = new BulkRequest { Operations = operations }; var serialisedJson = await SerializeAndGetJsonStringAsync(request); var sr = new StringReader(serialisedJson); var count = 0; while (true) { var line = sr.ReadLine(); if (line is not null) { await Verifier.VerifyJson(line).UseMethodName($"{nameof(BulkRequest_SerialisationTest)}_{++count}"); } else { break; } } }
public SearchFixture() { Animals = CloudantTestData.Animals.CreateAll(); Client = IntegrationTestsRuntime.CreateCloudantClient(); var bulk = new BulkRequest(); bulk.Include(Animals.Select(i => Client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = Client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var animal = Animals.Single(i => i.AnimalId == row.Id); Client.Entities.Reflector.RevMember.SetValueTo(animal, row.Rev); } Client.Documents.PostAsync(CloudantTestData.Views.Views101).Wait(); var queries = CloudantTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter))); foreach (var query in queries) { Client.Views.QueryAsync(query).Wait(); } }
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { BulkRequest request = (BulkRequest)value; if (request == null) { return; } foreach (var indexItem in request.IndexItems) { writer.WriteStartObject(); writer.WritePropertyName(indexItem.ActionType); writer.WriteStartObject(); writer.WritePropertyName("_index"); writer.WriteValue(indexItem.Index); writer.WritePropertyName("_type"); writer.WriteValue(indexItem.Type); writer.WritePropertyName("_id"); writer.WriteValue(indexItem.DocuemntId); writer.WriteEndObject(); writer.WriteEndObject(); writer.WriteRawValue("\n"); if (indexItem.ActionType != "delete") { serializer.Serialize(writer, indexItem.DocumentEntity); writer.WriteRawValue("\n"); } } }
public bool BulkIndex <T>(IEnumerable <T> inputObjectList, string indexName, string typeName, bool isAsync = false) where T : class { try { if (!string.IsNullOrEmpty(indexName)) { indexName = indexName.ToLower(); } if (!string.IsNullOrEmpty(typeName)) { typeName = typeName.ToLower(); } var operations = inputObjectList.Select(item => new BulkIndexOperation <T>(item)).Cast <IBulkOperation>().ToList(); var request = new BulkRequest(indexName, typeName) { Refresh = true, Operations = operations }; if (isAsync) { var res = _client.BulkAsync(request); } else { var res = _client.Bulk(request); } return(true); } catch (Exception ex) { throw; } }
public async Task SyncData() { //1290213 = 13x100000 for (int i = 1; i < 14; i++) { var recepies = await _dbContext .Recipes.Where(r => r.Id <= 100000 *i && r.Id > 100000 *(i - 1)) .Include(r => r.Ingredients) .Include(r => r.RecipeTag) .ThenInclude(rt => rt.Tag) .ToListAsync(); var recepiesEsModel = _mapper.Map <List <RecipeSearchModel> >(recepies); var request = new BulkRequest(indexName) { Operations = new List <IBulkOperation>() }; var bulkOperations = recepiesEsModel.Select(r => new BulkIndexOperation <RecipeSearchModel>(r)); request.Operations.AddRange(bulkOperations); await _elasticClient.BulkAsync(request); } Console.WriteLine("Sync completed"); }
/// <summary> /// Res the index. /// </summary> /// <returns>Response data for a bulk insert of items</returns> /// <exception cref="System.Exception">Cound not delete log to start re-index - " + deleteResponse.DebugInformation</exception> public IBulkResponse ReIndex() { if (this.elasticClient.IndexExists(this.indexTablename).Exists) { var response = this.elasticClient.DeleteIndex(this.indexTablename); if (!response.IsValid) { throw new Exception("Cound not delete log to start re-index - " + response.DebugInformation); } } this.VerifyIndexStatus(); // this will get all records in the table. Needless to say this is not verify effective so we don't want to do this very often. var results = this.dynamoDbContext.ScanAsync <TEntity>(null); var bulkRequest = new BulkRequest(this.indexTablename) { Operations = new List <IBulkOperation>() }; while (!results.IsDone) { var batch = results.GetNextSetAsync().Result; foreach (var item in batch) { bulkRequest.Operations.Add(new BulkIndexOperation <TEntity>(item)); } } return(this.elasticClient.Bulk(bulkRequest)); }
private void InsertIntoCouchDB(string databaseName, string tableName, List <DatabaseRecord> rows, RowTransformer rowTransformer) { const int chunkSize = 1000; using (var client = new MyCouchClient(CouchDbUrl, databaseName)) { Parallel.ForEach(rows.Chunk(chunkSize), new ParallelOptions { MaxDegreeOfParallelism = 20 }, chunkEnumerable => { var chunk = chunkEnumerable.ToArray(); IEnumerable <DatabaseRecord> transformedChunk; if (rowTransformer == null) { transformedChunk = chunk; } else { transformedChunk = chunk.Select(row => rowTransformer(tableName, row)); } var documents = transformedChunk.Select(JsonConvert.SerializeObject).ToArray(); var request = new BulkRequest(); request.Include(documents); client.Documents.BulkAsync(request).Wait(); }); } }
static void Main(string[] args) { var totalThreads = 4; var bulkRequests = 500; var node = new Uri("http://myserver:9200"); var token = cancelToken; for (int i = 0; i < totalThreads; i++) { Task.Run(() => { while (!token.IsCancellationRequested) { var settings = new ConnectionSettings(node); var client = new ElasticClient(settings); var bulkRequest = new BulkRequest() { Operations = GetOperations(bulkRequests) }; var result = client.Bulk(bulkRequest); Console.WriteLine("Thread {0} publushed {1} requests with a status of {2}", i, result.IsValid, bulkRequests); } } , cancelToken); } }
public virtual async Task <DeleteManyResult> DeleteManyAsync(params DocumentHeader[] documents) { ThrowIfDisposed(); Ensure.That(documents, "documents").HasItems(); var request = new BulkRequest() .Delete(documents); var response = await Client.Documents.BulkAsync(request); ThrowIfNotSuccessfulResponse(response); return(new DeleteManyResult { Rows = response.Rows.Select(r => new DeleteManyResult.Row { Id = r.Id, Rev = r.Rev, Error = r.Error, Reason = r.Reason, Deleted = r.Succeeded }).ToArray() }); }
public async Task InsertRangeAsync <T>(IEnumerable <T> entity, string index) where T : class { var indexName = index.GetIndex <T>(); var exists = await IndexExistsAsync(indexName); if (!exists) { await((ElasticClient)_elasticClient).CreateIndexAsync <T>(indexName); await AddAliasAsync(indexName, typeof(T).Name); } var bulkRequest = new BulkRequest(indexName) { Operations = new List <IBulkOperation>() }; var operations = entity.Select(o => new BulkIndexOperation <T>(o)).Cast <IBulkOperation>().ToList(); bulkRequest.Operations = operations; var response = await _elasticClient.BulkAsync(bulkRequest); if (!response.IsValid) { throw new Exception("批量新增数据失败:" + response.OriginalException.Message); } }
public BulkOperationsResponse Delete(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List<IBulkOperation>() }; foreach (var doc in results.Items) { var bulkDeleteOperation = new BulkDeleteOperation<IElasticItem>(doc.Identifier); bulkDeleteOperation.Type = type.Type.Name; var bulkOp = bulkDeleteOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; } else { ingestionResponse.Deleted++; } } return ingestionResponse; }
/// <summary> /// Bulks the async. /// </summary> /// <returns>The async.</returns> /// <param name="ideasJson">Ideas json.</param> public async Task BulkAsync(List <string> ideasJson) { var request = new BulkRequest(); request.Include(ideasJson.ToArray()); var response = await Documents.BulkAsync(request); }
public Dictionary <string, List <Entity> > GetTargetEntitiesForLookupMatch(IOrganizationService crmService, List <LookupMatchCriteria> lookupMatchCriterias) { Dictionary <string, List <Entity> > entityLookups = new Dictionary <string, List <Entity> >(); Dictionary <string, QueryExpression> queriesToBeProcessed = new Dictionary <string, QueryExpression>(); foreach (LookupMatchCriteria lookupMatchCriteria in lookupMatchCriterias) { QueryExpression queryForLookupMatching = GetQueryForLookupMatching(lookupMatchCriteria); queriesToBeProcessed.Add(lookupMatchCriteria.EntityLogicalName, queryForLookupMatching); } BulkRequest bulkRequest = new BulkRequest(100); BulkResponse bulkResponse = bulkRequest.Read(crmService, queriesToBeProcessed); foreach (BulkResponseItem bulkResponseItem in bulkResponse.Responses) { if (bulkResponseItem.Results != null && bulkResponseItem.Results.Entities != null) { entityLookups.Add(bulkResponseItem.BulkRequestId, bulkResponseItem.Results.Entities.ToList()); } } return(entityLookups); }
public IAsyncEnumerable <string> GetJsonSkinsByIds( IReadOnlyCollection <int> itemIds, IProgress <ICollectionContext>?progress = default, CancellationToken cancellationToken = default ) { var producer = SplitQuery.Create <int, string>( async(range, ct) => { var request = new BulkRequest("/v2/skins") { Ids = range }; var json = await request.SendAsync(http, ct); return(json.Indent(false) .RootElement.EnumerateArray() .Select( item => item.ToString() ?? throw new InvalidOperationException("Unexpected null in JSON array.") ) .ToList()); }, progress ); return(producer.QueryAsync(itemIds, cancellationToken: cancellationToken)); }
public Artist[] Init(TestEnvironment environment) { if (_artists != null && _artists.Any()) { return(_artists); } IntegrationTestsRuntime.EnsureCleanEnvironment(); _artists = ClientTestData.Artists.CreateArtists(2); using (var client = IntegrationTestsRuntime.CreateDbClient()) { var bulk = new BulkRequest(); bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var artist = _artists.Single(i => i.ArtistId == row.Id); client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev); } client.Documents.PostAsync(ClientTestData.Shows.ArtistsShows).Wait(); } return(_artists); }
public virtual async Task <DeleteManyResult> DeleteManyAsync(DocumentHeader[] documents, CancellationToken cancellationToken = default) { ThrowIfDisposed(); EnsureArg.HasItems(documents, nameof(documents)); var request = new BulkRequest() .Delete(documents); var response = await Client.Documents.BulkAsync(request, cancellationToken); ThrowIfNotSuccessfulResponse(response); return(new DeleteManyResult { Rows = response.Rows.Select(r => new DeleteManyResult.Row { Id = r.Id, Rev = r.Rev, Error = r.Error, Reason = r.Reason, Deleted = r.Succeeded }).ToArray() }); }
public BulkRequestTests() { var newProject = new ElasticsearchProject { Id = 4, Name = "new-project" }; var request = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Operations = new List <IBulkOperation> { { new BulkIndexOperation <ElasticsearchProject>(newProject) { Id = "2" } }, { new BulkDeleteOperation <ElasticsearchProject>(6) }, { new BulkCreateOperation <ElasticsearchProject>(newProject) { Id = "6" } }, { new BulkUpdateOperation <ElasticsearchProject, object>(newProject, new { name = "new-project2" }) { Id = "3" } }, } }; var response = this._client.Bulk(request); this._status = response.ConnectionStatus; }
public void Delete() { using (var client = new MyCouchClient(_databaseUri, _databaseName)) { QueryViewRequest request = new QueryViewRequest( "bookorders", "allOrders"); request.Configure(parameters => parameters.IncludeDocs(true)); ViewQueryResponse <string> results = client.Views.QueryAsync <string>(request).Result; var bulkRequest = new BulkRequest() { AllOrNothing = false }; foreach (var resultsRow in results.Rows) { BookOrderDto dto = JsonConvert.DeserializeObject <BookOrderDto>( resultsRow.IncludedDoc); bulkRequest.Delete(dto._id, dto._rev); Console.WriteLine($"Delete request for id:{dto._id}"); } client.Documents.BulkAsync(bulkRequest).Wait(); } }
protected virtual string GenerateRequestBody(BulkRequest request) { var sb = new StringBuilder(); var documents = request.GetDocuments(); sb.Append("{"); if (request.AllOrNothing) { sb.Append("\"all_or_nothing\":true,"); } if (!request.NewEdits) { sb.Append("\"new_edits\":false,"); } sb.Append("\"docs\":["); for (var i = 0; i < documents.Length; i++) { sb.Append(documents[i]); if (i < documents.Length - 1) { sb.Append(","); } } sb.Append("]}"); return(sb.ToString()); }
public int BulkIndex(IEnumerable <GeoEntity> entities) { var client = ElasticClientFactory.CreateClient(_elasticSettings); CreateIndex(client); var errorCount = 0; var batch = 0; foreach (var block in entities.Batch(_elasticSettings.BatchSize)) { var bulkRequest = new BulkRequest() { Operations = new List <IBulkOperation>() }; foreach (var item in block) { bulkRequest.Operations.Add(new BulkIndexOperation <GeoEntity>(item)); } batch++; var result = client.Bulk(bulkRequest); var errors = result.ItemsWithErrors.Count(); if (errors > 0) { _logger.LogWarning("{}"); } errorCount += errors; } return(errorCount); }
private bool DeleteEsData(IEnumerable <UserMessageBoxInfoRelation> relations) { if (relations != null && relations.Any()) { var operations = new List <IBulkOperation>(); var exlogs = ConvertMessageLogToEsLog(relations); foreach (var log in exlogs) { operations.Add(new BulkDeleteOperation <UserMessageBoxInfoRelationC>(log.Id) { Routing = log.UserId }); } var bulkRequest = new BulkRequest(UserMessageBoxRelationIndex) { Operations = operations.ToArray() }; var client = ElasticsearchHelper.CreateClient(); #if DEBUG ElasticsearchHelper.EnableDebug(); #endif var responseOne = client.Bulk(bulkRequest); var errorcount = responseOne.ItemsWithErrors.Count(); DalMessageBox.WriteSyncLogs(relations); return(errorcount <= 0); } return(false); }
public ViewsFixture() { Artists = ClientTestData.Artists.CreateArtists(10); using (var client = IntegrationTestsRuntime.CreateNormalClient()) { var bulk = new BulkRequest(); bulk.Include(Artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray()); var bulkResponse = client.Documents.BulkAsync(bulk).Result; foreach (var row in bulkResponse.Rows) { var artist = Artists.Single(i => i.ArtistId == row.Id); client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev); } var tmp = client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Result; var touchView1 = new QueryViewRequest(ClientTestData.Views.ArtistsAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter)); var touchView2 = new QueryViewRequest(ClientTestData.Views.ArtistsNameNoValueViewId).Configure(q => q.Stale(Stale.UpdateAfter)); var touchView3 = new QueryViewRequest(ClientTestData.Views.ArtistsTotalNumOfAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter)); client.Views.QueryAsync(touchView1).Wait(); client.Views.QueryAsync(touchView2).Wait(); client.Views.QueryAsync(touchView3).Wait(); } }
private static BulkRequest CreateIndexBulkRequest <T>(IEnumerable <T> objects, Func <T, string> getIndex, string type, Func <T, string> getParent) where T : class { var bulkRequest = new BulkRequest(); TypeNameMarker typeNameMarker = type; bulkRequest.Type = typeNameMarker; var list = objects.Select(o => { var doc = new BulkIndexOperation <T>(o); if (getParent != null) { doc.Parent = getParent(o); } if (getIndex != null) { doc.Index = getIndex(o); } var versionedDoc = o as IVersioned; if (versionedDoc != null) { doc.Version = versionedDoc.Version.ToString(); } return(doc); }).Cast <IBulkOperation>().ToList(); bulkRequest.Operations = list; return(bulkRequest); }
public async Task <BulkResponse> BulkDocumentsAsync(BulkRequest request) { string url = request.GetUrl(_clusterUrl); var response = await SendAsync(url, HttpMethod.Post, request.GetBody()); return(new BulkResponse(response)); }
/// <summary> /// 批量保存 /// </summary> /// <typeparam name="T">实体类型</typeparam> /// <param name="indexName">索引名</param> /// <param name="entities">实体列表</param> public async Task BulkSaveAsync <T>(string indexName, IEnumerable <T> entities) where T : class { var client = await _builder.GetClientAsync(); if (!await ExistsAsync(indexName)) { await client.InitializeIndexMapAsync <T>(indexName); } var bulk = new BulkRequest(indexName) { Operations = new List <IBulkOperation>() }; foreach (var entity in entities) { bulk.Operations.Add(new BulkIndexOperation <T>(entity)); } var response = await client.BulkAsync(bulk); if (response.Errors) { throw new ElasticsearchException($"批量保存文档在索引 {indexName} 失败:{response.ServerError.Error.Reason}"); } }
/// <summary> /// Attempts a forced replace or insert. /// If conflicts are detected, they are dropped, and the local version is again updated. /// Returns null if the method should be invoked again, otherwise the final version retrieved from the DB /// </summary> /// <typeparam name="T">Entity type</typeparam> /// <param name="store">DB to place the entity in</param> /// <param name="item">Entity to place. The _rev member may be updated in the process</param> /// <returns></returns> public static async Task <T> TryForceReplace <T>(DataBase store, T item) where T : Entity { //try { if (store == null) { return(item); //during tests, db is not loaded } var serial = JsonConvert.SerializeObject(item, Newtonsoft.Json.Formatting.None, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); //store.Entities.Serializer.Serialize(item); //JsonConvert.SerializeObject(item); var header = await store.Documents.PutAsync(item._id, serial); //await store.Entities.PutAsync(item._id, item); if (!header.IsSuccess /*&& header.StatusCode == System.Net.HttpStatusCode.Conflict*/) { var head = await store.Documents.HeadAsync(item._id); if (head.IsSuccess) { item._rev = head.Rev; } return(null); } var echoRequest = new GetEntityRequest(header.Id); echoRequest.Conflicts = true; var echo = await store.Documents.GetAsync(header.Id); if (!echo.IsSuccess) { return(null); } if (echo.Conflicts != null && echo.Conflicts.Length > 0) { //we do have conflicts and don't quite know whether the new version is our version var deleteHeaders = new DocumentHeader[echo.Conflicts.Length]; for (int i = 0; i < echo.Conflicts.Length; i++) { deleteHeaders[i] = new DocumentHeader(header.Id, echo.Conflicts[i]); } BulkRequest breq = new BulkRequest().Delete(deleteHeaders); //delete all conflicts await store.Documents.BulkAsync(breq); //ignore result, but wait for it item._rev = echo.Rev; //replace again. we must win return(await TryForceReplace(store, item)); } return(JsonConvert.DeserializeObject <T>(echo.Content)); //all good } //catch (Exception ex) //{ // throw; //} }
public virtual HttpRequest Create(BulkRequest request) { Ensure.Any.IsNotNull(request, nameof(request)); return(new HttpRequest(HttpMethod.Post, GenerateRelativeUrl(request)) .SetRequestTypeHeader(request.GetType()) .SetJsonContent(GenerateRequestBody(request))); }
public virtual HttpRequest Create(BulkRequest request) { var createHttpRequest = CreateFor <BulkRequest>(HttpMethod.Post, GenerateRequestUrl(request)); createHttpRequest.SetContent(request.ToJson()); return(createHttpRequest); }
public virtual HttpRequest Create(BulkRequest request) { var createHttpRequest = CreateFor<BulkRequest>(HttpMethod.Post, GenerateRequestUrl(request)); createHttpRequest.SetContent(request.ToJson()); return createHttpRequest; }
private static async Task <T> GetConflictResolvedAsync(DataBase store, string id, Func <ICollection <T>, T> merger, CancellationToken cancellationToken = default(CancellationToken)) { var req = new GetDocumentRequest(id); req.Conflicts = true; var data = await store.Documents.GetAsync(req, cancellationToken); //await store.Entities.GetAsync<T>(req, cancellationToken); if (!data.IsSuccess) { return(null); } var mainContent = JsonConvert.DeserializeObject <T>(data.Content); if (data.Conflicts == null || data.Conflicts.Length == 0) { return(mainContent); } List <T> conflicting = new List <T>(); List <DocumentHeader> toDelete = new List <DocumentHeader>(); conflicting.Add(mainContent); foreach (var fetch in data.Conflicts) { var item = await store.Documents.GetAsync(id, fetch, cancellationToken); if (item.IsSuccess) { conflicting.Add(JsonConvert.DeserializeObject <T>(item.Content)); toDelete.Add(new DocumentHeader(item.Id, item.Rev)); } else { return(null); //assume connection lost or competing merge } } T merged = merger(conflicting); if (merged == null) { Log.Error("Error trying to fetch conflicting data from " + store + ":" + id + " with no merger set. Fixed. Result will be arbitrary"); merged = mainContent; } merged._id = id; merged._rev = data.Rev; var putResult = await store.Entities.PutAsync(merged); if (putResult.IsSuccess) { var deleteRequest = new BulkRequest().Delete(toDelete.ToArray()); deleteRequest.AllOrNothing = true; await store.Documents.BulkAsync(deleteRequest); //await, ignore result, let others clean it up if failed return(putResult.Content); } return(null); //assume connection lost or asynchronous update }
public BulkRequestTests() { var newProject = new ElasticsearchProject { Id = 4, Name = "new-project" }; var request = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Operations = new List<IBulkOperation> { { new BulkIndexOperation<ElasticsearchProject>(newProject) { Id= "2"}}, { new BulkDeleteOperation<ElasticsearchProject>(6) }, { new BulkCreateOperation<ElasticsearchProject>(newProject) { Id = "6" } }, { new BulkUpdateOperation<ElasticsearchProject, object>(newProject, new { name = "new-project2"}) { Id = "3" } }, } }; var response = this._client.Bulk(request); this._status = response.ConnectionStatus; }
// // Send the messages to Elasticsearch (bulk) // private DateTime transmitBulkData(IEnumerable<JObject> bulkItems, string bulkIndexName, string bulkTypeName, ElasticClient client, DateTime lastFlushTime, List<JObject> messages) { var bulkRequest = new BulkRequest() {Refresh = true}; bulkRequest.Operations = new List<IBulkOperation>(); foreach (var json in bulkItems) { // ES requires a timestamp, add one if not present var ts = json["@timestamp"]; if (ts == null) json["@timestamp"] = DateTime.UtcNow; var bi = new BulkIndexOperation<JObject>(json); bi.Index = bulkIndexName; bi.Type = bulkTypeName; bulkRequest.Operations.Add(bi); } // The total messages processed for this operation. int numMessages = bulkItems.Count(); var response = client.Bulk(bulkRequest); if (!response.IsValid) { LogManager.GetCurrentClassLogger().Error("Failed to send: {0}", response); Interlocked.Increment(ref _errorCount); interlockedInsert(messages); // Put the messages back into the queue } else // Success! { lastFlushTime = DateTime.UtcNow; LogManager.GetCurrentClassLogger() .Info("Successfully sent {0} messages in a single bulk request", numMessages); Interlocked.Add(ref _sentMessages, numMessages); } // Remove them from the working list messages.RemoveRange(0, numMessages); return lastFlushTime; }
protected virtual string GenerateRequestUrl(BulkRequest request) { return string.Format("{0}/_bulk_docs", Connection.Address); }
/// <summary> /// (Re)Indexes the complete page tree for pagedata that inherits from <see cref="IIndexablePageData"/> /// </summary> /// <param name="swapWithErrors">if set to <c>true</c> and if an error occured during the index process the new index is swapped to live</param> /// <param name="onStatusChanges">the action that is executed on status changed during the index process</param> /// <returns></returns> public IEnumerable<IBulkResponse> IndexPageTree(bool swapWithErrors = false, Action<string> onStatusChanges = null) { foreach (var languageBranch in LanguageBranchRepository.ListEnabled()) { var language = languageBranch.Culture; InitializeIndex(language); var aliasName = GetAliasName(language); var indexName1 = GetIndexName(language, 1); var indexName2 = GetIndexName(language, 2); var aliasIndices = ElasticClient.GetAlias(x => x.Name(aliasName)).Indices; var liveIndexName = aliasIndices.First().Key; var reIndexName = liveIndexName.Equals(indexName1) ? indexName2 : indexName1; // clear the reIndex indice since we are re-building it Clear(reIndexName, language); var indexablePages = GetIndexablePages(language).ToArray(); var bulkOperations = new List<IBulkOperation>(); bool hasErrors = false; for (var i = 0; i < indexablePages.Length; i++) { var indexablePageData = indexablePages[i]; if (!indexablePageData.ShouldIndex()) continue; var indexModel = CreateIndexModel(indexablePageData); bulkOperations.Add(CreateBulkOperation(indexModel)); if (bulkOperations.Count == Options.BulkSize || i == indexablePages.Length - 1) { var bulkRequest = new BulkRequest(reIndexName) {Operations = new List<IBulkOperation>(bulkOperations)}; var bulkResponse = ElasticClient.Bulk(bulkRequest); if (!bulkResponse.IsValid || bulkResponse.Errors) hasErrors = true; bulkOperations.Clear(); yield return bulkResponse; } onStatusChanges?.Invoke($"Insterted bulk into {reIndexName} currentIndex: {i}"); } if (!hasErrors || swapWithErrors) SwapIndex(liveIndexName, aliasName, reIndexName); } }
private async Task SendEventsAsync(IEnumerable<EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken) { if (events == null) { return; } try { string currentIndexName = this.GetIndexName(this.connectionData); if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal)) { await this.EnsureIndexExists(currentIndexName, this.connectionData.Client); this.connectionData.LastIndexName = currentIndexName; } BulkRequest request = new BulkRequest(); request.Refresh = true; List<IBulkOperation> operations = new List<IBulkOperation>(); foreach (EventData eventData in events) { BulkCreateOperation<EventData> operation = new BulkCreateOperation<EventData>(eventData); operation.Index = currentIndexName; operation.Type = EventDocumentTypeName; operations.Add(operation); } request.Operations = operations; if (cancellationToken.IsCancellationRequested) { return; } // Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance // between different SendEventsAsync callbacks. // Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3 IBulkResponse response = await this.connectionData.Client.BulkAsync(request); if (!response.IsValid) { this.ReportEsRequestError(response, "Bulk upload"); } this.ReportListenerHealthy(); } catch (Exception e) { this.ReportListenerProblem("Diagnostics data upload has failed." + Environment.NewLine + e.ToString()); } }
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results) { OpenSearchFactory.RemoveLinksByRel(ref results, "self"); OpenSearchFactory.RemoveLinksByRel(ref results, "search"); IElasticCollection docs = type.FromOpenSearchResultCollection(results); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = type.Index, Type = type.Type, Operations = new List<IBulkOperation>() }; RootObjectMapping currentMapping = null; try { var mappingResponse = client.GetMapping<IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name)); currentMapping = mappingResponse.Mapping; } catch (Exception) { } var rootObjectMapping = type.GetRootMapping(); if (!rootObjectMapping.Equals(currentMapping)) { client.Map<IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name)); } foreach (var doc in docs.ElasticItems) { var bulkIndexOperation = new BulkIndexOperation<IElasticItem>(doc); bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier; bulkIndexOperation.Type = type.Type.Name; var bulkOp = bulkIndexOperation; bulkRequest.Operations.Add(bulkOp); } var response = client.Bulk(bulkRequest); BulkOperationsResponse ingestionResponse = new BulkOperationsResponse(); foreach (var item in response.Items) { if (!item.IsValid) { ingestionResponse.Errors++; continue; } if (item.Version == "1") ingestionResponse.Added++; else ingestionResponse.Updated++; } return ingestionResponse; }
public object Post(TypeImportRequest request) { OpenSearchEngine ose = new OpenSearchEngine(); ose.LoadPlugins(); ose.DefaultTimeOut = 60000; OpenSearchUrl url = new OpenSearchUrl(request.url); IOpenSearchable entity = new GenericOpenSearchable(url, ose); IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName); IOpenSearchResultCollection osres = ose.Query(entity, new NameValueCollection()); OpenSearchFactory.RemoveLinksByRel(ref osres, "alternate"); OpenSearchFactory.RemoveLinksByRel(ref osres, "via"); OpenSearchFactory.RemoveLinksByRel(ref osres, "self"); OpenSearchFactory.RemoveLinksByRel(ref osres, "search"); IElasticCollection documents = type.FromOpenSearchResultCollection(osres); BulkRequest bulkRequest = new BulkRequest() { Refresh = true, Consistency = Consistency.One, Index = request.IndexName }; foreach (var doc in documents.Items) { bulkRequest.Operations.Add(new BulkIndexOperation<IElasticItem>((IElasticItem)doc) { Id = doc.Id }); } var response = client.Bulk(bulkRequest); return response; }
public void index() { LogReader LogReaderInst = new LogReader(fileName); Stopwatch sw = new Stopwatch(); long LineCount = 0; sw.Start(); if (mapping.Equals("msit")) { List<MSITPackage> packages = LogReaderInst.MSITPackagesWrapper(bulkSize); while (packages != null && packages.Count >= 1) { //Index LineCount += packages.Count; Console.WriteLine("Current total lines: {0}", LineCount); try { if(updateFlag.Equals("F") || updateFlag.Equals("f") || updateFlag.Equals("T") || updateFlag.Equals("t")) {//Indexing var result = Client.IndexMany<MSITPackage>(packages, indexName); if (!result.IsValid) { foreach (var item in result.ItemsWithErrors) { Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error); } Console.WriteLine(result.ConnectionStatus.OriginalException.Message); //Console.Read(); Environment.Exit(1); } } else//Updating { List<IBulkOperation> bulkOpt = new List<IBulkOperation>(bulkSize); for(int i = 0; i < packages.Count; ++i) { MSITPackage p = packages[i]; /* Client.Update<MSITPackage, object>(u => u .IdFrom(p) .Doc(new {CustomData = p.CustomData, VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category }) .DocAsUpsert() ); */ BulkUpdateOperation<MSITPackage, object> bo = new BulkUpdateOperation<MSITPackage, object>(p, new { CustomData = "123435CustomData", VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category }, false); bo.RetriesOnConflict = 3; //BulkUpdateOperation<MSITPackage, object> bo = new BulkUpdateOperation<MSITPackage, object>(p, new { CustomData = p.CustomData, VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category }); bulkOpt.Add(bo); } var request = new BulkRequest() { Refresh = true, Consistency = Elasticsearch.Net.Consistency.One, Operations = bulkOpt }; //Console.WriteLine("Bulk indexing"); var result = Client.Bulk(request); if (!result.IsValid) { foreach (var item in result.ItemsWithErrors) { Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error); } Console.WriteLine(result.ConnectionStatus.OriginalException.Message); //Console.Read(); Environment.Exit(1); } } } catch(Exception e) { Console.WriteLine(e.Message); } packages = LogReaderInst.MSITPackagesWrapper(bulkSize); } } else if (mapping.Equals("feed")) { List<FeedPackage> packages = LogReaderInst.FeedPackagesWrapper(bulkSize); while (packages != null && packages.Count >= 1) { //Index LineCount += packages.Count; Console.WriteLine("Current total lines: {0}", LineCount); try { if(updateFlag.Equals("F") || updateFlag.Equals("f") || updateFlag.Equals("T") || updateFlag.Equals("t")) { var result = Client.IndexMany<FeedPackage>(packages, indexName); if (!result.IsValid) { foreach (var item in result.ItemsWithErrors) { Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error); } Console.WriteLine(result.ConnectionStatus.OriginalException.Message); //Console.Read(); Environment.Exit(1); } } else { /* for (int i = 0; i < packages.Count; ++i) { FeedPackage p = packages[i]; Client.Update<FeedPackage, object>(u => u .IdFrom(p) .Doc(new { sequence_number = p.sequence_number }) .DocAsUpsert() ); } */ List<IBulkOperation> bulkOpt = new List<IBulkOperation>(bulkSize); for (int i = 0; i < packages.Count; ++i) { FeedPackage p = packages[i]; /* Client.Update<MSITPackage, object>(u => u .IdFrom(p) .Doc(new {CustomData = p.CustomData, VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category }) .DocAsUpsert() ); */ BulkUpdateOperation<FeedPackage, object> bo = new BulkUpdateOperation<FeedPackage, object>(p, new { sequence_number = p.sequence_number }); bulkOpt.Add(bo); } var request = new BulkRequest() { Refresh = true, Consistency = Elasticsearch.Net.Consistency.One, Operations = bulkOpt }; var result = Client.Bulk(request); if (!result.IsValid) { foreach (var item in result.ItemsWithErrors) { Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error); } Console.WriteLine(result.ConnectionStatus.OriginalException.Message); //Console.Read(); Environment.Exit(1); } } } catch(Exception e) { Console.WriteLine(e.Message); } packages = LogReaderInst.FeedPackagesWrapper(bulkSize); } } sw.Stop(); Console.WriteLine("Time for indexing: {0}", sw.Elapsed); }