public Artist[] Init(TestEnvironment environment)
{
if (_artists != null && _artists.Any())
{
return(_artists);
}
IntegrationTestsRuntime.EnsureCleanEnvironment();
_artists = ClientTestData.Artists.CreateArtists(10);
using (var client = IntegrationTestsRuntime.CreateDbClient())
{
var bulk = new BulkRequest();
bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var artist = _artists.Single(i => i.ArtistId == row.Id);
client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Wait();
var queryRequests = ClientTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter)));
var queries = queryRequests.Select(q => client.Views.QueryAsync(q) as Task).ToArray();
Task.WaitAll(queries);
}
return(_artists);
}
/// <summary>
/// 推送ES数据列表
/// </summary>
/// <typeparam name="T">数据模型</typeparam>
/// <param name="index">索引名称 [db]</param>
/// <param name="type">索引类型 [table]</param>
/// <param name="primarykeyName">推送的主键名称</param>
/// <param name="data">推送数据</param>
/// <returns></returns>
public static bool Pulk <T>(string index, string type, string primarykeyName, List <T> data) where T : class
{
var bulkRequest = new BulkRequest(index, type)
{
Operations = new List <IBulkOperation>()
};
var indexs = data.Select(p =>
new BulkIndexOperation <T>(p)
{
Id = p.GetType().GetProperty(primarykeyName).GetValue(p, null).ToString()
})
.Cast <IBulkOperation>()
.ToList();
bulkRequest.Operations = indexs;
var response = client.Bulk(bulkRequest);
if (response.OriginalException != null)
{
throw new ArgumentException(response.OriginalException.Message, response.OriginalException);
}
if (DebuggerMode && response.ApiCall != null)
{
var url = response.ApiCall.Uri.AbsoluteUri;
var query = System.Text.Encoding.UTF8.GetString(response.ApiCall.RequestBodyInBytes);
var responseText = System.Text.Encoding.UTF8.GetString(response.ApiCall.ResponseBodyInBytes);
Debug.WriteLine(string.Format("url：{0}\t\nquery：{1}\t\n response：{2}", url, query, responseText));
}
return(response.IsValid);
}
public async Task BulkRequest_SerialisationTest()
{
var operations = new BulkOperationsCollection
{
new BulkIndexOperation <Project>(FixedProject)
{
Index = "project"
},
new BulkIndexOperation <Project>(FixedProject)
};
var request = new BulkRequest
{
Operations = operations
};
var serialisedJson = await SerializeAndGetJsonStringAsync(request);
var sr = new StringReader(serialisedJson);
var count = 0;
while (true)
{
var line = sr.ReadLine();
if (line is not null)
{
await Verifier.VerifyJson(line).UseMethodName($"{nameof(BulkRequest_SerialisationTest)}_{++count}");
}
else
{
break;
}
}
}
public SearchFixture()
{
Animals = CloudantTestData.Animals.CreateAll();
Client = IntegrationTestsRuntime.CreateCloudantClient();
var bulk = new BulkRequest();
bulk.Include(Animals.Select(i => Client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = Client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var animal = Animals.Single(i => i.AnimalId == row.Id);
Client.Entities.Reflector.RevMember.SetValueTo(animal, row.Rev);
}
Client.Documents.PostAsync(CloudantTestData.Views.Views101).Wait();
var queries = CloudantTestData.Views.AllViewIds.Select(id => new QueryViewRequest(id).Configure(q => q.Stale(Stale.UpdateAfter)));
foreach (var query in queries)
{
Client.Views.QueryAsync(query).Wait();
}
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
BulkRequest request = (BulkRequest)value;
if (request == null)
{
return;
}
foreach (var indexItem in request.IndexItems)
{
writer.WriteStartObject();
writer.WritePropertyName(indexItem.ActionType);
writer.WriteStartObject();
writer.WritePropertyName("_index");
writer.WriteValue(indexItem.Index);
writer.WritePropertyName("_type");
writer.WriteValue(indexItem.Type);
writer.WritePropertyName("_id");
writer.WriteValue(indexItem.DocuemntId);
writer.WriteEndObject();
writer.WriteEndObject();
writer.WriteRawValue("\n");
if (indexItem.ActionType != "delete")
{
serializer.Serialize(writer, indexItem.DocumentEntity);
writer.WriteRawValue("\n");
}
}
}
public bool BulkIndex <T>(IEnumerable <T> inputObjectList, string indexName, string typeName, bool isAsync = false) where T : class
{
try
{
if (!string.IsNullOrEmpty(indexName))
{
indexName = indexName.ToLower();
}
if (!string.IsNullOrEmpty(typeName))
{
typeName = typeName.ToLower();
}
var operations = inputObjectList.Select(item => new BulkIndexOperation <T>(item)).Cast <IBulkOperation>().ToList();
var request = new BulkRequest(indexName, typeName)
{
Refresh = true,
Operations = operations
};
if (isAsync)
{
var res = _client.BulkAsync(request);
}
else
{
var res = _client.Bulk(request);
}
return(true);
}
catch (Exception ex)
{
throw;
}
}
public async Task SyncData()
{
//1290213 = 13x100000
for (int i = 1; i < 14; i++)
{
var recepies = await _dbContext
.Recipes.Where(r => r.Id <= 100000 *i && r.Id > 100000 *(i - 1))
.Include(r => r.Ingredients)
.Include(r => r.RecipeTag)
.ThenInclude(rt => rt.Tag)
.ToListAsync();
var recepiesEsModel = _mapper.Map <List <RecipeSearchModel> >(recepies);
var request = new BulkRequest(indexName)
{
Operations = new List <IBulkOperation>()
};
var bulkOperations = recepiesEsModel.Select(r => new BulkIndexOperation <RecipeSearchModel>(r));
request.Operations.AddRange(bulkOperations);
await _elasticClient.BulkAsync(request);
}
Console.WriteLine("Sync completed");
}
/// <summary>
/// Res the index.
/// </summary>
/// <returns>Response data for a bulk insert of items</returns>
/// <exception cref="System.Exception">Cound not delete log to start re-index - " + deleteResponse.DebugInformation</exception>
public IBulkResponse ReIndex()
{
if (this.elasticClient.IndexExists(this.indexTablename).Exists)
{
var response = this.elasticClient.DeleteIndex(this.indexTablename);
if (!response.IsValid)
{
throw new Exception("Cound not delete log to start re-index - " + response.DebugInformation);
}
}
this.VerifyIndexStatus();
// this will get all records in the table. Needless to say this is not verify effective so we don't want to do this very often.
var results = this.dynamoDbContext.ScanAsync <TEntity>(null);
var bulkRequest = new BulkRequest(this.indexTablename)
{
Operations = new List <IBulkOperation>()
};
while (!results.IsDone)
{
var batch = results.GetNextSetAsync().Result;
foreach (var item in batch)
{
bulkRequest.Operations.Add(new BulkIndexOperation <TEntity>(item));
}
}
return(this.elasticClient.Bulk(bulkRequest));
}
private void InsertIntoCouchDB(string databaseName, string tableName, List <DatabaseRecord> rows, RowTransformer rowTransformer)
{
const int chunkSize = 1000;
using (var client = new MyCouchClient(CouchDbUrl, databaseName))
{
Parallel.ForEach(rows.Chunk(chunkSize), new ParallelOptions {
MaxDegreeOfParallelism = 20
}, chunkEnumerable =>
{
var chunk = chunkEnumerable.ToArray();
IEnumerable <DatabaseRecord> transformedChunk;
if (rowTransformer == null)
{
transformedChunk = chunk;
}
else
{
transformedChunk = chunk.Select(row => rowTransformer(tableName, row));
}
var documents = transformedChunk.Select(JsonConvert.SerializeObject).ToArray();
var request = new BulkRequest();
request.Include(documents);
client.Documents.BulkAsync(request).Wait();
});
}
}
static void Main(string[] args)
{
var totalThreads = 4;
var bulkRequests = 500;
var node = new Uri("http://myserver:9200");
var token = cancelToken;
for (int i = 0; i < totalThreads; i++)
{
Task.Run(() => {
while (!token.IsCancellationRequested)
{
var settings = new ConnectionSettings(node);
var client = new ElasticClient(settings);
var bulkRequest = new BulkRequest()
{
Operations = GetOperations(bulkRequests)
};
var result = client.Bulk(bulkRequest);
Console.WriteLine("Thread {0} publushed {1} requests with a status of {2}", i, result.IsValid, bulkRequests);
}
}
, cancelToken);
}
}
public virtual async Task <DeleteManyResult> DeleteManyAsync(params DocumentHeader[] documents)
{
ThrowIfDisposed();
Ensure.That(documents, "documents").HasItems();
var request = new BulkRequest()
.Delete(documents);
var response = await Client.Documents.BulkAsync(request);
ThrowIfNotSuccessfulResponse(response);
return(new DeleteManyResult
{
Rows = response.Rows.Select(r => new DeleteManyResult.Row
{
Id = r.Id,
Rev = r.Rev,
Error = r.Error,
Reason = r.Reason,
Deleted = r.Succeeded
}).ToArray()
});
}
public async Task InsertRangeAsync <T>(IEnumerable <T> entity, string index) where T : class
{
var indexName = index.GetIndex <T>();
var exists = await IndexExistsAsync(indexName);
if (!exists)
{
await((ElasticClient)_elasticClient).CreateIndexAsync <T>(indexName);
await AddAliasAsync(indexName, typeof(T).Name);
}
var bulkRequest = new BulkRequest(indexName)
{
Operations = new List <IBulkOperation>()
};
var operations = entity.Select(o => new BulkIndexOperation <T>(o)).Cast <IBulkOperation>().ToList();
bulkRequest.Operations = operations;
var response = await _elasticClient.BulkAsync(bulkRequest);
if (!response.IsValid)
{
throw new Exception("批量新增数据失败:" + response.OriginalException.Message);
}
}
public BulkOperationsResponse Delete(IOpenSearchableElasticType type, IOpenSearchResultCollection results)
{
BulkRequest bulkRequest = new BulkRequest() {
Refresh = true,
Consistency = Consistency.One,
Index = type.Index,
Type = type.Type,
Operations = new List<IBulkOperation>()
};
foreach (var doc in results.Items) {
var bulkDeleteOperation = new BulkDeleteOperation<IElasticItem>(doc.Identifier);
bulkDeleteOperation.Type = type.Type.Name;
var bulkOp = bulkDeleteOperation;
bulkRequest.Operations.Add(bulkOp);
}
var response = client.Bulk(bulkRequest);
BulkOperationsResponse ingestionResponse = new BulkOperationsResponse();
foreach (var item in response.Items) {
if (!item.IsValid) {
ingestionResponse.Errors++;
} else {
ingestionResponse.Deleted++;
}
}
return ingestionResponse;
}
/// <summary>
/// Bulks the async.
/// </summary>
/// <returns>The async.</returns>
/// <param name="ideasJson">Ideas json.</param>
public async Task BulkAsync(List <string> ideasJson)
{
var request = new BulkRequest();
request.Include(ideasJson.ToArray());
var response = await Documents.BulkAsync(request);
}
public Dictionary <string, List <Entity> > GetTargetEntitiesForLookupMatch(IOrganizationService crmService, List <LookupMatchCriteria> lookupMatchCriterias)
{
Dictionary <string, List <Entity> > entityLookups = new Dictionary <string, List <Entity> >();
Dictionary <string, QueryExpression> queriesToBeProcessed = new Dictionary <string, QueryExpression>();
foreach (LookupMatchCriteria lookupMatchCriteria in lookupMatchCriterias)
{
QueryExpression queryForLookupMatching = GetQueryForLookupMatching(lookupMatchCriteria);
queriesToBeProcessed.Add(lookupMatchCriteria.EntityLogicalName, queryForLookupMatching);
}
BulkRequest bulkRequest = new BulkRequest(100);
BulkResponse bulkResponse = bulkRequest.Read(crmService, queriesToBeProcessed);
foreach (BulkResponseItem bulkResponseItem in bulkResponse.Responses)
{
if (bulkResponseItem.Results != null && bulkResponseItem.Results.Entities != null)
{
entityLookups.Add(bulkResponseItem.BulkRequestId, bulkResponseItem.Results.Entities.ToList());
}
}
return(entityLookups);
}
public IAsyncEnumerable <string> GetJsonSkinsByIds(
IReadOnlyCollection <int> itemIds,
IProgress <ICollectionContext>?progress = default,
CancellationToken cancellationToken = default
)
{
var producer = SplitQuery.Create <int, string>(
async(range, ct) =>
{
var request = new BulkRequest("/v2/skins")
{
Ids = range
};
var json = await request.SendAsync(http, ct);
return(json.Indent(false)
.RootElement.EnumerateArray()
.Select(
item => item.ToString()
?? throw new InvalidOperationException("Unexpected null in JSON array.")
)
.ToList());
},
progress
);
return(producer.QueryAsync(itemIds, cancellationToken: cancellationToken));
}
public Artist[] Init(TestEnvironment environment)
{
if (_artists != null && _artists.Any())
{
return(_artists);
}
IntegrationTestsRuntime.EnsureCleanEnvironment();
_artists = ClientTestData.Artists.CreateArtists(2);
using (var client = IntegrationTestsRuntime.CreateDbClient())
{
var bulk = new BulkRequest();
bulk.Include(_artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var artist = _artists.Single(i => i.ArtistId == row.Id);
client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
client.Documents.PostAsync(ClientTestData.Shows.ArtistsShows).Wait();
}
return(_artists);
}
public virtual async Task <DeleteManyResult> DeleteManyAsync(DocumentHeader[] documents, CancellationToken cancellationToken = default)
{
ThrowIfDisposed();
EnsureArg.HasItems(documents, nameof(documents));
var request = new BulkRequest()
.Delete(documents);
var response = await Client.Documents.BulkAsync(request, cancellationToken);
ThrowIfNotSuccessfulResponse(response);
return(new DeleteManyResult
{
Rows = response.Rows.Select(r => new DeleteManyResult.Row
{
Id = r.Id,
Rev = r.Rev,
Error = r.Error,
Reason = r.Reason,
Deleted = r.Succeeded
}).ToArray()
});
}
public BulkRequestTests()
{
var newProject = new ElasticsearchProject {
Id = 4, Name = "new-project"
};
var request = new BulkRequest()
{
Refresh = true,
Consistency = Consistency.One,
Operations = new List <IBulkOperation>
{
{ new BulkIndexOperation <ElasticsearchProject>(newProject)
{
Id = "2"
} },
{ new BulkDeleteOperation <ElasticsearchProject>(6) },
{ new BulkCreateOperation <ElasticsearchProject>(newProject)
{
Id = "6"
} },
{ new BulkUpdateOperation <ElasticsearchProject, object>(newProject, new { name = "new-project2" })
{
Id = "3"
} },
}
};
var response = this._client.Bulk(request);
this._status = response.ConnectionStatus;
}
public void Delete()
{
using (var client = new MyCouchClient(_databaseUri, _databaseName))
{
QueryViewRequest request = new QueryViewRequest(
"bookorders", "allOrders");
request.Configure(parameters => parameters.IncludeDocs(true));
ViewQueryResponse <string> results =
client.Views.QueryAsync <string>(request).Result;
var bulkRequest = new BulkRequest()
{
AllOrNothing = false
};
foreach (var resultsRow in results.Rows)
{
BookOrderDto dto = JsonConvert.DeserializeObject <BookOrderDto>(
resultsRow.IncludedDoc);
bulkRequest.Delete(dto._id, dto._rev);
Console.WriteLine($"Delete request for id:{dto._id}");
}
client.Documents.BulkAsync(bulkRequest).Wait();
}
}
protected virtual string GenerateRequestBody(BulkRequest request)
{
var sb = new StringBuilder();
var documents = request.GetDocuments();
sb.Append("{");
if (request.AllOrNothing)
{
sb.Append("\"all_or_nothing\":true,");
}
if (!request.NewEdits)
{
sb.Append("\"new_edits\":false,");
}
sb.Append("\"docs\":[");
for (var i = 0; i < documents.Length; i++)
{
sb.Append(documents[i]);
if (i < documents.Length - 1)
{
sb.Append(",");
}
}
sb.Append("]}");
return(sb.ToString());
}
public int BulkIndex(IEnumerable <GeoEntity> entities)
{
var client = ElasticClientFactory.CreateClient(_elasticSettings);
CreateIndex(client);
var errorCount = 0;
var batch = 0;
foreach (var block in entities.Batch(_elasticSettings.BatchSize))
{
var bulkRequest = new BulkRequest()
{
Operations = new List <IBulkOperation>()
};
foreach (var item in block)
{
bulkRequest.Operations.Add(new BulkIndexOperation <GeoEntity>(item));
}
batch++;
var result = client.Bulk(bulkRequest);
var errors = result.ItemsWithErrors.Count();
if (errors > 0)
{
_logger.LogWarning("{}");
}
errorCount += errors;
}
return(errorCount);
}
private bool DeleteEsData(IEnumerable <UserMessageBoxInfoRelation> relations)
{
if (relations != null && relations.Any())
{
var operations = new List <IBulkOperation>();
var exlogs = ConvertMessageLogToEsLog(relations);
foreach (var log in exlogs)
{
operations.Add(new BulkDeleteOperation <UserMessageBoxInfoRelationC>(log.Id)
{
Routing = log.UserId
});
}
var bulkRequest = new BulkRequest(UserMessageBoxRelationIndex)
{
Operations = operations.ToArray()
};
var client = ElasticsearchHelper.CreateClient();
#if DEBUG
ElasticsearchHelper.EnableDebug();
#endif
var responseOne = client.Bulk(bulkRequest);
var errorcount = responseOne.ItemsWithErrors.Count();
DalMessageBox.WriteSyncLogs(relations);
return(errorcount <= 0);
}
return(false);
}
public ViewsFixture()
{
Artists = ClientTestData.Artists.CreateArtists(10);
using (var client = IntegrationTestsRuntime.CreateNormalClient())
{
var bulk = new BulkRequest();
bulk.Include(Artists.Select(i => client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = client.Documents.BulkAsync(bulk).Result;
foreach (var row in bulkResponse.Rows)
{
var artist = Artists.Single(i => i.ArtistId == row.Id);
client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
var tmp = client.Documents.PostAsync(ClientTestData.Views.ArtistsViews).Result;
var touchView1 = new QueryViewRequest(ClientTestData.Views.ArtistsAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter));
var touchView2 = new QueryViewRequest(ClientTestData.Views.ArtistsNameNoValueViewId).Configure(q => q.Stale(Stale.UpdateAfter));
var touchView3 = new QueryViewRequest(ClientTestData.Views.ArtistsTotalNumOfAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter));
client.Views.QueryAsync(touchView1).Wait();
client.Views.QueryAsync(touchView2).Wait();
client.Views.QueryAsync(touchView3).Wait();
}
}
private static BulkRequest CreateIndexBulkRequest <T>(IEnumerable <T> objects, Func <T, string> getIndex, string type, Func <T, string> getParent) where T : class
{
var bulkRequest = new BulkRequest();
TypeNameMarker typeNameMarker = type;
bulkRequest.Type = typeNameMarker;
var list = objects.Select(o => {
var doc = new BulkIndexOperation <T>(o);
if (getParent != null)
{
doc.Parent = getParent(o);
}
if (getIndex != null)
{
doc.Index = getIndex(o);
}
var versionedDoc = o as IVersioned;
if (versionedDoc != null)
{
doc.Version = versionedDoc.Version.ToString();
}
return(doc);
}).Cast <IBulkOperation>().ToList();
bulkRequest.Operations = list;
return(bulkRequest);
}
public async Task <BulkResponse> BulkDocumentsAsync(BulkRequest request)
{
string url = request.GetUrl(_clusterUrl);
var response = await SendAsync(url, HttpMethod.Post, request.GetBody());
return(new BulkResponse(response));
}
/// <summary>
/// 批量保存
/// </summary>
/// <typeparam name="T">实体类型</typeparam>
/// <param name="indexName">索引名</param>
/// <param name="entities">实体列表</param>
public async Task BulkSaveAsync <T>(string indexName, IEnumerable <T> entities) where T : class
{
var client = await _builder.GetClientAsync();
if (!await ExistsAsync(indexName))
{
await client.InitializeIndexMapAsync <T>(indexName);
}
var bulk = new BulkRequest(indexName)
{
Operations = new List <IBulkOperation>()
};
foreach (var entity in entities)
{
bulk.Operations.Add(new BulkIndexOperation <T>(entity));
}
var response = await client.BulkAsync(bulk);
if (response.Errors)
{
throw new ElasticsearchException($"批量保存文档在索引 {indexName} 失败：{response.ServerError.Error.Reason}");
}
}
/// <summary>
/// Attempts a forced replace or insert.
/// If conflicts are detected, they are dropped, and the local version is again updated.
/// Returns null if the method should be invoked again, otherwise the final version retrieved from the DB
/// </summary>
/// <typeparam name="T">Entity type</typeparam>
/// <param name="store">DB to place the entity in</param>
/// <param name="item">Entity to place. The _rev member may be updated in the process</param>
/// <returns></returns>
public static async Task <T> TryForceReplace <T>(DataBase store, T item) where T : Entity
{
//try
{
if (store == null)
{
return(item); //during tests, db is not loaded
}
var serial = JsonConvert.SerializeObject(item,
Newtonsoft.Json.Formatting.None,
new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
});
//store.Entities.Serializer.Serialize(item);
//JsonConvert.SerializeObject(item);
var header = await store.Documents.PutAsync(item._id, serial);
//await store.Entities.PutAsync(item._id, item);
if (!header.IsSuccess /*&& header.StatusCode == System.Net.HttpStatusCode.Conflict*/)
{
var head = await store.Documents.HeadAsync(item._id);
if (head.IsSuccess)
{
item._rev = head.Rev;
}
return(null);
}
var echoRequest = new GetEntityRequest(header.Id);
echoRequest.Conflicts = true;
var echo = await store.Documents.GetAsync(header.Id);
if (!echo.IsSuccess)
{
return(null);
}
if (echo.Conflicts != null && echo.Conflicts.Length > 0)
{
//we do have conflicts and don't quite know whether the new version is our version
var deleteHeaders = new DocumentHeader[echo.Conflicts.Length];
for (int i = 0; i < echo.Conflicts.Length; i++)
{
deleteHeaders[i] = new DocumentHeader(header.Id, echo.Conflicts[i]);
}
BulkRequest breq = new BulkRequest().Delete(deleteHeaders); //delete all conflicts
await store.Documents.BulkAsync(breq); //ignore result, but wait for it
item._rev = echo.Rev; //replace again. we must win
return(await TryForceReplace(store, item));
}
return(JsonConvert.DeserializeObject <T>(echo.Content)); //all good
}
//catch (Exception ex)
//{
// throw;
//}
}
public virtual HttpRequest Create(BulkRequest request)
{
Ensure.Any.IsNotNull(request, nameof(request));
return(new HttpRequest(HttpMethod.Post, GenerateRelativeUrl(request))
.SetRequestTypeHeader(request.GetType())
.SetJsonContent(GenerateRequestBody(request)));
}
public virtual HttpRequest Create(BulkRequest request)
{
var createHttpRequest = CreateFor <BulkRequest>(HttpMethod.Post, GenerateRequestUrl(request));
createHttpRequest.SetContent(request.ToJson());
return(createHttpRequest);
}
public virtual HttpRequest Create(BulkRequest request)
{
var createHttpRequest = CreateFor<BulkRequest>(HttpMethod.Post, GenerateRequestUrl(request));
createHttpRequest.SetContent(request.ToJson());
return createHttpRequest;
}
private static async Task <T> GetConflictResolvedAsync(DataBase store, string id, Func <ICollection <T>, T> merger, CancellationToken cancellationToken = default(CancellationToken))
{
var req = new GetDocumentRequest(id);
req.Conflicts = true;
var data = await store.Documents.GetAsync(req, cancellationToken);
//await store.Entities.GetAsync<T>(req, cancellationToken);
if (!data.IsSuccess)
{
return(null);
}
var mainContent = JsonConvert.DeserializeObject <T>(data.Content);
if (data.Conflicts == null || data.Conflicts.Length == 0)
{
return(mainContent);
}
List <T> conflicting = new List <T>();
List <DocumentHeader> toDelete = new List <DocumentHeader>();
conflicting.Add(mainContent);
foreach (var fetch in data.Conflicts)
{
var item = await store.Documents.GetAsync(id, fetch, cancellationToken);
if (item.IsSuccess)
{
conflicting.Add(JsonConvert.DeserializeObject <T>(item.Content));
toDelete.Add(new DocumentHeader(item.Id, item.Rev));
}
else
{
return(null); //assume connection lost or competing merge
}
}
T merged = merger(conflicting);
if (merged == null)
{
Log.Error("Error trying to fetch conflicting data from " + store + ":" + id + " with no merger set. Fixed. Result will be arbitrary");
merged = mainContent;
}
merged._id = id;
merged._rev = data.Rev;
var putResult = await store.Entities.PutAsync(merged);
if (putResult.IsSuccess)
{
var deleteRequest = new BulkRequest().Delete(toDelete.ToArray());
deleteRequest.AllOrNothing = true;
await store.Documents.BulkAsync(deleteRequest); //await, ignore result, let others clean it up if failed
return(putResult.Content);
}
return(null); //assume connection lost or asynchronous update
}
public BulkRequestTests()
{
var newProject = new ElasticsearchProject { Id = 4, Name = "new-project" };
var request = new BulkRequest()
{
Refresh = true,
Consistency = Consistency.One,
Operations = new List<IBulkOperation>
{
{ new BulkIndexOperation<ElasticsearchProject>(newProject) { Id= "2"}},
{ new BulkDeleteOperation<ElasticsearchProject>(6) },
{ new BulkCreateOperation<ElasticsearchProject>(newProject) { Id = "6" } },
{ new BulkUpdateOperation<ElasticsearchProject, object>(newProject, new { name = "new-project2"}) { Id = "3" } },
}
};
var response = this._client.Bulk(request);
this._status = response.ConnectionStatus;
}
//
// Send the messages to Elasticsearch (bulk)
//
private DateTime transmitBulkData(IEnumerable<JObject> bulkItems, string bulkIndexName, string bulkTypeName,
ElasticClient client, DateTime lastFlushTime, List<JObject> messages)
{
var bulkRequest = new BulkRequest() {Refresh = true};
bulkRequest.Operations = new List<IBulkOperation>();
foreach (var json in bulkItems)
{
// ES requires a timestamp, add one if not present
var ts = json["@timestamp"];
if (ts == null)
json["@timestamp"] = DateTime.UtcNow;
var bi = new BulkIndexOperation<JObject>(json);
bi.Index = bulkIndexName;
bi.Type = bulkTypeName;
bulkRequest.Operations.Add(bi);
}
// The total messages processed for this operation.
int numMessages = bulkItems.Count();
var response = client.Bulk(bulkRequest);
if (!response.IsValid)
{
LogManager.GetCurrentClassLogger().Error("Failed to send: {0}", response);
Interlocked.Increment(ref _errorCount);
interlockedInsert(messages); // Put the messages back into the queue
}
else // Success!
{
lastFlushTime = DateTime.UtcNow;
LogManager.GetCurrentClassLogger()
.Info("Successfully sent {0} messages in a single bulk request", numMessages);
Interlocked.Add(ref _sentMessages, numMessages);
}
// Remove them from the working list
messages.RemoveRange(0, numMessages);
return lastFlushTime;
}
protected virtual string GenerateRequestUrl(BulkRequest request)
{
return string.Format("{0}/_bulk_docs", Connection.Address);
}
/// <summary>
/// (Re)Indexes the complete page tree for pagedata that inherits from <see cref="IIndexablePageData"/>
/// </summary>
/// <param name="swapWithErrors">if set to <c>true</c> and if an error occured during the index process the new index is swapped to live</param>
/// <param name="onStatusChanges">the action that is executed on status changed during the index process</param>
/// <returns></returns>
public IEnumerable<IBulkResponse> IndexPageTree(bool swapWithErrors = false, Action<string> onStatusChanges = null)
{
foreach (var languageBranch in LanguageBranchRepository.ListEnabled())
{
var language = languageBranch.Culture;
InitializeIndex(language);
var aliasName = GetAliasName(language);
var indexName1 = GetIndexName(language, 1);
var indexName2 = GetIndexName(language, 2);
var aliasIndices = ElasticClient.GetAlias(x => x.Name(aliasName)).Indices;
var liveIndexName = aliasIndices.First().Key;
var reIndexName = liveIndexName.Equals(indexName1) ? indexName2 : indexName1;
// clear the reIndex indice since we are re-building it
Clear(reIndexName, language);
var indexablePages = GetIndexablePages(language).ToArray();
var bulkOperations = new List<IBulkOperation>();
bool hasErrors = false;
for (var i = 0; i < indexablePages.Length; i++)
{
var indexablePageData = indexablePages[i];
if (!indexablePageData.ShouldIndex())
continue;
var indexModel = CreateIndexModel(indexablePageData);
bulkOperations.Add(CreateBulkOperation(indexModel));
if (bulkOperations.Count == Options.BulkSize || i == indexablePages.Length - 1)
{
var bulkRequest = new BulkRequest(reIndexName) {Operations = new List<IBulkOperation>(bulkOperations)};
var bulkResponse = ElasticClient.Bulk(bulkRequest);
if (!bulkResponse.IsValid || bulkResponse.Errors)
hasErrors = true;
bulkOperations.Clear();
yield return bulkResponse;
}
onStatusChanges?.Invoke($"Insterted bulk into {reIndexName} currentIndex: {i}");
}
if (!hasErrors || swapWithErrors)
SwapIndex(liveIndexName, aliasName, reIndexName);
}
}
private async Task SendEventsAsync(IEnumerable<EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (events == null)
{
return;
}
try
{
string currentIndexName = this.GetIndexName(this.connectionData);
if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal))
{
await this.EnsureIndexExists(currentIndexName, this.connectionData.Client);
this.connectionData.LastIndexName = currentIndexName;
}
BulkRequest request = new BulkRequest();
request.Refresh = true;
List<IBulkOperation> operations = new List<IBulkOperation>();
foreach (EventData eventData in events)
{
BulkCreateOperation<EventData> operation = new BulkCreateOperation<EventData>(eventData);
operation.Index = currentIndexName;
operation.Type = EventDocumentTypeName;
operations.Add(operation);
}
request.Operations = operations;
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance
// between different SendEventsAsync callbacks.
// Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3
IBulkResponse response = await this.connectionData.Client.BulkAsync(request);
if (!response.IsValid)
{
this.ReportEsRequestError(response, "Bulk upload");
}
this.ReportListenerHealthy();
}
catch (Exception e)
{
this.ReportListenerProblem("Diagnostics data upload has failed." + Environment.NewLine + e.ToString());
}
}
public BulkOperationsResponse Ingest(IOpenSearchableElasticType type, IOpenSearchResultCollection results)
{
OpenSearchFactory.RemoveLinksByRel(ref results, "self");
OpenSearchFactory.RemoveLinksByRel(ref results, "search");
IElasticCollection docs = type.FromOpenSearchResultCollection(results);
BulkRequest bulkRequest = new BulkRequest() {
Refresh = true,
Consistency = Consistency.One,
Index = type.Index,
Type = type.Type,
Operations = new List<IBulkOperation>()
};
RootObjectMapping currentMapping = null;
try {
var mappingResponse = client.GetMapping<IElasticType>(g => g.Index(type.Index.Name).Type(type.Type.Name));
currentMapping = mappingResponse.Mapping;
} catch (Exception) {
}
var rootObjectMapping = type.GetRootMapping();
if (!rootObjectMapping.Equals(currentMapping)) {
client.Map<IElasticType>(m => m.Index(type.Index.Name).Type(type.Type.Name));
}
foreach (var doc in docs.ElasticItems) {
var bulkIndexOperation = new BulkIndexOperation<IElasticItem>(doc);
bulkIndexOperation.Id = ((IOpenSearchResultItem)doc).Identifier;
bulkIndexOperation.Type = type.Type.Name;
var bulkOp = bulkIndexOperation;
bulkRequest.Operations.Add(bulkOp);
}
var response = client.Bulk(bulkRequest);
BulkOperationsResponse ingestionResponse = new BulkOperationsResponse();
foreach (var item in response.Items) {
if (!item.IsValid) {
ingestionResponse.Errors++;
continue;
}
if (item.Version == "1")
ingestionResponse.Added++;
else
ingestionResponse.Updated++;
}
return ingestionResponse;
}
public object Post(TypeImportRequest request)
{
OpenSearchEngine ose = new OpenSearchEngine();
ose.LoadPlugins();
ose.DefaultTimeOut = 60000;
OpenSearchUrl url = new OpenSearchUrl(request.url);
IOpenSearchable entity = new GenericOpenSearchable(url, ose);
IOpenSearchableElasticType type = ecf.GetOpenSearchableElasticTypeByNameOrDefault(request.IndexName, request.TypeName);
IOpenSearchResultCollection osres = ose.Query(entity, new NameValueCollection());
OpenSearchFactory.RemoveLinksByRel(ref osres, "alternate");
OpenSearchFactory.RemoveLinksByRel(ref osres, "via");
OpenSearchFactory.RemoveLinksByRel(ref osres, "self");
OpenSearchFactory.RemoveLinksByRel(ref osres, "search");
IElasticCollection documents = type.FromOpenSearchResultCollection(osres);
BulkRequest bulkRequest = new BulkRequest() {
Refresh = true,
Consistency = Consistency.One,
Index = request.IndexName
};
foreach (var doc in documents.Items) {
bulkRequest.Operations.Add(new BulkIndexOperation<IElasticItem>((IElasticItem)doc) { Id = doc.Id });
}
var response = client.Bulk(bulkRequest);
return response;
}
public void index()
{
LogReader LogReaderInst = new LogReader(fileName);
Stopwatch sw = new Stopwatch();
long LineCount = 0;
sw.Start();
if (mapping.Equals("msit"))
{
List<MSITPackage> packages = LogReaderInst.MSITPackagesWrapper(bulkSize);
while (packages != null && packages.Count >= 1)
{
//Index
LineCount += packages.Count;
Console.WriteLine("Current total lines: {0}", LineCount);
try
{
if(updateFlag.Equals("F") || updateFlag.Equals("f") || updateFlag.Equals("T") || updateFlag.Equals("t")) {//Indexing
var result = Client.IndexMany<MSITPackage>(packages, indexName);
if (!result.IsValid)
{
foreach (var item in result.ItemsWithErrors)
{
Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error);
}
Console.WriteLine(result.ConnectionStatus.OriginalException.Message);
//Console.Read();
Environment.Exit(1);
}
}
else//Updating
{
List<IBulkOperation> bulkOpt = new List<IBulkOperation>(bulkSize);
for(int i = 0; i < packages.Count; ++i)
{
MSITPackage p = packages[i];
/*
Client.Update<MSITPackage, object>(u => u
.IdFrom(p)
.Doc(new {CustomData = p.CustomData, VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category })
.DocAsUpsert()
);
*/
BulkUpdateOperation<MSITPackage, object> bo = new BulkUpdateOperation<MSITPackage, object>(p, new { CustomData = "123435CustomData", VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category }, false);
bo.RetriesOnConflict = 3;
//BulkUpdateOperation<MSITPackage, object> bo = new BulkUpdateOperation<MSITPackage, object>(p, new { CustomData = p.CustomData, VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category });
bulkOpt.Add(bo);
}
var request = new BulkRequest()
{
Refresh = true,
Consistency = Elasticsearch.Net.Consistency.One,
Operations = bulkOpt
};
//Console.WriteLine("Bulk indexing");
var result = Client.Bulk(request);
if (!result.IsValid)
{
foreach (var item in result.ItemsWithErrors)
{
Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error);
}
Console.WriteLine(result.ConnectionStatus.OriginalException.Message);
//Console.Read();
Environment.Exit(1);
}
}
}
catch(Exception e)
{
Console.WriteLine(e.Message);
}
packages = LogReaderInst.MSITPackagesWrapper(bulkSize);
}
}
else if (mapping.Equals("feed"))
{
List<FeedPackage> packages = LogReaderInst.FeedPackagesWrapper(bulkSize);
while (packages != null && packages.Count >= 1)
{
//Index
LineCount += packages.Count;
Console.WriteLine("Current total lines: {0}", LineCount);
try
{
if(updateFlag.Equals("F") || updateFlag.Equals("f") || updateFlag.Equals("T") || updateFlag.Equals("t"))
{
var result = Client.IndexMany<FeedPackage>(packages, indexName);
if (!result.IsValid)
{
foreach (var item in result.ItemsWithErrors)
{
Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error);
}
Console.WriteLine(result.ConnectionStatus.OriginalException.Message);
//Console.Read();
Environment.Exit(1);
}
}
else
{
/*
for (int i = 0; i < packages.Count; ++i)
{
FeedPackage p = packages[i];
Client.Update<FeedPackage, object>(u => u
.IdFrom(p)
.Doc(new { sequence_number = p.sequence_number })
.DocAsUpsert()
);
}
*/
List<IBulkOperation> bulkOpt = new List<IBulkOperation>(bulkSize);
for (int i = 0; i < packages.Count; ++i)
{
FeedPackage p = packages[i];
/*
Client.Update<MSITPackage, object>(u => u
.IdFrom(p)
.Doc(new {CustomData = p.CustomData, VerdictCF = p.VerdictCF, OriginalIP = p.OriginalIP, Category = p.Category })
.DocAsUpsert()
);
*/
BulkUpdateOperation<FeedPackage, object> bo = new BulkUpdateOperation<FeedPackage, object>(p, new { sequence_number = p.sequence_number });
bulkOpt.Add(bo);
}
var request = new BulkRequest()
{
Refresh = true,
Consistency = Elasticsearch.Net.Consistency.One,
Operations = bulkOpt
};
var result = Client.Bulk(request);
if (!result.IsValid)
{
foreach (var item in result.ItemsWithErrors)
{
Console.WriteLine("Failed to index document {0}:{1}", item.Id, item.Error);
}
Console.WriteLine(result.ConnectionStatus.OriginalException.Message);
//Console.Read();
Environment.Exit(1);
}
}
}
catch(Exception e)
{
Console.WriteLine(e.Message);
}
packages = LogReaderInst.FeedPackagesWrapper(bulkSize);
}
}
sw.Stop();
Console.WriteLine("Time for indexing: {0}", sw.Elapsed);
}