public static void Setup()
{
var client = ElasticsearchConfiguration.Client;
var projects = NestTestData.Data;
var people = NestTestData.People;
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping<Person>(m => m.MapFromAttributes())
);
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping<Person>(m => m.MapFromAttributes())
);
var bulk = new BulkDescriptor();
foreach (var p in projects)
bulk.Index<ElasticSearchProject>(i=>i.Object(p));
foreach (var p in people)
bulk.Index<Person>(i => i.Object(p));
client.Bulk(bulk);
client.Refresh(new[] {ElasticsearchConfiguration.DefaultIndex, ElasticsearchConfiguration.DefaultIndex + "_clone"});
}
public static void Setup()
{
var client = ElasticsearchConfiguration.Client;
var projects = NestTestData.Data;
var people = NestTestData.People;
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping <Person>(m => m.MapFromAttributes())
);
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping <Person>(m => m.MapFromAttributes())
);
var bulk = new BulkDescriptor();
foreach (var p in projects)
{
bulk.Index <ElasticSearchProject>(i => i.Object(p));
}
foreach (var p in people)
{
bulk.Index <Person>(i => i.Object(p));
}
client.Bulk(bulk);
client.Refresh(new[] { ElasticsearchConfiguration.DefaultIndex, ElasticsearchConfiguration.DefaultIndex + "_clone" });
}
public async Task AddTestDataInLoop()
{
log.LogInformation("Inside AddTest data");
try
{
await Task.Delay(5000);
for (int i = 0; i < 4; i++)
{
IPingResponse x = await client.PingAsync();
if (x.ConnectionStatus.Success)
{
log.LogInformation(DateTime.Now.ToLongTimeString() + " Connection successful to - " + x.ConnectionStatus.RequestUrl);
break;
}
else
{
log.LogWarning(DateTime.Now.ToLongTimeString() + " Unable to connect to - " + x.ConnectionStatus.RequestUrl);
await Task.Delay(i * 1000);
}
}
var allProducts = new List<Product>();
var descriptor = new BulkDescriptor();
for (int i = 0; i < itemCount; i++)
{
Product p = new Product() { Id = i, Title = "test " + i, Price = i, CategoryId = 1 };
descriptor.Index<Product>(op => op.Document(p));
allProducts.Add(p);
}
for (int j = 0; j < 10; j++)
{
Category c = new Category()
{
Id = j,
Name = "Category " + j,
Products = allProducts.Where(p => p.CategoryId == j).ToList<Product>()
};
descriptor.Index<Category>(op => op.Document(c));
}
log.LogWarning("before bulk async");
var result = await client.BulkAsync(descriptor);
log.LogWarning("after bulk async");
}
catch (Exception ex)
{
log.LogError(DateTime.Now.ToLongTimeString() + " - Ex Caught:" + ex.Message);
}
}
public async Task AddTestDataInLoop()
{
log.LogInformation("Inside AddTest data");
try
{
await Task.Delay(5000);
for (int i = 0; i < 4; i++)
{
IPingResponse x = await client.PingAsync();
if (x.ApiCall.Success)
{
log.LogInformation(DateTime.Now.ToLocalTime() + " Connection successful to - " + x.ApiCall.Uri);
break;
}
else
{
log.LogWarning(DateTime.Now.ToLocalTime() + " Unable to connect to - " + x.ApiCall.Uri);
await Task.Delay(i * 1000);
}
}
var allProducts = new List <Product>();
var descriptor = new BulkDescriptor();
for (int i = 0; i < itemCount; i++)
{
Product p = new Product()
{
Id = i, Title = "test " + i, Price = i, CategoryId = 1
};
descriptor.Index <Product>(op => op.Document(p));
allProducts.Add(p);
}
for (int j = 0; j < 10; j++)
{
Category c = new Category()
{
Id = j,
Name = "Category " + j,
Products = allProducts.Where(p => p.CategoryId == j).ToList <Product>()
};
descriptor.Index <Category>(op => op.Document(c));
}
log.LogWarning("before bulk async");
var result = await client.BulkAsync(descriptor);
log.LogWarning("after bulk async");
}
catch (Exception ex)
{
log.LogError(DateTime.Now.ToLocalTime() + " - Ex Caught:" + ex.Message);
}
}
public void BulkIndexWithPercolate()
{
// register up some percolator queries to test matching
var query1 = "bulkindex-test-doc-1";
this._client.UnregisterPercolator(query1, ur => ur.Index <ElasticsearchProject>());
var perc = this._client.RegisterPercolator <ElasticsearchProject>(query1, p => p
.Query(q => q
.Term(f => f.Country, "netherlands")
)
);
this._client.Refresh(r => r.Index <ElasticsearchProject>());
var descriptor = new BulkDescriptor();
// match against any doc
descriptor.Index <ElasticsearchProject>(i => i
.Object(new ElasticsearchProject {
Id = 2, Country = "netherlands"
})
.Percolate("*") // match on any percolated docs
);
// no percolate requested this time
descriptor.Index <ElasticsearchProject>(i => i
.Object(new ElasticsearchProject {
Id = 3, Country = "netherlands"
})
);
var result = this._client.Bulk(d => descriptor);
result.Should().NotBeNull();
result.IsValid.Should().BeTrue();
result.Errors.Should().BeFalse();
result.Items.Should().NotBeNull().And.NotBeEmpty().And.HaveCount(2);
var indexResponses = result.Items.OfType <BulkIndexResponseItem>().ToList();
// tests on percolated responses
indexResponses.Should().HaveCount(2);
indexResponses.First().Id.Should().BeEquivalentTo("2");
indexResponses.First().Index.Should().BeEquivalentTo(ElasticsearchConfiguration.DefaultIndex);
indexResponses.First().Type.Should().BeEquivalentTo(this._client.Infer.TypeName <ElasticsearchProject>());
indexResponses.First().Matches.Should().NotBeNull();
indexResponses.ElementAt(1).Id.Should().BeEquivalentTo("3");
indexResponses.ElementAt(1).Index.Should().BeEquivalentTo(ElasticsearchConfiguration.DefaultIndex);
indexResponses.ElementAt(1).Type.Should().BeEquivalentTo(this._client.Infer.TypeName <ElasticsearchProject>());
indexResponses.ElementAt(1).Matches.Should().BeNull();
// cleanup
this._client.UnregisterPercolator(query1, ur => ur.Index <ElasticsearchProject>());
}
public void BulkIndexWithPercolate()
{
// register up some percolator queries to test matching
var query1 = "bulkindex-test-doc-1";
this._client.UnregisterPercolator<ElasticSearchProject>(query1);
var perc = this._client.RegisterPercolator<ElasticSearchProject>(p => p
.Name(query1)
.Query(q => q
.Term(f => f.Country, "netherlands")
)
);
this._client.Refresh<ElasticSearchProject>();
var descriptor = new BulkDescriptor();
// match against any doc
descriptor.Index<ElasticSearchProject>(i => i
.Object(new ElasticSearchProject { Id = 2, Country = "netherlands" })
.Percolate("*") // match on any percolated docs
);
// no percolate requested this time
descriptor.Index<ElasticSearchProject>(i => i
.Object(new ElasticSearchProject { Id = 3, Country = "netherlands" })
);
var result = this._client.Bulk(descriptor);
result.Should().NotBeNull();
result.IsValid.Should().BeTrue();
result.Items.Should().NotBeNull().And.NotBeEmpty().And.HaveCount(2).And.OnlyContain(r => r.OK);
var indexResponses = result.Items.OfType<BulkIndexResponseItem>().ToList();
// tests on percolated responses
indexResponses.Should().HaveCount(2);
indexResponses.First().Id.Should().BeEquivalentTo("2");
indexResponses.First().Index.Should().BeEquivalentTo(ElasticsearchConfiguration.DefaultIndex);
indexResponses.First().Type.Should().BeEquivalentTo(this._client.Infer.TypeName<ElasticSearchProject>());
indexResponses.First().Matches.Should().NotBeNull();
indexResponses.ElementAt(1).Id.Should().BeEquivalentTo("3");
indexResponses.ElementAt(1).Index.Should().BeEquivalentTo(ElasticsearchConfiguration.DefaultIndex);
indexResponses.ElementAt(1).Type.Should().BeEquivalentTo(this._client.Infer.TypeName<ElasticSearchProject>());
indexResponses.ElementAt(1).Matches.Should().BeNull();
// cleanup
this._client.UnregisterPercolator<ElasticSearchProject>(query1);
}
public bool BulkIndexDesc <T>(IEnumerable <T> inputObject, bool isAsync = false) where T : class
{
try
{
var descriptor = new BulkDescriptor();
foreach (T pageview in inputObject)
{
T item = pageview;
descriptor.Index <T>(op => op.Document(item));
}
if (isAsync)
{
_client.BulkAsync(descriptor);
}
else
{
_client.Bulk(descriptor);
}
return(true);
}
catch (Exception ex)
{
throw;
}
}
private void Push(ElasticClient client, IReadOnlyCollection <string> buffer)
{
Logger.Verbose("A buffer of {count} records is about to be sent for bulk indexing", buffer.Count);
var descriptor = new BulkDescriptor();
foreach (var line in buffer)
{
var instance = JObject.Parse(line);
var identifier = instance.Property("_id");
var id = identifier.Value.ToString();
identifier.Remove();
descriptor
.Index <JObject>(i => i
.Index("data")
.Type("geo")
.Id(id)
.Document(instance));
}
var response = client.Bulk(descriptor);
if (response.Errors)
{
Logger.Debug("Bulk insert failed: {errors}", response.ItemsWithErrors);
}
}
/// <summary>
/// Inserts the passed data into elastic search index Async Mode
/// </summary>
/// <param name="entities">The list of the entities that will be indexed</param>
/// <param name="refresh">Flag to refresh index state with new data</param>
public static async void BulkInsertAttachmentAsync <T>(List <T> entities, bool refresh = true) where T : EsAttachment
{
if (entities.Count > 0)
{
entities[0].CreateIndex <T>();
var indexName = entities[0].IndexAlias;
var esClient = Manager.EsClient;
var descriptor = new BulkDescriptor();
foreach (var doc in entities)
{
descriptor.Index <T>(i => i
.Index(indexName)
.Document(doc)
.Pipeline(doc.PipeLineName));
}
var response = await esClient.BulkAsync(descriptor);
if (!response.IsValid)
{
throw new Exception(response.OriginalException.Message);
}
if (refresh)
{
esClient.Indices.Refresh(indexName);
}
}
else
{
return;
}
}
/// <summary>
/// Inserts the passed data into elastic search index
/// </summary>
/// <param name="entities">The list of the entities that will be indexed</param>
/// <param name="refresh">Flag to refresh index state with new data</param>
public static void BulkInsert <T>(List <T> entities, bool refresh = true) where T : EsDocument
{
if (entities.Count > 0)
{
entities[0].CreateIndex <T>();
var indexName = entities[0].IndexAlias;
var esClient = Manager.EsClient;
var descriptor = new BulkDescriptor();
descriptor.TypeQueryString("_doc");
foreach (var doc in entities)
{
descriptor.Index <T>(i => i
.Index(indexName)
.Document(doc));
}
var response = esClient.Bulk(descriptor);
if (!response.IsValid)
{
throw new Exception(response.OriginalException.Message);
}
if (refresh)
{
esClient.Indices.Refresh(indexName);
}
}
else
{
return;
}
}
/// <summary>
/// Indexes the specified documents.
/// </summary>
/// <param name="documents">The documents.</param>
public override void Index(IEnumerable <Document> documents)
{
var descriptor = new BulkDescriptor();
_ = descriptor.Refresh(Refresh.WaitFor);
var docId = 0;
foreach (var document in documents)
{
foreach (var field in document.Fields)
{
var indexId = Connection.GetIndexId(field.Key);
dynamic updateDoc = new System.Dynamic.ExpandoObject();
updateDoc.value = field.Value;
_ = descriptor.Index <object>(op => op
.Index(indexId)
// TODO: docIds should come from document object, this is only for testing purposes
.Id(document.Id > 0 ? document.Id : docId++)
.Document(updateDoc)
);
}
}
var result = Connection.Client.Bulk(descriptor);
}
public void BulkUpdateObject()
{
//Lets first insert some documents with id range 5000-6000
var descriptor = new BulkDescriptor();
foreach (var i in Enumerable.Range(5000, 1000))
descriptor.Index<ElasticsearchProject>(op => op.Object(new ElasticsearchProject { Id = i }));
var result = this._client.Bulk(d=>descriptor);
result.Should().NotBeNull();
result.IsValid.Should().BeTrue();
//Now lets update all of them giving them a name
descriptor = new BulkDescriptor().Refresh();
foreach (var i in Enumerable.Range(5000, 1000))
{
int id = i;
descriptor.Update<ElasticsearchProject, object>(op => op
.Object(new ElasticsearchProject { Id = id })
.Document(new { name = "SufixedName-" + id})
);
}
result = this._client.Bulk(d=>descriptor);
result.Should().NotBeNull();
result.IsValid.Should().BeTrue();
result.Errors.Should().BeFalse();
result.Items.Count().Should().Be(1000);
result.Items.All(i => i != null).Should().BeTrue();
var updatedObject = this._client.Source<ElasticsearchProject>(i=>i.Id(5000));
Assert.NotNull(updatedObject);
Assert.AreEqual(updatedObject.Name, "SufixedName-5000");
}
public async Task <int> CreateAsync(IList <T> items, Refresh refresh = Refresh.WaitFor)
{
if (!items.HasItems())
{
throw new ArgumentException(string.Format(Utils.ARGUMENT_EMPTY_LIST_MESSAGE, nameof(items)), nameof(items));
}
var descriptor = new BulkDescriptor();
foreach (var item in items)
{
if (string.IsNullOrWhiteSpace(item.Id))
{
descriptor.Index <T>(x => x
.Document(item));
}
else
{
descriptor.Create <T>(x => x
.Document(item));
}
}
descriptor.Refresh(refresh);
var response = await _client.BulkAsync(descriptor);
return(!response.IsValid ? 0 : response.Items.Count);
}
public async Task <bool> TryAddAsync(TKey key, ISet <T> value, bool overwrite = false, CancellationToken cancellationToken = default(CancellationToken))
{
var items = await value.AsEnumerableAsync(cancellationToken).ConfigureAwait(false);
var documents = new Dictionary <string, T>();
await items.ForEachAsync(
async item =>
{
documents.Add($"{key}:{GetSubKeyValue(item)}", item);
},
cancellationToken);
var descriptor = new BulkDescriptor();
foreach (var document in documents)
{
descriptor.Index <T>(op => op
.Id(document.Key)
.Document(document.Value));
}
var result = await ElasticClient.BulkAsync(descriptor, cancellationToken);
return(result.IsValid);
}
public void Setup()
{
_indexName = ElasticsearchConfiguration.NewUniqueIndexName();
_repositoryName = ElasticsearchConfiguration.NewUniqueIndexName();
_snapshotName = ElasticsearchConfiguration.NewUniqueIndexName();
var descriptor = new BulkDescriptor();
_indexedElements = new List <ElasticsearchProject>();
for (int i = 0; i < 100; i++)
{
var elementToIndex = new ElasticsearchProject()
{
Id = i,
Name = "Coboles",
Content = "COBOL elasticsearch client"
};
descriptor = descriptor.Index <ElasticsearchProject>(d => d.Index(_indexName).Document(elementToIndex));
_indexedElements.Add(elementToIndex);
}
var bulkResponse = this.Client.Bulk(d => descriptor);
this.Client.CreateRepository(_repositoryName, r => r
.FileSystem(@"local\\path", o => o
.Compress()
.ConcurrentStreams(10)));
}
public static void Bulk_Index_Iterate(string url)
{
var uri = new Uri(url);
var config = new ConnectionSettings(uri);
config.SetDefaultIndex("rolling-stone-500");
var client = new ElasticClient(config);
List <Album> albums = new List <Album> {
new Album(),
new Album(),
new Album(),
};
var bulkDescriptor = new BulkDescriptor();
foreach (var album in albums)
{
bulkDescriptor.Index <Album>(bid => bid
.Id(album.Rank)
.Document(album)
);
}
var result = client.Bulk(bulkDescriptor);
}
public void Setup()
{
_indexName = ElasticsearchConfiguration.NewUniqueIndexName();
_repositoryName = ElasticsearchConfiguration.NewUniqueIndexName();
_snapshotName = ElasticsearchConfiguration.NewUniqueIndexName();
var descriptor = new BulkDescriptor();
_indexedElements = new List<ElasticsearchProject>();
for (int i = 0; i < 100; i++)
{
var elementToIndex = new ElasticsearchProject()
{
Id = i,
Name = "Coboles",
Content = "COBOL elasticsearch client"
};
descriptor = descriptor.Index<ElasticsearchProject>(d => d.Index(_indexName).Document(elementToIndex));
_indexedElements.Add(elementToIndex);
}
var bulkResponse = this.Client.Bulk(d => descriptor);
this.Client.CreateRepository(_repositoryName, r => r
.FileSystem(@"local\\path", o => o
.Compress()
.ConcurrentStreams(10)));
}
public ReindexApiTests(ManualReindexCluster cluster)
{
_client = cluster.Client;
// create a couple of projects
var projects = Project.Generator.Generate(2);
_client.IndexMany(projects, IndexName);
_client.Indices.Refresh(IndexName);
// create a thousand commits and associate with the projects
var commits = CommitActivity.Generator.Generate(5000);
var bb = new BulkDescriptor();
for (var i = 0; i < commits.Count; i++)
{
var commit = commits[i];
var project = i % 2 == 0
? projects[0].Name
: projects[1].Name;
bb.Index <CommitActivity>(bi => bi
.Document(UpdateJoin(commit, project))
.Index(IndexName)
.Id(commit.Id)
.Routing(project)
);
}
CommitActivity UpdateJoin(CommitActivity a, string p)
{
a.ProjectName = p;
return(a);
}
var bulkResult = _client.Bulk(b => bb);
bulkResult.ShouldBeValid();
_client.Indices.Refresh(IndexName);
_reindexManyTypesResult = _client.Reindex <ILazyDocument>(r => r
.BackPressureFactor(10)
.ScrollAll("1m", 2, s => s
.Search(ss => ss
.Index(IndexName)
)
.MaxDegreeOfParallelism(4)
)
.BulkAll(b => b
.Index(NewManyTypesIndexName)
.Size(100)
.MaxDegreeOfParallelism(2)
.RefreshOnCompleted()
)
);
_reindexSingleTypeResult = _client.Reindex <Project>(IndexName, NewSingleTypeIndexName);
_reindexProjectionResult = _client.Reindex <CommitActivity, CommitActivityVersion2>(
IndexName, NewProjectionIndex, p => new CommitActivityVersion2(p));
}
private void IndexAll <TRoyal, TParent>(BulkDescriptor bulk, Func <IEnumerable <TRoyal> > create, TParent parent = null,
Action <TRoyal> indexChildren = null
)
where TRoyal : class, IRoyal
where TParent : class, IRoyal
{
var current = create();
//looping twice horrible but easy to debug :)
var royals = current.ToList();
foreach (var royal in royals)
{
var royal1 = royal;
if (parent == null)
{
royal.Join = JoinField.Root <TRoyal>();
}
if (royal.Join == null)
{
royal.Join = JoinField.Link <TRoyal, TParent>(parent);
}
bulk.Index <TRoyal>(i => i.Document(royal1).Index(_index).Type(RoyalType).Routing(parent == null ? royal.Name : parent.Name));
}
if (indexChildren == null)
{
return;
}
foreach (var royal in royals)
{
indexChildren(royal);
}
}
public async Task IndexProject(List <Project> projects)
{
if (projects != null)
{
var _ElasticClient = ConnectionHelper.GetClientSingleton(_configurationSettings);
List <Project> items = projects;
var descriptor = new BulkDescriptor();
if (items != null && items.Count > 0)
{
for (int index = 0; index < items.Count; index++)
{
var indexingItem = items[index];
if (indexingItem != null)
{
descriptor.Index <Project>(o => o.Document(indexingItem)
.Routing(indexingItem.Id)
.Id(indexingItem.Id)
.Index(_configurationSettings.AliasName)).Refresh(Elasticsearch.Net.Refresh.True);
}
}
}
var response = await _ElasticClient.BulkAsync(descriptor);
if (!response.IsValid || response.ItemsWithErrors.Any())
{
_logger.LogError(string.Format("Error on IndexItem. isValid : {0} , errorItems : {1} {2}", response.IsValid, response.ItemsWithErrors.Count(), response.ServerError.Error));
}
}
}
private BulkDescriptor ToElasticSearchDocs(IEnumerable <ValueSet> docs, string indexTarget)
{
var descriptor = new BulkDescriptor();
foreach (var d in docs)
{
//this is just a dictionary
var ad = new Document
{
["Id"] = d.Id,
[FormatFieldName(LuceneIndex.ItemIdFieldName)] = d.Id,
[FormatFieldName(LuceneIndex.ItemTypeFieldName)] = d.ItemType,
[FormatFieldName(LuceneIndex.CategoryFieldName)] = d.Category
};
foreach (var i in d.Values)
{
if (i.Value.Count > 0)
{
ad[FormatFieldName(i.Key)] = i.Value.Count == 1 ? i.Value[0]: i.Value;
}
}
var docArgs = new DocumentWritingEventArgs(d, ad);
OnDocumentWriting(docArgs);
descriptor.Index <Document>(op => op.Index(indexTarget).Document(ad).Id(d.Id));
}
return(descriptor);
}
/// <summary>
/// Indexes the specified document ids.
/// </summary>
/// <param name="documentIds">The document ids.</param>
/// <param name="fields">The fields.</param>
public override void Index(IEnumerable <long> documentIds, IDictionary <string, object?> fields)
{
var descriptor = new BulkDescriptor();
// https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-refresh.html
// Refresh.WaitFor: waits until the indexing completed and changes are visisble for search
_ = descriptor.Refresh(Refresh.WaitFor);
foreach (var field in fields)
{
var indexId = Connection.GetIndexId(field.Key);
dynamic updateDoc = new System.Dynamic.ExpandoObject();
updateDoc.value = field.Value;
foreach (var documentId in documentIds)
{
_ = descriptor.Index <object>(op => op
.Index(indexId)
.Id(documentId)
.Document(updateDoc)
);
}
}
var result = Connection.Client.Bulk(descriptor);
}
private BulkDescriptor GenerateElasticDescriptor(IEnumerable <ElasticOwnershipIndexCollection> t)
{
var descriptor = new BulkDescriptor();
foreach (var ownershipIndex in t)
{
if (ownershipIndex.ElasticOwnership.Count > 0)
{
foreach (var elasticData in ownershipIndex.ElasticOwnership.Select(ownership => new ElasticOwnershipIndex()
{
Id = ownershipIndex.Isbn + ownership.ScopeId,
TotalCopies = ownership.TotalCopies,
CirculationCopies = ownership.CirculationCopies,
HoldsCopies = ownership.HoldsCopies,
HoldsRatio = ownership.HoldsRatio,
ScopeId = ownership.ScopeId,
Subscriptions = ownership.Subscriptions,
Sop = ownership.Sop,
Expirations = ownership.Expirations,
}))
{
descriptor.Index <ElasticOwnershipIndex>(op => op.Document(elasticData).Parent(ownershipIndex.Isbn));
}
}
}
return(descriptor);
}
public IBulkResponse IndexSearchResults(ISearchResponse <T> searchResult, IObserver <IReindexResponse <T> > observer, string toIndex, int page)
{
if (!searchResult.IsValid)
{
throw new ReindexException(searchResult.ConnectionStatus, "reindex failed on scroll #" + page);
}
var bb = new BulkDescriptor();
foreach (var d in searchResult.Hits)
{
IHit <T> d1 = d;
bb.Index <T>(bi => bi.Document(d1.Source).Type(d1.Type).Index(toIndex).Id(d.Id));
}
var indexResult = this.CurrentClient.Bulk(b => bb);
if (!indexResult.IsValid)
{
throw new ReindexException(indexResult.ConnectionStatus, "reindex failed when indexing page " + page);
}
observer.OnNext(new ReindexResponse <T>()
{
BulkResponse = indexResult,
SearchResponse = searchResult,
Scroll = page
});
return(indexResult);
}
void WriteIndex <T>(IEnumerable <T> entities, Indexes indexes, Func <T, object> idAccessor = null, bool recreate = true) where T : class
{
if (entities == null)
{
return;
}
var index = IndexHelper.GetIndexName(indexes);
if (recreate) //TODO: might have to check to see if index exists first time
{
_client.DeleteIndex(index);
_client.CreateIndex(index);
}
var batches = entities.Partition(5000).ToArray(); //split into batches of up to 5000
foreach (var batch in batches)
{
var bulkOperation = new BulkDescriptor();
foreach (var item in batch)
{
T localItem = item;
if (idAccessor == null) //if null let elasticsearch set the id
{
bulkOperation.Index <T>(b => b.Document(localItem).Index(index));
}
else
{
var id = idAccessor(localItem); //Be tricky so we can handle number and string ids
if (id is int)
{
bulkOperation.Index <T>(b => b.Document(localItem).Id((int)id).Index(index));
}
else
{
bulkOperation.Index <T>(b => b.Document(localItem).Id(id.ToString()).Index(index));
}
}
}
_client.Bulk(_ => bulkOperation);
}
}
public static void Setup()
{
var client = ElasticsearchConfiguration.Client;
var projects = NestTestData.Data;
var people = NestTestData.People;
var boolTerms = NestTestData.BoolTerms;
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping <Person>(m => m.MapFromAttributes())
.AddMapping <BoolTerm>(m => m.Properties(pp => pp
.String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.not_analyzed))
.String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.not_analyzed))
))
);
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping <Person>(m => m.MapFromAttributes())
.AddMapping <BoolTerm>(m => m.Properties(pp => pp
.String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.not_analyzed))
.String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.not_analyzed))
))
);
var bulk = new BulkDescriptor();
foreach (var p in projects)
{
bulk.Index <ElasticSearchProject>(i => i.Object(p));
}
foreach (var p in people)
{
bulk.Index <Person>(i => i.Object(p));
}
foreach (var p in boolTerms)
{
bulk.Index <BoolTerm>(i => i.Object(p));
}
client.Bulk(bulk);
client.Refresh(new[] { ElasticsearchConfiguration.DefaultIndex, ElasticsearchConfiguration.DefaultIndex + "_clone" });
}
private static BulkDescriptor CreateBulkDescriptor(string indexName)
{
var bulkDescriptor = new BulkDescriptor();
bulkDescriptor.Index(indexName);
return(bulkDescriptor);
}
public static void Indexing()
{
using (LocationContext dbContext = new LocationContext())
{
var customerLocationList = dbContext.CustomerLocations.Where(s => !s.IsDeleted)
.Include(s => s.Customer)
.Select(s => new CustomerModel
{
CustomerId = s.CustomerId,
CustomerName = s.Customer.Name,
LocationId = s.Id,
LocationName = s.Name,
Location = new GeoLocation(Convert.ToDouble(s.Latitude), Convert.ToDouble(s.Longitude))
}).ToList();
var defaultIndex = "customerlocation";
var client = new ElasticClient();
if (client.IndexExists(defaultIndex).Exists)
{
client.DeleteIndex(defaultIndex);
}
if (!elasticClient.IndexExists("location_alias").Exists)
{
client.CreateIndex(defaultIndex, c => c
.Mappings(m => m
.Map <CustomerModel>(mm => mm
.AutoMap()
)
).Aliases(a => a.Alias("location_alias"))
);
}
// Insert Data Classic
// for (int i = 0; i < customerLocationList.Count; i++)
// {
// var item = customerLocationList[i];
// elasticClient.Index<CustomerModel>(item, idx => idx.Index("customerlocation").Id(item.LocationId));
// }
// Bulk Insert
var bulkIndexer = new BulkDescriptor();
foreach (var document in customerLocationList)
{
bulkIndexer.Index <CustomerModel>(i => i
.Document(document)
.Id(document.LocationId)
.Index("customerlocation"));
}
elasticClient.Bulk(bulkIndexer);
}
}
public void bulkIndexCreate(string indexName, Object modelList)
{
var descriptor = new BulkDescriptor();
descriptor.Index <ElasticDataModel>(op => op
.Document(modelList as ElasticDataModel)
);
var result = esClient.Bulk(descriptor);
}
public string InsertBulkText(List <Text> texts, string index)
{
var descriptor = new BulkDescriptor();
descriptor.Index(new IndexName()
{
Name = index
});
foreach (var text in texts)
{
//For each Product in Product-list, index it to the given index
descriptor.Index <Text>(i => i.Document(text));
}
string response = client.Bulk(descriptor).ToString();
return(response);
}
protected virtual void AddElementsToIndex(BulkDescriptor descriptor, List <T> elements)
{
foreach (T element in elements)
{
descriptor.Index <T>(op => op.Index(_indexName)
.Version(DateTime.Now.Ticks)
.VersionType(VersionType.External)
.Document(element));
}
}
public IBulkResponse BulkInsertCandidates(IEnumerable <VacancyElasticModel> candidates)
{
var bulk = new BulkDescriptor();
candidates.ForEach(cand => bulk.Index <VacancyElasticModel>(i => i
.Index(IndexName)
.Id(cand.Id)
.Document(cand)
));
return(_client.Bulk(bulk));
}
public static void InsertBulkDocument()
{
var descriptor = new BulkDescriptor();
foreach (var employeeObj in PopulateEmployees())
{
Employee obj = employeeObj;
descriptor.Index <Employee>(op => op.Document(obj));
}
var bulkresult = EsClient.Bulk(descriptor);
}
public void IndexBulk(IEnumerable <ElasticArchiveRecord> records)
{
var descriptor = new BulkDescriptor();
foreach (var r in records)
{
descriptor.Index <ElasticArchiveRecord>(op => op.Document(r).Id(r.ArchiveRecordId));
}
Client.Bulk(descriptor);
}
public void BulkAlternativeWayOfWriting()
{
var descriptor = new BulkDescriptor();
foreach (var i in Enumerable.Range(3000, 1000))
descriptor.Index<ElasticsearchProject>(op => op.Object(new ElasticsearchProject {Id = i}));
var result = this._client.Bulk(d=>descriptor);
result.Should().NotBeNull();
result.IsValid.Should().BeTrue();
result.Items.Should().NotBeNull().And.NotBeEmpty().And.HaveCount(1000).And.OnlyContain(r => r.OK);
}
public IResponse AddDocuments(List <Document> documents)
{
var bulkDescriptor = new BulkDescriptor();
foreach (var doc in documents)
{
bulkDescriptor.Index <Document>(x => x
.Index(indexName)
.Document(doc));
}
return(client.Bulk(bulkDescriptor));
}
public static void Setup()
{
var client = ElasticsearchConfiguration.Client;
var projects = NestTestData.Data;
var people = NestTestData.People;
var boolTerms = NestTestData.BoolTerms;
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping<Person>(m => m.MapFromAttributes())
.AddMapping<BoolTerm>(m => m.Properties(pp=>pp
.String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.not_analyzed))
.String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.not_analyzed))
))
);
client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
.AddMapping<Person>(m => m.MapFromAttributes())
.AddMapping<BoolTerm>(m => m.Properties(pp => pp
.String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.not_analyzed))
.String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.not_analyzed))
))
);
var bulk = new BulkDescriptor();
foreach (var p in projects)
bulk.Index<ElasticSearchProject>(i=>i.Object(p));
foreach (var p in people)
bulk.Index<Person>(i => i.Object(p));
foreach (var p in boolTerms)
bulk.Index<BoolTerm>(i => i.Object(p));
client.Bulk(bulk);
client.Refresh(new[] {ElasticsearchConfiguration.DefaultIndex, ElasticsearchConfiguration.DefaultIndex + "_clone"});
}
private async Task<ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) {
const int pageSize = 100;
const string scroll = "10s";
var scanResults = await _client.SearchAsync<JObject>(s => s.Index(workItem.OldIndex).AllTypes().Filter(f => startTime.HasValue ? f.Range(r => r.OnField(workItem.TimestampField ?? "_timestamp").Greater(startTime.Value)) : f.MatchAll()).From(0).Take(pageSize).SearchType(SearchType.Scan).Scroll(scroll)).AnyContext();
if (!scanResults.IsValid || scanResults.ScrollId == null) {
Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return new ReindexResult();
}
long totalHits = scanResults.Total;
long completed = 0;
int page = 0;
var results = await _client.ScrollAsync<JObject>(scroll, scanResults.ScrollId).AnyContext();
while (results.Documents.Any()) {
var bulkDescriptor = new BulkDescriptor();
foreach (var hit in results.Hits) {
var h = hit;
// TODO: Add support for doing JObject based schema migrations
bulkDescriptor.Index<JObject>(idx => idx.Index(workItem.NewIndex).Type(h.Type).Id(h.Id).Document(h.Source));
}
var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext();
if (!bulkResponse.IsValid) {
string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} page={page} message={bulkResponse.GetErrorMessage()}";
Logger.Warn().Message(message).Write();
// try each doc individually so we can see which doc is breaking us
foreach (var hit in results.Hits) {
var h = hit;
var response = await _client.IndexAsync(h.Source, d => d.Index(workItem.NewIndex).Type(h.Type).Id(h.Id)).AnyContext();
if (response.IsValid)
continue;
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} page={page} message={response.GetErrorMessage()}";
Logger.Error().Message(message).Write();
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += bulkResponse.Items.Count();
await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
results = await _client.ScrollAsync<JObject>(scroll, results.ScrollId).AnyContext();
page++;
}
return new ReindexResult {
Total = totalHits,
Completed = completed
};
}
public bool AddSales(Admin admin, Sales sales, List<SalesInfo> salesInfo, bool stock)
{
if (sales == null || salesInfo == null) throw new Exception(ErrorConstants.REQUIRED_FIELD_EMPTY);
if (admin == null || admin.type != (int)BillingEnums.USER_TYPE.ADMIN) throw new Exception(ErrorConstants.NO_PREVILAGE);
try
{
var elasticClient = GetElasticClient();
var salesResponse = elasticClient.Index<Sales>(sales, i => i
.Index(ElasticMappingConstants.INDEX_NAME)
.Type(ElasticMappingConstants.TYPE_SALES)
);
var insertDescriptor = new BulkDescriptor();
foreach (var item in salesInfo)
{
item.salesid = sales.salesid;
insertDescriptor.Index<SalesInfo>(i => i
.Index(ElasticMappingConstants.INDEX_NAME)
.Type(ElasticMappingConstants.TYPE_SALES_INFO)
.Document(item)
);
}
var bulkResponse = elasticClient.Bulk(insertDescriptor);
return salesResponse.RequestInformation.Success && bulkResponse.RequestInformation.Success;
}
catch (Exception e)
{
throw e;
}
}
public string LogData()
{
string result = string.Empty;
try
{
List<VODDetails> objList = new List<VODDetails>();
objList = DAL.FetchData();
#region ElasticSearch --Begin
string indexName = "test";
string typeName = "test01";
var node = new Uri("http://localhost:9200");
var settings = new ConnectionSettings(node);
var client = new ElasticClient(settings);
BulkDescriptor objbulk = new BulkDescriptor();
foreach (var value in objList)
{
objbulk.Index<object>(i => i
.Index(indexName)
.Type(typeName)
.Id(value.strAssetID)
.Document(value));
client.Bulk(objbulk);
}
result = "Data successfully inserted";
#endregion ElasticSearch --End
}
catch (Exception ex)
{
result = ex.Message;
}
return result;
}
private async Task<ReindexResult> ReindexAsync(ReindexWorkItem workItem, WorkItemContext context, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) {
const int pageSize = 100;
const string scroll = "5m";
string timestampField = workItem.TimestampField ?? "_timestamp";
long completed = 0;
var scanResults = await _client.SearchAsync<JObject>(s => s
.Index(workItem.OldIndex)
.AllTypes()
.Filter(f => startTime.HasValue
? f.Range(r => r.OnField(timestampField).Greater(startTime.Value))
: f.MatchAll())
.From(0).Take(pageSize)
.SearchType(SearchType.Scan)
.Scroll(scroll)).AnyContext();
if (!scanResults.IsValid || scanResults.ScrollId == null) {
Logger.Error().Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return new ReindexResult();
}
long totalHits = scanResults.Total;
var parentMap = workItem.ParentMaps?.ToDictionary(p => p.Type, p => p.ParentPath) ?? new Dictionary<string, string>();
var results = await _client.ScrollAsync<JObject>(scroll, scanResults.ScrollId).AnyContext();
while (results.Documents.Any()) {
var bulkDescriptor = new BulkDescriptor();
foreach (var hit in results.Hits) {
var h = hit;
// TODO: Add support for doing JObject based schema migrations
bulkDescriptor.Index<JObject>(idx => {
idx
.Index(workItem.NewIndex)
.Type(h.Type)
.Id(h.Id)
.Document(h.Source);
if (String.IsNullOrEmpty(h.Type))
Logger.Error().Message("Hit type empty. id={0}", h.Id).Write();
if (parentMap.ContainsKey(h.Type)) {
if (String.IsNullOrEmpty(parentMap[h.Type]))
Logger.Error().Message("Parent map has empty value. id={0} type={1}", h.Id, h.Type).Write();
var parentId = h.Source.SelectToken(parentMap[h.Type]);
if (!String.IsNullOrEmpty(parentId?.ToString()))
idx.Parent(parentId.ToString());
else
Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write();
}
return idx;
});
}
var bulkResponse = await _client.BulkAsync(bulkDescriptor).AnyContext();
if (!bulkResponse.IsValid) {
string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse.GetErrorMessage()}";
Logger.Warn().Message(message).Write();
// try each doc individually so we can see which doc is breaking us
foreach (var hit in results.Hits) {
var h = hit;
var response = await _client.IndexAsync<JObject>(h.Source, d => {
d
.Index(workItem.NewIndex)
.Type(h.Type)
.Id(h.Id);
if (parentMap.ContainsKey(h.Type)) {
var parentId = h.Source.SelectToken(parentMap[h.Type]);
if (!String.IsNullOrEmpty(parentId?.ToString()))
d.Parent(parentId.ToString());
else
Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write();
}
return d;
}).AnyContext();
if (response.IsValid)
continue;
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={hit.Id} completed={completed} message={response.GetErrorMessage()}";
Logger.Error().Message(message).Write();
var errorDoc = new JObject(new {
h.Type,
Content = h.Source.ToString(Formatting.Indented)
});
if (parentMap.ContainsKey(h.Type)) {
var parentId = h.Source.SelectToken(parentMap[h.Type]);
if (!String.IsNullOrEmpty(parentId?.ToString()))
errorDoc["ParentId"] = parentId.ToString();
else
Logger.Error().Message("Unable to get parent id. id={0} path={1}", h.Id, parentMap[h.Type]).Write();
}
// put the document into an error index
response = await _client.IndexAsync<JObject>(errorDoc, d => {
d
.Index(workItem.NewIndex + "-error")
.Id(h.Id);
return d;
}).AnyContext();
if (response.IsValid)
continue;
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += bulkResponse.Items.Count();
await context.ReportProgressAsync(CalculateProgress(totalHits, completed, startProgress, endProgress),
$"Total: {totalHits} Completed: {completed}").AnyContext();
Logger.Info().Message($"Reindex Progress: {CalculateProgress(totalHits, completed, startProgress, endProgress)} Completed: {completed} Total: {totalHits}").Write();
results = await _client.ScrollAsync<JObject>(scroll, results.ScrollId).AnyContext();
}
return new ReindexResult { Total = totalHits, Completed = completed };
}
public async Task AddTestData()
{
log.LogInformation("Inside AddTest data");
try
{
await Task.Delay(5000);
for (int i = 0; i < 4; i++)
{
IPingResponse x = await client.PingAsync();
if (x.ConnectionStatus.Success)
{
log.LogInformation(DateTime.Now.ToLongTimeString() + " Connection successful to - " + x.ConnectionStatus.RequestUrl);
break;
}
else
{
log.LogWarning(DateTime.Now.ToLongTimeString() + " Unable to connect to - " + x.ConnectionStatus.RequestUrl);
await Task.Delay(i * 1000);
}
}
Category entertainment = new Category()
{
Id = 1,
Name = "Entertainment"
};
Category equipment = new Category()
{
Id = 2,
Name = "Equipment"
};
Category foodsupply = new Category()
{
Id = 3,
Name = "Food Supply"
};
int id = 0;
var allProducts = new List<Product>();
allProducts.Add(new Product()
{
Id = id++,
Title = "Pet Rock",
Quantity = 20,
Price = 5.0M,
Description = @"Why be lonely when you can have a pet? The Pet Rock is the lowest maintenance pet you'll ever own",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/Pet-Bio-Rock-with-terrarium.jpg",
CategoryId = entertainment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Robo Buddy",
Quantity = 1,
Price = 399.99M,
Description = @"Robo Buddy is the ultimate Robot toy that every child and adult needs!",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/ROBO-BUDDY.jpg",
CategoryId = entertainment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Jet Pack",
Quantity = 5,
Price = 999.99M,
Description = @"Be the envy of your planetary colony with this deluxe hydrogen-powered Jet Pack.",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/Jet-Pack.jpg",
CategoryId = equipment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Moon Boots",
Price = 299.99M,
Description = @"Hand crafted and heat moldable, these boots will keep you warm when the temperatures hit below 50 degrees!",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/Moon-Boot.jpg",
CategoryId = equipment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Indestructible Flag Pole",
Quantity = 300,
Price = 75.00M,
Description = @"This indescructible, high-suction flag pole helps adventurers claim what is righfully theirs.",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/Indestructable-High-Suction-flag-pole.jpg",
CategoryId = equipment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Emergency Beacon",
Quantity = 7,
Price = 125.00M,
Description = @"This solar powered emergency beacon is a must-have for any adventurer. ",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/emergency-beacon.jpg",
CategoryId = equipment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Short range Lazer blaster",
Quantity = 40,
Price = 800.00M,
Description = @"The best defense is a good offense and the Lazer blaster gives adventurers piece of mind to handle whatever they may encounter.",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/LAzer.jpg",
CategoryId = equipment.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Crunch Bar",
Quantity = 36,
Price = 2.75M,
Description = @"Organic, gluten free, and flavor free, Crunch Bars provide the perfect ratio of protein, carbs, and fat for hungry adventurers.",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/Crunch-Bar.jpg",
CategoryId = foodsupply.Id
});
allProducts.Add(new Product()
{
Id = id++,
Title = "Hydro Drink",
Quantity = 12,
Price = 3.50M,
Description = @"Hydro Drink packs in double the caffeine of other energy drinks but with potassium and electryolytes to quickly rehydrate.",
ProductArtUrl = @"https://dockerbook.blob.core.windows.net/images/Hydo-Drink.jpg",
CategoryId = foodsupply.Id
});
entertainment.Products = allProducts.Where(c => c.Id == 1).ToList<Product>();
equipment.Products = allProducts.Where(c => c.Id == 2).ToList<Product>();
foodsupply.Products = allProducts.Where(c => c.Id == 3).ToList<Product>();
var descriptor = new BulkDescriptor();
descriptor.Index<Category>(op => op.Document(entertainment));
descriptor.Index<Category>(op => op.Document(equipment));
descriptor.Index<Category>(op => op.Document(foodsupply));
foreach (var p in allProducts)
{
descriptor.Index<Product>(op => op.Document(p));
}
log.LogWarning("before bulk async");
var result = await client.BulkAsync(descriptor);
log.LogWarning("after bulk async");
}
catch (Exception ex)
{
log.LogError(DateTime.Now.ToLongTimeString() + " - Ex Caught:" + ex.Message);
}
}
private void ConfigureIndexItem(BulkDescriptor d, IHit<JObject> hit, string targetIndex) {
d.Index<JObject>(idx => ConfigureItem(idx, hit, targetIndex));
}
public static void LoadData(IElasticClient client)
{
var r = client.CreateIndex("entities", c => c
.AddMapping<JsonObject>(m => m
.IdField(i => i.SetIndex("not_analyzed"))
.TypeName("locations")
.Properties(p => p
.String(s => s.Name("id"))
.String(s => s.Name("name").Index(FieldIndexOption.analyzed).IndexAnalyzer("standard"))
.String(s => s.Name("parentId")))
));
var all = new List<JsonObject>();
var reader = new StreamReader(File.OpenRead(@"c:\temp\countries.csv"), new UTF8Encoding());
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (line == null) continue;
var values = line.Split(',');
values[2] = values[2].Replace("\"", "");
var location = CreateObject.CreateMiniEntity(values[0], values[1], values[2]);
all.Add(location);
}
var allObjects = all.ToDictionary(json => json.Id);
foreach (var item in all)
{
var path = new List<string>();
if (!String.IsNullOrEmpty(item["parentId"].ToString()))
{
RecGetParent(path, allObjects, item);
path.Reverse();
path.Add(item["name"].ToString());
item.Add("fullPath", String.Join("#.#", path));
}
else
item.Add("fullPath", String.Empty);
}
var insertCount = 0;
var bulker = new BulkDescriptor();
for (var index = 0; index < all.Count; index++)
{
var item = all[index];
bulker.Index<JsonObject>(op =>
op.Index("entities")
.Id(Convert.ToString(item["id"]))
.Type("locations")
.Object(item));
insertCount++;
if (insertCount != 1000 && index != (all.Count - 1)) continue;
//PushToElastic(client, bulker);
var result = client.Bulk(bulker);
insertCount = 0;
bulker = new BulkDescriptor();
}
}