private static async Task LoadSingleDoc(string collectionName, string docType, string indexId)
{
try
{
MongoConnectionSettings myMongo = new MongoConnectionSettings
{
limit = Convert.ToInt32(ConfigurationManager.AppSettings["MongoCount"]),
skipCount = Convert.ToInt32(ConfigurationManager.AppSettings["SkipCount"]),
databaseName = ConfigurationManager.AppSettings["MongoDatabase"],
collectionName = collectionName
};
var Client = new MongoClient((mongoEnvironment.ToLower() == "development") ? connectionString : connectionStringRS);
var DB = Client.GetDatabase(myMongo.databaseName);
var collection = DB.GetCollection <BsonDocument>(myMongo.collectionName);
var mongoCount = collection.CountDocuments(new BsonDocument());
Console.Write("{0} Records Acquired from MongoDB in {1}\n", mongoCount, collectionName);
var indexName = indexPrefix + myMongo.collectionName.ToLower();
var documentType = myMongo.collectionName.ToLower();
StringBuilder bulk = new StringBuilder();
bulk.Clear();
var results = await collection.Find(x => x["type"] == docType && x["_id"] == indexId).ToListAsync();
foreach (var doc in results)
{
var documentId = doc["_id"].ToString();
doc.Remove("_id");
string metadata = "{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_type\" : \"" + docType.ToLower() + "\", \"_id\" : \"" + documentId + "\" } } \n";
bulk.Append(metadata);
bulk.Append(doc.ToJson(new JsonWriterSettings {
OutputMode = JsonOutputMode.Strict
}) + "\n");
}
ErrorLog el = await SendToElasticBulkApi(bulk);
if (el.isError)
{
int newResultsCount = results.Count - el.errorCount;
Console.Write("\n+++ STATS +++\n");
Console.Write("\nElasticSearch responded with errors, please review logs for more information.\n");
Console.Write("{0} Errors\n", el.errorCount.ToString());
Console.Write("{0} Record(s) Loaded in this Batch\n", newResultsCount.ToString());
}
else
{
Console.Write("\n{0} Records Loaded in this Batch\n", results.Count.ToString());
}
if (saveLogsToFS)
{
if (saveLastIndexToFS)
{
u.SaveToFile(logPath + "LastIndex.txt", indexName + "\t" + indexId + "\t" + (results.Count - el.errorCount).ToString() + "\t" + el.errorCount.ToString() + "\t" + "SINGLE REINDEX");
}
u.SaveToFile(logPath + "LastIndex_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", indexName + "\t" + indexId + "\t" + (results.Count - el.errorCount).ToString() + "\t" + el.errorCount.ToString() + "\t" + "SINGLE REINDEX");
}
else
{
await SaveLogToElastic("{ \"indexDateTime\" : \"" + DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss") + "\", \"indexName\" : \"" + indexName + "\", \"lastIndexId\" : \"" + indexId + "\", \"lastSuccessCount\" : \"" + (results.Count - el.errorCount).ToString() + "\", \"lastFailCount\" : \"" + el.errorCount.ToString() + "\", \"loadType\" : \"SINGLE REINDEX\" }\n");
}
}
catch (Exception e)
{
if (saveLogsToFS)
{
u.SaveToFile(logPath + "Exceptions_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", "LoadSingleDoc()\t" + e.Message.ToString() + " (" + e.InnerException.Message.ToString() + ")");
}
else
{
SaveExceptionToElastic("LoadSingleDoc()", e).Wait();
}
}
}
/// <summary>
/// Connects to MongoDB and incrementally pulls data to be parsed and bulk loaded into the ElasticSearch instance via the Bulk API. This method only performs a delta update.
/// </summary>
/// <param name="collectionName">string</param>
/// <param name="lastKnownIndexId">string</param>
/// <returns></returns>
private static async Task DeltaBulkData(string collectionName, string lastKnownIndexId)
{
try
{
MongoConnectionSettings myMongo = new MongoConnectionSettings
{
limit = Convert.ToInt32(ConfigurationManager.AppSettings["MongoCount"]),
skipCount = Convert.ToInt32(ConfigurationManager.AppSettings["SkipCount"]),
databaseName = ConfigurationManager.AppSettings["MongoDatabase"],
collectionName = collectionName
};
var limit = myMongo.limit;
var skipCount = myMongo.skipCount;
var Client = new MongoClient((mongoEnvironment.ToLower() == "development") ? connectionString : connectionStringRS);
var DB = Client.GetDatabase(myMongo.databaseName);
var collection = DB.GetCollection <BsonDocument>(myMongo.collectionName);
var mongoCount = collection.CountDocuments(new BsonDocument());
Console.Write("{0} Records Acquired from MongoDB in {1}\n", mongoCount, collectionName);
var indexName = indexPrefix + myMongo.collectionName.ToLower();
bool isDeltaFound = false;
StringBuilder bulk = new StringBuilder();
for (int i = 0; i < mongoCount; i += limit)
{
bulk.Clear();
var filter = Builders <BsonDocument> .Filter.Gt(x => x["_id"], lastKnownIndexId);
var results = await collection.Find(filter)
.Sort(Builders <BsonDocument> .Sort.Ascending("_id"))
.Limit(limit)
.Skip(skipCount)
.ToListAsync();
if (results.Count == 0 && isDeltaFound == false)
{
continue;
}
else if (results.Count >= 1 && isDeltaFound == false)
{
isDeltaFound = true;
}
else if (results.Count == 0 && isDeltaFound)
{
continue;;
}
skipCount += limit;
string lastIndexId = results[results.Count - 1]["_id"].ToString();
foreach (var doc in results)
{
var documentId = doc["_id"].ToString();
doc.Remove("_id");
string metadata = "{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_type\" : \"" + doc["type"].ToString().ToLower() + "\", \"_id\" : \"" + documentId + "\" } } \n";
bulk.Append(metadata);
bulk.Append(doc.ToJson(new JsonWriterSettings {
OutputMode = JsonOutputMode.Strict
}) + "\n");
}
var bodyLength = bulk.Length;
var bulkJson = bulk;
ErrorLog el = await SendToElasticBulkApi(bulkJson);
if (el.isError)
{
int newResultsCount = results.Count - el.errorCount;
Console.Write("\n+++ STATS +++\n");
Console.Write("\nElasticSearch responded with errors, please review logs for more information.\n");
Console.Write("{0} Errors\n", el.errorCount.ToString());
Console.Write("{0} Record(s) Loaded in this Batch\n", newResultsCount.ToString());
}
else
{
Console.Write("\n{0} Records Loaded in this Batch\n", results.Count.ToString());
}
if (saveLogsToFS)
{
if (saveLastIndexToFS)
{
u.SaveToFile(logPath + "LastIndex.txt", indexName + "\t" + lastIndexId + "\t" + (results.Count - el.errorCount).ToString() + "\t" + el.errorCount.ToString() + "\t" + "BULK DELTA");
}
u.SaveToFile(logPath + "LastIndex_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", indexName + "\t" + lastIndexId + "\t" + (results.Count - el.errorCount).ToString() + "\t" + el.errorCount.ToString() + "\t" + "BULK DELTA");
}
else
{
await SaveLogToElastic("{ \"indexDateTime\" : \"" + DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss") + "\", \"indexName\" : \"" + indexName + "\", \"lastIndexId\" : \"" + lastIndexId + "\", \"lastSuccessCount\" : \"" + (results.Count - el.errorCount).ToString() + "\", \"lastFailCount\" : \"" + el.errorCount.ToString() + "\", \"loadType\" : \"BULK DELTA\" }\n");
}
}
if (!isDeltaFound)
{
Console.Write(">> No delta records found in {0}\n\n", collectionName);
}
else
{
Console.Write("\n\n");
}
}
catch (Exception e)
{
if (saveLogsToFS)
{
u.SaveToFile(logPath + "Exceptions_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", "DeltaBulkData()\t" + e.Message.ToString() + " (" + e.InnerException.Message.ToString() + ")");
}
else
{
SaveExceptionToElastic("DeltaBulkData()", e).Wait();
}
}
}
