/// <summary> /// Gets the lastIndexId for an index via the file system. /// </summary> /// <param name="filePath">string</param> /// <param name="indexName">string</param> /// <returns></returns> private static List <LastIndex> ReadLastIndex(string filePath, string indexName) { try { List <LastIndex> indexLog = new List <LastIndex>(); if (System.IO.File.Exists(filePath)) { System.IO.StreamReader sr = new System.IO.StreamReader(filePath); string line; while ((line = sr.ReadLine()) != null) { LastIndex l = new LastIndex(); string[] logLineItems = line.Split(new char[] { '\t' }, StringSplitOptions.None); if (logLineItems[1].ToLower() == indexName.ToLower()) { l.IndexDateTime = Convert.ToDateTime(logLineItems[0]); l.IndexName = logLineItems[1]; l.LastIndexId = logLineItems[2]; l.LoadSuccessCount = Convert.ToInt32(logLineItems[3]); l.LoadFailCount = Convert.ToInt32(logLineItems[4]); l.LoadType = logLineItems[5]; indexLog.Add(l); } } } return(indexLog); } catch (Exception e) { if (saveLogsToFS) { u.SaveToFile(logPath + "Exceptions_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", "ReadLastIndex()\t" + e.Message.ToString()); } else { SaveExceptionToElastic("ReadLastIndex()", e).Wait(); } return(null); } }
/// <summary> /// Removes old lastIndexIds and keeping the most recent in ElasticSearch. /// </summary> /// <param name="indices">List<LastIndex></param> /// <returns></returns> private static async Task <List <LastIndex> > CleanLastIndexLog(List <LastIndex> indices) { try { LastIndex keepLastIndex = indices.OrderByDescending(x => x.IndexDateTime).First(); foreach (LastIndex index in indices) { if (index.Id == keepLastIndex.Id) { continue; } var response = await _client.DeleteAsync("lastindices/lastindex/" + index.Id.ToString()); if (response.IsSuccessStatusCode) { continue; } } indices.RemoveAll(x => x.Id != keepLastIndex.Id); return(indices); } catch (Exception e) { if (saveLogsToFS) { u.SaveToFile(logPath + "Exceptions_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", "CleanLastIndexLog()\t" + e.Message.ToString()); } else { SaveExceptionToElastic("CleanLastIndexLog()", e).Wait(); } return(null); } }
/// <summary> /// Gets the lastIndexId for an index via the ElasticSearch instance. /// </summary> /// <param name="indexName">string</param> /// <returns></returns> private static async Task <List <LastIndex> > ReadLastIndex(string indexName) { try { string logIndex = "lastindices"; bool isLogExist = await CheckIndexExists(logIndex); if (isLogExist) { var content = GetFromElastic(logIndex + "/"); dynamic esResponse = await GetFromElastic(logIndex + "/_search?size=100"); var t = esResponse.GetType(); if (t.Equals(typeof(string))) { JObject o = JObject.Parse(esResponse); LastIndex li = new LastIndex(); var documents = o["hits"]["hits"]; List <LastIndex> indices = new List <LastIndex>(); foreach (var index in documents) { if (index["_source"]["indexName"].ToString().ToLower() == indexName.ToLower()) { li = new LastIndex { Id = (Guid)index["_id"], IndexDateTime = (DateTime)index["_source"]["indexDateTime"], IndexName = index["_source"]["indexName"].ToString(), LastIndexId = index["_source"]["lastIndexId"].ToString(), LoadSuccessCount = (int)index["_source"]["lastSuccessCount"], LoadFailCount = (int)index["_source"]["lastFailCount"], LoadType = index["_source"]["loadType"].ToString() }; indices.Add(li); } else { continue; } } if (indices.Count == 0 || indices == null) { return(new List <LastIndex>()); } else { List <LastIndex> refinedIndices = await CleanLastIndexLog(indices); return(refinedIndices); } } else if (t.Equals(typeof(ErrorLog))) { new Exception("ElasticSearch returned an error on querying lastIndexId from \"" + logIndex + "\"."); } } return(new List <LastIndex>()); } catch (Exception e) { if (saveLogsToFS) { u.SaveToFile(logPath + "Exceptions_" + DateTime.Today.ToString("yyyyMMdd") + ".txt", "ReadLastIndex()\t" + e.Message.ToString()); } else { SaveExceptionToElastic("ReadLastIndex()", e).Wait(); } return(new List <LastIndex>()); } }