public void Create(AbstractConnection connection, Process process, Entity entity) { var client = new ElasticSearchClientFactory().Create(connection, entity); client.Client.IndicesCreate(client.Index, "{ \"settings\":{}}"); var fields = GetFields(entity); var properties = new Dictionary <string, object>() { { "properties", fields } }; var body = new Dictionary <string, object>() { { client.Type, properties } }; var json = JsonConvert.SerializeObject(body); var response = client.Client.IndicesPutMapping(client.Index, client.Type, json); if (response.Success) { return; } throw new TransformalizeException(_logger, entity.Name, response.ServerError.Error); }
public bool Check(AbstractConnection connection) { var hashCode = connection.Uri().GetHashCode(); if (Checks.ContainsKey(hashCode)) { return(Checks[hashCode]); } var client = new ElasticSearchClientFactory().Create(connection, null); try { var response = client.Client.Ping(); if (response.HttpStatusCode != null && response.HttpStatusCode == 200) { _logger.Debug("Successful ping of {0}.", connection.Name); return(true); } _logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, response.ServerError.Error); } catch (Exception e) { _logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, e.Message); return(false); } return(false); }
public override void WriteEndVersion(Process process, AbstractConnection input, Entity entity, bool force = false) { if (entity.Updates + entity.Inserts > 0 || force) { var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName)); var versionType = entity.Version == null ? "string" : entity.Version.SimpleType; string end; if (versionType.Equals("datetime") && entity.End is DateTime) { end = ((DateTime)entity.End).ToString("yyyy-MM-ddTHH:mm:ss.fff"); } else if (versionType.Equals("byte[]") || versionType.Equals("rowversion")) { end = Common.BytesToHexString((byte[])entity.End); } else { end = new DefaultFactory(Logger).Convert(entity.End, versionType).ToString(); } var body = new { id = entity.TflBatchId, tflbatchid = entity.TflBatchId, process = entity.ProcessName, connection = input.Name, entity = entity.Alias, updates = entity.Updates, inserts = entity.Inserts, deletes = entity.Deletes, version = end, version_type = versionType, tflupdate = DateTime.UtcNow }; client.Client.Index(client.Index, client.Type, body); } }
public void Drop(AbstractConnection connection, Entity entity) { if (!EntityExists.Exists(connection, entity)) return; var client = new ElasticSearchClientFactory().Create(connection, entity); var response = client.Client.IndicesDelete(client.Index); if (response.Success) return; connection.Logger.EntityWarn(entity.Name, response.ServerError.Error); connection.Logger.EntityWarn(entity.Name, "Trouble deleting {0} {1}.", client.Index, client.Type); }
public override void LoadBeginVersion(Entity entity) { var tflBatchId = GetMaxTflBatchId(entity); if (tflBatchId > 0) { var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName)); var result = client.Client.SearchGet(client.Index, client.Type, s => s .AddQueryString("q", "tflbatchid:" + tflBatchId) .AddQueryString("_source_include", "version,version_type") .AddQueryString("size", 1) ); var hits = result.Response["hits"].hits; var versionType = (string)hits[0]["_source"]["version_type"].Value; entity.Begin = Common.GetObjectConversionMap()[versionType](hits[0]["_source"]["version"].Value); entity.HasRange = true; } }
public bool RecordsExist(AbstractConnection connection, Entity entity) { var checker = new ElasticSearchConnectionChecker(connection.Logger); if (checker.Check(connection)) { var client = new ElasticSearchClientFactory().Create(connection, entity); const string body = @"{ _source: false, from: 0, size:1, query: { match_all: {} } }"; var response = client.Client.Search(client.Index, client.Type, body); if (!response.Success) return false; var result = response.Response; return result["hits"].total > 0; } return false; }
private int GetMaxTflBatchId(string processName) { var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(processName)); var body = new { aggs = new { tflbatchid = new { max = new { field = "tflbatchid" } } }, size = 0 }; var result = client.Client.Search(client.Index, client.Type, body); return(Convert.ToInt32((result.Response["aggregations"]["tflbatchid"]["value"].Value ?? 0))); }
public override void LoadBeginVersion(Entity entity) { var tflBatchId = GetMaxTflBatchId(entity); if (tflBatchId > 0) { var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName)); var result = client.Client.SearchGet(client.Index, client.Type, s => s .AddQueryString("q", "tflbatchid:" + tflBatchId) .AddQueryString("_source_include", "version,version_type") .AddQueryString("size", 1) ); var hits = result.Response["hits"].hits; var versionType = (string)hits[0]["_source"]["version_type"].Value; entity.Begin = Common.GetObjectConversionMap()[versionType](hits[0]["_source"]["version"].Value); entity.HasRange = true; } }
public override void LoadEndVersion(Entity entity) { var client = new ElasticSearchClientFactory().Create(this, entity); var body = new { aggs = new { version = new { max = new { field = entity.Version.Alias.ToLower() } } }, size = 0 }; var result = client.Client.Search(client.Index, client.Type, body); entity.End = Common.GetObjectConversionMap()[entity.Version.SimpleType](result.Response["aggregations"]["version"]["value"].Value); entity.HasRows = entity.End != null; }
public void Drop(AbstractConnection connection, Entity entity) { if (!EntityExists.Exists(connection, entity)) { return; } var client = new ElasticSearchClientFactory().Create(connection, entity); var response = client.Client.IndicesDelete(client.Index); if (response.Success) { return; } connection.Logger.EntityWarn(entity.Name, response.ServerError.Error); connection.Logger.EntityWarn(entity.Name, "Trouble deleting {0} {1}.", client.Index, client.Type); }
public void Create(AbstractConnection connection, Process process, Entity entity) { var client = new ElasticSearchClientFactory().Create(connection, entity); client.Client.IndicesCreate(client.Index, "{ \"settings\":{}}"); var fields = GetFields(entity); var properties = new Dictionary<string, object>() { { "properties", fields } }; var body = new Dictionary<string, object>() { { client.Type, properties } }; var json = JsonConvert.SerializeObject(body); var response = client.Client.IndicesPutMapping(client.Index, client.Type, json); if (response.Success) return; throw new TransformalizeException(_logger, entity.Name, response.ServerError.Error); }
public bool RecordsExist(AbstractConnection connection, Entity entity) { var checker = new ElasticSearchConnectionChecker(connection.Logger); if (checker.Check(connection)) { var client = new ElasticSearchClientFactory().Create(connection, entity); const string body = @"{ _source: false, from: 0, size:1, query: { match_all: {} } }"; var response = client.Client.Search(client.Index, client.Type, body); if (!response.Success) { return(false); } var result = response.Response; return(result["hits"].total > 0); } return(false); }
public bool Check(AbstractConnection connection) { var hashCode = connection.Uri().GetHashCode(); if (Checks.ContainsKey(hashCode)) { return Checks[hashCode]; } var client = new ElasticSearchClientFactory().Create(connection, null); try { var response = client.Client.Ping(); if (response.HttpStatusCode != null && response.HttpStatusCode == 200) { _logger.Debug("Successful ping of {0}.", connection.Name); return true; } _logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, response.ServerError.Error); } catch (Exception e) { _logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, e.Message); return false; } return false; }
public override Fields GetEntitySchema(Process process, Entity entity, bool isMaster = false) { var client = new ElasticSearchClientFactory().CreateNest(this, entity); var mapping = client.Client.GetMapping <dynamic>(m => m .Index(client.Index) .Type(client.Type) ); if (!mapping.IsValid) { throw new TransformalizeException(Logger, "Trouble getting mapping for {0}:{1} {2}", client.Index, client.Type, mapping.ServerError.Error); } var fields = new Fields(); foreach (var pair in mapping.Mapping.Properties) { fields.Add(new Field(pair.Value.Type.Name, "64", FieldType.None, true, "") { Name = pair.Key.Name }); } return(fields); }
public override void WriteEndVersion(Process process, AbstractConnection input, Entity entity, bool force = false) { if (entity.Updates + entity.Inserts > 0 || force) { var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName)); var versionType = entity.Version == null ? "string" : entity.Version.SimpleType; string end; if (versionType.Equals("datetime") && entity.End is DateTime) { end = ((DateTime)entity.End).ToString("yyyy-MM-ddTHH:mm:ss.fff"); } else if (versionType.Equals("byte[]") || versionType.Equals("rowversion")) { end = Common.BytesToHexString((byte[])entity.End); } else { end = new DefaultFactory(Logger).Convert(entity.End, versionType).ToString(); } var body = new { id = entity.TflBatchId, tflbatchid = entity.TflBatchId, process = entity.ProcessName, connection = input.Name, entity = entity.Alias, updates = entity.Updates, inserts = entity.Inserts, deletes = entity.Deletes, version = end, version_type = versionType, tflupdate = DateTime.UtcNow }; client.Client.Index(client.Index, client.Type, body); } }
public override Fields GetEntitySchema(Process process, Entity entity, bool isMaster = false) { var client = new ElasticSearchClientFactory().CreateNest(this, entity); var mapping = client.Client.GetMapping<dynamic>(m => m .Index(client.Index) .Type(client.Type) ); if (!mapping.IsValid) { throw new TransformalizeException(Logger, "Trouble getting mapping for {0}:{1} {2}", client.Index, client.Type, mapping.ServerError.Error); } var fields = new Fields(); foreach (var pair in mapping.Mapping.Properties) { fields.Add(new Field(pair.Value.Type.Name, "64", FieldType.None, true, "") { Name = pair.Key.Name }); } return fields; }
public override void LoadEndVersion(Entity entity) { var client = new ElasticSearchClientFactory().Create(this, entity); var body = new { aggs = new { version = new { max = new { field = entity.Version.Alias.ToLower() } } }, size = 0 }; var result = client.Client.Search(client.Index, client.Type, body); entity.End = Common.GetObjectConversionMap()[entity.Version.SimpleType](result.Response["aggregations"]["version"]["value"].Value); entity.HasRows = entity.End != null; }
private int GetMaxTflBatchId(string processName) { var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(processName)); var body = new { aggs = new { tflbatchid = new { max = new { field = "tflbatchid" } } }, size = 0 }; var result = client.Client.Search(client.Index, client.Type, body); return Convert.ToInt32((result.Response["aggregations"]["tflbatchid"]["value"].Value ?? 0)); }