public void Create(AbstractConnection connection, Process process, Entity entity)
{
var client = new ElasticSearchClientFactory().Create(connection, entity);
client.Client.IndicesCreate(client.Index, "{ \"settings\":{}}");
var fields = GetFields(entity);
var properties = new Dictionary <string, object>()
{
{ "properties", fields }
};
var body = new Dictionary <string, object>()
{
{ client.Type, properties }
};
var json = JsonConvert.SerializeObject(body);
var response = client.Client.IndicesPutMapping(client.Index, client.Type, json);
if (response.Success)
{
return;
}
throw new TransformalizeException(_logger, entity.Name, response.ServerError.Error);
}
public bool Check(AbstractConnection connection)
{
var hashCode = connection.Uri().GetHashCode();
if (Checks.ContainsKey(hashCode))
{
return(Checks[hashCode]);
}
var client = new ElasticSearchClientFactory().Create(connection, null);
try {
var response = client.Client.Ping();
if (response.HttpStatusCode != null && response.HttpStatusCode == 200)
{
_logger.Debug("Successful ping of {0}.", connection.Name);
return(true);
}
_logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, response.ServerError.Error);
} catch (Exception e) {
_logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, e.Message);
return(false);
}
return(false);
}
public override void WriteEndVersion(Process process, AbstractConnection input, Entity entity, bool force = false) {
if (entity.Updates + entity.Inserts > 0 || force) {
var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName));
var versionType = entity.Version == null ? "string" : entity.Version.SimpleType;
string end;
if (versionType.Equals("datetime") && entity.End is DateTime) {
end = ((DateTime)entity.End).ToString("yyyy-MM-ddTHH:mm:ss.fff");
} else if (versionType.Equals("byte[]") || versionType.Equals("rowversion")) {
end = Common.BytesToHexString((byte[])entity.End);
} else {
end = new DefaultFactory(Logger).Convert(entity.End, versionType).ToString();
}
var body = new {
id = entity.TflBatchId,
tflbatchid = entity.TflBatchId,
process = entity.ProcessName,
connection = input.Name,
entity = entity.Alias,
updates = entity.Updates,
inserts = entity.Inserts,
deletes = entity.Deletes,
version = end,
version_type = versionType,
tflupdate = DateTime.UtcNow
};
client.Client.Index(client.Index, client.Type, body);
}
}
public void Drop(AbstractConnection connection, Entity entity) {
if (!EntityExists.Exists(connection, entity))
return;
var client = new ElasticSearchClientFactory().Create(connection, entity);
var response = client.Client.IndicesDelete(client.Index);
if (response.Success)
return;
connection.Logger.EntityWarn(entity.Name, response.ServerError.Error);
connection.Logger.EntityWarn(entity.Name, "Trouble deleting {0} {1}.", client.Index, client.Type);
}
public override void LoadBeginVersion(Entity entity) {
var tflBatchId = GetMaxTflBatchId(entity);
if (tflBatchId > 0) {
var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName));
var result = client.Client.SearchGet(client.Index, client.Type, s => s
.AddQueryString("q", "tflbatchid:" + tflBatchId)
.AddQueryString("_source_include", "version,version_type")
.AddQueryString("size", 1)
);
var hits = result.Response["hits"].hits;
var versionType = (string)hits[0]["_source"]["version_type"].Value;
entity.Begin = Common.GetObjectConversionMap()[versionType](hits[0]["_source"]["version"].Value);
entity.HasRange = true;
}
}
public bool RecordsExist(AbstractConnection connection, Entity entity) {
var checker = new ElasticSearchConnectionChecker(connection.Logger);
if (checker.Check(connection)) {
var client = new ElasticSearchClientFactory().Create(connection, entity);
const string body = @"{ _source: false, from: 0, size:1, query: { match_all: {} } }";
var response = client.Client.Search(client.Index, client.Type, body);
if (!response.Success)
return false;
var result = response.Response;
return result["hits"].total > 0;
}
return false;
}
private int GetMaxTflBatchId(string processName)
{
var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(processName));
var body = new {
aggs = new {
tflbatchid = new {
max = new {
field = "tflbatchid"
}
}
},
size = 0
};
var result = client.Client.Search(client.Index, client.Type, body);
return(Convert.ToInt32((result.Response["aggregations"]["tflbatchid"]["value"].Value ?? 0)));
}
public override void LoadBeginVersion(Entity entity)
{
var tflBatchId = GetMaxTflBatchId(entity);
if (tflBatchId > 0)
{
var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName));
var result = client.Client.SearchGet(client.Index, client.Type, s => s
.AddQueryString("q", "tflbatchid:" + tflBatchId)
.AddQueryString("_source_include", "version,version_type")
.AddQueryString("size", 1)
);
var hits = result.Response["hits"].hits;
var versionType = (string)hits[0]["_source"]["version_type"].Value;
entity.Begin = Common.GetObjectConversionMap()[versionType](hits[0]["_source"]["version"].Value);
entity.HasRange = true;
}
}
public override void LoadEndVersion(Entity entity)
{
var client = new ElasticSearchClientFactory().Create(this, entity);
var body = new {
aggs = new {
version = new {
max = new {
field = entity.Version.Alias.ToLower()
}
}
},
size = 0
};
var result = client.Client.Search(client.Index, client.Type, body);
entity.End = Common.GetObjectConversionMap()[entity.Version.SimpleType](result.Response["aggregations"]["version"]["value"].Value);
entity.HasRows = entity.End != null;
}
public void Drop(AbstractConnection connection, Entity entity)
{
if (!EntityExists.Exists(connection, entity))
{
return;
}
var client = new ElasticSearchClientFactory().Create(connection, entity);
var response = client.Client.IndicesDelete(client.Index);
if (response.Success)
{
return;
}
connection.Logger.EntityWarn(entity.Name, response.ServerError.Error);
connection.Logger.EntityWarn(entity.Name, "Trouble deleting {0} {1}.", client.Index, client.Type);
}
public void Create(AbstractConnection connection, Process process, Entity entity) {
var client = new ElasticSearchClientFactory().Create(connection, entity);
client.Client.IndicesCreate(client.Index, "{ \"settings\":{}}");
var fields = GetFields(entity);
var properties = new Dictionary<string, object>() { { "properties", fields } };
var body = new Dictionary<string, object>() { { client.Type, properties } };
var json = JsonConvert.SerializeObject(body);
var response = client.Client.IndicesPutMapping(client.Index, client.Type, json);
if (response.Success)
return;
throw new TransformalizeException(_logger, entity.Name, response.ServerError.Error);
}
public bool RecordsExist(AbstractConnection connection, Entity entity)
{
var checker = new ElasticSearchConnectionChecker(connection.Logger);
if (checker.Check(connection))
{
var client = new ElasticSearchClientFactory().Create(connection, entity);
const string body = @"{ _source: false, from: 0, size:1, query: { match_all: {} } }";
var response = client.Client.Search(client.Index, client.Type, body);
if (!response.Success)
{
return(false);
}
var result = response.Response;
return(result["hits"].total > 0);
}
return(false);
}
public bool Check(AbstractConnection connection) {
var hashCode = connection.Uri().GetHashCode();
if (Checks.ContainsKey(hashCode)) {
return Checks[hashCode];
}
var client = new ElasticSearchClientFactory().Create(connection, null);
try {
var response = client.Client.Ping();
if (response.HttpStatusCode != null && response.HttpStatusCode == 200) {
_logger.Debug("Successful ping of {0}.", connection.Name);
return true;
}
_logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, response.ServerError.Error);
} catch (Exception e) {
_logger.Warn("Failed to connect to {0}, {1}:{2}. {3}", connection.Name, connection.Server, connection.Port, e.Message);
return false;
}
return false;
}
public override Fields GetEntitySchema(Process process, Entity entity, bool isMaster = false)
{
var client = new ElasticSearchClientFactory().CreateNest(this, entity);
var mapping = client.Client.GetMapping <dynamic>(m => m
.Index(client.Index)
.Type(client.Type)
);
if (!mapping.IsValid)
{
throw new TransformalizeException(Logger, "Trouble getting mapping for {0}:{1} {2}", client.Index, client.Type, mapping.ServerError.Error);
}
var fields = new Fields();
foreach (var pair in mapping.Mapping.Properties)
{
fields.Add(new Field(pair.Value.Type.Name, "64", FieldType.None, true, "")
{
Name = pair.Key.Name
});
}
return(fields);
}
public override void WriteEndVersion(Process process, AbstractConnection input, Entity entity, bool force = false)
{
if (entity.Updates + entity.Inserts > 0 || force)
{
var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(entity.ProcessName));
var versionType = entity.Version == null ? "string" : entity.Version.SimpleType;
string end;
if (versionType.Equals("datetime") && entity.End is DateTime)
{
end = ((DateTime)entity.End).ToString("yyyy-MM-ddTHH:mm:ss.fff");
}
else if (versionType.Equals("byte[]") || versionType.Equals("rowversion"))
{
end = Common.BytesToHexString((byte[])entity.End);
}
else
{
end = new DefaultFactory(Logger).Convert(entity.End, versionType).ToString();
}
var body = new {
id = entity.TflBatchId,
tflbatchid = entity.TflBatchId,
process = entity.ProcessName,
connection = input.Name,
entity = entity.Alias,
updates = entity.Updates,
inserts = entity.Inserts,
deletes = entity.Deletes,
version = end,
version_type = versionType,
tflupdate = DateTime.UtcNow
};
client.Client.Index(client.Index, client.Type, body);
}
}
public override Fields GetEntitySchema(Process process, Entity entity, bool isMaster = false) {
var client = new ElasticSearchClientFactory().CreateNest(this, entity);
var mapping = client.Client.GetMapping<dynamic>(m => m
.Index(client.Index)
.Type(client.Type)
);
if (!mapping.IsValid) {
throw new TransformalizeException(Logger, "Trouble getting mapping for {0}:{1} {2}", client.Index, client.Type, mapping.ServerError.Error);
}
var fields = new Fields();
foreach (var pair in mapping.Mapping.Properties) {
fields.Add(new Field(pair.Value.Type.Name, "64", FieldType.None, true, "") { Name = pair.Key.Name });
}
return fields;
}
public override void LoadEndVersion(Entity entity) {
var client = new ElasticSearchClientFactory().Create(this, entity);
var body = new {
aggs = new {
version = new {
max = new {
field = entity.Version.Alias.ToLower()
}
}
},
size = 0
};
var result = client.Client.Search(client.Index, client.Type, body);
entity.End = Common.GetObjectConversionMap()[entity.Version.SimpleType](result.Response["aggregations"]["version"]["value"].Value);
entity.HasRows = entity.End != null;
}
private int GetMaxTflBatchId(string processName) {
var client = new ElasticSearchClientFactory().Create(this, TflBatchEntity(processName));
var body = new {
aggs = new {
tflbatchid = new {
max = new {
field = "tflbatchid"
}
}
},
size = 0
};
var result = client.Client.Search(client.Index, client.Type, body);
return Convert.ToInt32((result.Response["aggregations"]["tflbatchid"]["value"].Value ?? 0));
}
