public ISession GetSession()
{
if (Transaction.Current != null)
{
// Currently in case of DTC the consumer and produce of messages use an own session due to a bug in the ActiveMQ NMS client:
// https://issues.apache.org/jira/browse/AMQNET-405 . When this issue is resolved then we should return the same session within
// a DTC transaction to be able to use Single Phase Commits in case no other systems are involved in the transaction for better
// performance.
return(sessionsForTransactions.GetOrAdd(
Transaction.Current.TransactionInformation.LocalIdentifier, id => GetSessionForTransaction()));
}
return(pooledSessionFactory.GetSession());
}
// private static readonly string EnsureTableQuery = @"SELECT * FROM {0}.{1}";
public static async Task EnsureTableAsync(ISessionFactory sessionFactory, ReplicationTable table)
{
var session = sessionFactory.GetSession();
try
{
Logger.Info($"Creating Schema... {table.SchemaName}");
await session.Execute($"CREATE KEYSPACE IF NOT EXISTS {table.SchemaName} WITH REPLICATION = {{'class' : 'SimpleStrategy', 'replication_factor' : 1}};");
var rows = await session.Execute(string.Format(EnsureTableQuery, table.SchemaName, table.TableName));
Logger.Info($"Creating Table: {string.Format(EnsureTableQuery, table.SchemaName, table.TableName)}");
// check if table exists
var count = (long)rows.First()["count"];
if (count == 0)
{
// create table
var querySb = new StringBuilder($@"CREATE TABLE IF NOT EXISTS
{Utility.Utility.GetSafeName(table.SchemaName, '"')}.{Utility.Utility.GetSafeName(table.TableName, '"')}(");
var primaryKeySb = new StringBuilder("(");
var hasPrimaryKey = false;
//get all tables which are pk, put them together like a normal cassandra upload
var pkColumnList = new List <string> {
};
foreach (var column in table.Columns)
{
querySb.Append(
// $"{Utility.Utility.GetSafeName(column.ColumnName)} {column.DataType}{(column.PrimaryKey ? " PRIMARY KEY" : "")},");
$"{Utility.Utility.GetSafeName(column.ColumnName)} {column.DataType},");
if (column.PrimaryKey)
{
pkColumnList.Add(column.ColumnName);
// primaryKeySb.Append($"{Utility.Utility.GetSafeName(column.ColumnName)},");
hasPrimaryKey = true;
}
}
if (pkColumnList.Count > 0)
{
querySb.Append($"PRIMARY KEY (\"{string.Join("\",\"", pkColumnList)}\"),");
}
querySb.Length--;
querySb.Append(");");
var query = querySb.ToString();
Logger.Info($"Creating Table: {query}");
await session.Execute(query);
}
}
finally
{
//noop
}
}
public ActionResult <List <TestEntity> > Get()
{
var items = _sessionFactory.GetSession().Query <TestEntity>("select * from Test").ToList();
var list = new List <TestEntity>();
list.Add(new TestEntity()
{
Id = 1, Name = "Hello1", CreateTime = DateTime.Now
});
list.Add(new TestEntity()
{
Id = 2, Name = "Hello2", CreateTime = DateTime.Now
});
list.Add(new TestEntity()
{
Id = 3, Name = "Hello3", CreateTime = DateTime.Now
});
list.Add(new TestEntity()
{
Id = 4, Name = "Hello4", CreateTime = DateTime.Now
});
list.Add(new TestEntity()
{
Id = 5, Name = "Hello5", CreateTime = DateTime.Now
});
return(items);
}
public static async Task <Schema> GetRefreshSchemaForTable(ISessionFactory sessionFactory, Schema schema,
int sampleSize = 5)
{
var decomposed = DecomposeSafeName(schema.Id).TrimEscape();
var session = sessionFactory.GetSession();
var rows = await session.Execute(string.Format(GetTableAndColumnsQuery, decomposed.Schema, decomposed.Table));
var refreshProperties = new List <Property>();
foreach (var row in rows)
{
var property = new Property
{
Id = row[2].ToString(),
Name = row[2].ToString(),
IsKey = (row[4].ToString() == "partition_key" || row[4].ToString() == "clustering"),
IsNullable = !(row[4].ToString() == "partition_key" || row[4].ToString() == "clustering"),
Type = GetType(row[3].ToString()),
TypeAtSource = row[3].ToString(),
};
refreshProperties.Add(property);
}
// add properties
schema.Properties.Clear();
schema.Properties.AddRange(refreshProperties);
// get sample and count
return(await AddSampleAndCount(sessionFactory, schema, sampleSize));
}
public static async Task DropTableAsync(ISessionFactory sessionFactory, ReplicationTable table)
{
var session = sessionFactory.GetSession();
var rows = await session.Execute(string.Format(DropTableQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"')
));
}
public ISession Connect()
{
_sessionFactory = _sessionFactory ?? new DefualtSessionFactory();
var _session = _sessionFactory.GetSession(_sessionType);
_initAction(_session);
_session.Pipeline.AddOutbound("SessionSend", new SessionSendHandler(_session));
_session.Connect(Address, Port);
return(_session);
}
public ISession GetSession()
{
ISession session;
if (sessionsForThreads.TryGetValue(Thread.CurrentThread.ManagedThreadId, out session))
{
return(session);
}
return(pooledSessionFactory.GetSession());
}
public static async IAsyncEnumerable <Schema> GetAllSchemas(ISessionFactory sessionFactory, int sampleSize = 5)
{
var session = sessionFactory.GetSession();
var rows = await session.Execute(GetAllTablesAndColumnsQuery);
Schema schema = null;
var currentSchemaId = "";
foreach (var row in rows)
{
//keyspace.table
var schemaId = $"\"{row[0]}\".\"{row[1]}\"";
if (schemaId != currentSchemaId)
{
//return previous schema
if (schema != null)
{
// get sample and count
yield return(await AddSampleAndCount(sessionFactory, schema, sampleSize));
}
// start new schema
currentSchemaId = schemaId;
var parts = DecomposeSafeName(currentSchemaId).TrimEscape();
schema = new Schema
{
Id = currentSchemaId,
Name = $"{parts.Schema}.{parts.Table}",
Properties = { },
DataFlowDirection = Schema.Types.DataFlowDirection.Read
};
}
var property = new Property
{
Id = row[2].ToString(),
Name = row[2].ToString(),
IsKey = (row[4].ToString() == "partition_key" || row[4].ToString() == "clustering"),
IsNullable = !(row[4].ToString() == "partition_key" || row[4].ToString() == "clustering"),
Type = GetType(row[3].ToString()),
TypeAtSource = row[3].ToString(),
// reader.GetValueById(CharacterMaxLength))
};
schema?.Properties.Add(property);
}
if (schema != null)
{
// get sample and count
yield return(await AddSampleAndCount(sessionFactory, schema, sampleSize));
}
}
public static async Task DeleteRecordAsync(ISessionFactory sessionFactory, ReplicationTable table,
string primaryKeyValue)
{
var session = sessionFactory.GetSession();
await session.Execute(string.Format(DeleteRecordQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"'),
Utility.Utility.GetSafeName(table.Columns.Find(c => c.PrimaryKey == true).ColumnName, '"'),
primaryKeyValue));
}
public static async Task <bool> RecordExistsAsync(ISessionFactory sessionFactory, ReplicationTable table,
string primaryKeyValue)
{
var session = sessionFactory.GetSession();
var rows = await session.Execute(string.Format(RecordExistsQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"'),
Utility.Utility.GetSafeName(table.Columns.Find(c => c.PrimaryKey == true).ColumnName, '"'),
primaryKeyValue));
var count = (long)rows.First()["count"];
return(count != 0);
}
public static async Task <ReplicationMetaData> GetPreviousReplicationMetaDataAsync(
ISessionFactory sessionFactory,
string jobId,
ReplicationTable table)
{
try
{
ReplicationMetaData replicationMetaData = null;
// ensure replication metadata table
await EnsureTableAsync(sessionFactory, table);
// check if metadata exists
var session = sessionFactory.GetSession();
var rows = await session.Execute(string.Format(GetMetaDataQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"'),
Utility.Utility.GetSafeName(Constants.ReplicationMetaDataJobId, '"'),
jobId));
foreach (var row in rows)
{
var request = JsonConvert.DeserializeObject <PrepareWriteRequest>(row[Constants.ReplicationMetaDataRequest].ToString());
var shapeName = row[Constants.ReplicationMetaDataReplicatedShapeName].ToString();
var shapeId = row[Constants.ReplicationMetaDataReplicatedShapeId].ToString();
var timestamp = DateTime.Parse(row[Constants.ReplicationMetaDataTimestamp].ToString());
replicationMetaData = new ReplicationMetaData
{
Request = request,
ReplicatedShapeName = shapeName,
ReplicatedShapeId = shapeId,
Timestamp = timestamp
};
}
return(replicationMetaData);
}
catch (Exception e)
{
Logger.Error(e, e.Message);
throw;
}
}
public static async IAsyncEnumerable <Schema> GetRefreshSchemas(ISessionFactory sessionFactory,
RepeatedField <Schema> refreshSchemas, int sampleSize = 5)
{
var session = sessionFactory.GetSession();
foreach (var schema in refreshSchemas)
{
if (string.IsNullOrWhiteSpace(schema.Query))
{
yield return(await GetRefreshSchemaForTable(sessionFactory, schema, sampleSize));
continue;
}
var rows = await session.Execute(schema.Query);
var properties = new List <Property>();
foreach (var column in rows.Columns)
{
//Query system_schema.columns for col info
var colInfo = await session.Execute(string.Format(ColumnInfoQuery, column.Keyspace, column.Table, column.Name));
var row = colInfo.First();
var property = new Property
{
Id = row[2].ToString(),
Name = row[2].ToString(),
IsKey = (row[4].ToString() == "partition_key" || row[4].ToString() == "clustering"),
IsNullable = !(row[4].ToString() == "partition_key" || row[4].ToString() == "clustering"),
Type = GetType(row[3].ToString()),
TypeAtSource = row[3].ToString(),
};
properties.Add(property);
}
schema.Properties.Clear();
schema.Properties.AddRange(properties);
// get sample and count
yield return(await AddSampleAndCount(sessionFactory, schema, sampleSize));
}
}
public static async Task UpsertReplicationMetaDataAsync(ISessionFactory sessionFactory, ReplicationTable table,
ReplicationMetaData metaData)
{
var session = sessionFactory.GetSession();
try
{
await session.Execute(string.Format(InsertMetaDataQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"'),
metaData.Request.DataVersions.JobId,
// JsonConvert.SerializeObject(metaData.Request).Replace("\\", "\\\\"),
JsonConvert.SerializeObject(metaData.Request),
metaData.ReplicatedShapeId,
metaData.ReplicatedShapeName,
metaData.Timestamp
));
}
catch (Exception e)
{
try
{
session.Execute(string.Format(UpdateMetaDataQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"'),
JsonConvert.SerializeObject(metaData.Request),
metaData.ReplicatedShapeId,
metaData.ReplicatedShapeName,
metaData.Timestamp,
metaData.Request.DataVersions.JobId
));
// update if it failed
}
catch (Exception exception)
{
Logger.Error(e, $"Error Insert: {e.Message}");
Logger.Error(exception, $"Error Update: {exception.Message}");
throw;
}
}
}
public void SendMessage(TransportMessage message, string destination, string destinationPrefix)
{
var session = sessionFactory.GetSession();
try
{
var jmsMessage = activeMqMessageMapper.CreateJmsMessage(message, session);
using (var producer = session.CreateProducer())
{
producer.Send(destinationEvaluator.GetDestination(session, destination, destinationPrefix), jmsMessage);
}
// We assign here the Id to the underlying id which was chosen by the broker.
// TODO: Why do we need this daniel/remo?
//message.Id = jmsMessage.NMSMessageId;
}
finally
{
sessionFactory.Release(session);
}
}
public static async Task <Dictionary <string, object> > GetRecordAsync(ISessionFactory sessionFactory,
ReplicationTable table,
string primaryKeyValue)
{
var session = sessionFactory.GetSession();
var rows = await session.Execute(string.Format(GetRecordQuery,
Utility.Utility.GetSafeName(table.SchemaName, '"'),
Utility.Utility.GetSafeName(table.TableName, '"'),
Utility.Utility.GetSafeName(table.Columns.Find(c => c.PrimaryKey == true).ColumnName, '"'),
primaryKeyValue));
Dictionary <string, object> recordMap = null;
// check if record exists
foreach (var row in rows)
{
recordMap = new Dictionary <string, object>();
foreach (var column in table.Columns)
{
try
{
recordMap[column.ColumnName] = row[column.ColumnName];
}
catch (Exception e)
{
Logger.Error(e, $"No column with column name: {column.ColumnName}");
Logger.Error(e, e.Message);
recordMap[column.ColumnName] = null;
}
}
}
return(recordMap);
}
public static async Task <Count> GetCountOfRecords(ISessionFactory sessionFactory, Schema schema)
{
//var conn = connFactory.GetConnection();
var session = sessionFactory.GetSession();
try
{
var rows = await session.Execute($"SELECT COUNT(*) FROM {schema.Id}");
var count = -1;
foreach (var row in rows)
{
count = Int32.Parse(row["count"].ToString());
}
return(count == -1
? new Count
{
Kind = Count.Types.Kind.Unavailable,
}
: new Count
{
Kind = Count.Types.Kind.Exact,
Value = count
});
}
catch (Exception e)
{
return(new Count
{
Kind = Count.Types.Kind.Unavailable,
});
//noop
}
}
public static async IAsyncEnumerable <Record> ReadRecords(ISessionFactory sessionFactory, Schema schema)
{
var session = sessionFactory.GetSession();
try
{
var query = schema.Query;
if (string.IsNullOrWhiteSpace(query))
{
query = $"SELECT * FROM {schema.Id}";
}
RowSet rows;
try
{
rows = await session.Execute(query);
}
catch (Exception e)
{
Logger.Error(e, e.Message);
yield break;
}
if (rows != null)
{
foreach (var row in rows)
{
var recordMap = new Dictionary <string, object>();
foreach (var property in schema.Properties)
{
try
{
switch (property.Type)
{
case PropertyType.String:
case PropertyType.Text:
case PropertyType.Decimal:
recordMap[property.Id] = row[property.Id].ToString();
break;
default:
recordMap[property.Id] = row[property.Id];
break;
}
}
catch (Exception e)
{
Logger.Error(e, $"No column or no data with property Id: {property.Id}");
Logger.Error(e, e.Message);
recordMap[property.Id] = null;
}
}
var record = new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(recordMap)
};
yield return(record);
}
// while (await reader.ReadAsync())
// {
// var recordMap = new Dictionary<string, object>();
//
// foreach (var property in schema.Properties)
// {
// try
// {
// switch (property.Type)
// {
// case PropertyType.String:
// case PropertyType.Text:
// case PropertyType.Decimal:
// recordMap[property.Id] = reader.GetValueById(property.Id, '`').ToString();
// break;
// default:
// recordMap[property.Id] = reader.GetValueById(property.Id, '`');
// break;
// }
// }
// catch (Exception e)
// {
// Logger.Error(e, $"No column with property Id: {property.Id}");
// Logger.Error(e, e.Message);
// recordMap[property.Id] = null;
// }
// }
//
// var record = new Record
// {
// Action = Record.Types.Action.Upsert,
// DataJson = JsonConvert.SerializeObject(recordMap)
// };
//
// yield return record;
// }
}
}
finally
{
//Noop
}
}
public virtual void Start(TransactionSettings transactionSettings)
{
this.transactionSettings = transactionSettings;
session = sessionFactory.GetSession();
}
public static async Task UpsertRecordAsync(ISessionFactory sessionFactory,
ReplicationTable table,
Dictionary<string, object> recordMap)
{
var session = sessionFactory.GetSession();
try
{
// try to insert
var querySb =
new StringBuilder(
$"INSERT INTO {Utility.Utility.GetSafeName(table.SchemaName, '"')}.{Utility.Utility.GetSafeName(table.TableName, '"')}(");
foreach (var column in table.Columns)
{
querySb.Append($"{Utility.Utility.GetSafeName(column.ColumnName, '"')},");
}
querySb.Length--;
querySb.Append(") VALUES (");
foreach (var column in table.Columns)
{
if (recordMap.ContainsKey(column.ColumnName))
{
var rawValue = recordMap[column.ColumnName];
if (column.Serialize)
{
rawValue = JsonConvert.SerializeObject(rawValue);
}
switch (column.DataType)
{
case("bigint"):
case("smallint"):
case("varint"):
case("int"):
case("decimal"):
case("float"):
case("counter"):
querySb.Append(rawValue != null
? $"{Utility.Utility.GetSafeString(rawValue.ToString())},"
: $"NULL,");
break;
default:
querySb.Append(rawValue != null
? $"'{Utility.Utility.GetSafeString(rawValue.ToString(), "\"")}',"
: $"NULL,");
break;
}
}
else
{
querySb.Append($"NULL,");
}
}
querySb.Length--;
querySb.Append(");");
var query = querySb.ToString();
Logger.Debug($"Insert record query: {query}");
await session.Execute(query);
}
catch (Exception e)
{
try
{
// update if it failed
var querySb =
new StringBuilder(
$"UPDATE {Utility.Utility.GetSafeName(table.SchemaName, '"')}.{Utility.Utility.GetSafeName(table.TableName, '"')} SET ");
foreach (var column in table.Columns)
{
if (!column.PrimaryKey)
{
if (recordMap.ContainsKey(column.ColumnName))
{
var rawValue = recordMap[column.ColumnName];
if (column.Serialize)
{
rawValue = JsonConvert.SerializeObject(rawValue);
}
if (rawValue != null)
{
querySb.Append(
$"{Utility.Utility.GetSafeName(column.ColumnName, '"')}='{Utility.Utility.GetSafeString(rawValue.ToString(), "'", "''")}',");
}
else
{
querySb.Append($"{Utility.Utility.GetSafeName(column.ColumnName, '"')}=NULL,");
}
}
else
{
querySb.Append($"{Utility.Utility.GetSafeName(column.ColumnName, '"')}=NULL,");
}
}
}
querySb.Length--;
var primaryKey = table.Columns.Find(c => c.PrimaryKey);
var primaryValue = recordMap[primaryKey.ColumnName];
if (primaryKey.Serialize)
{
primaryValue = JsonConvert.SerializeObject(primaryValue);
}
querySb.Append($" WHERE {primaryKey.ColumnName} = '{primaryValue}'");
var query = querySb.ToString();
await session.Execute(query);
}
catch (Exception exception)
{
Logger.Error(e, $"Error Insert: {e.Message}");
Logger.Error(exception, $"Error Update: {exception.Message}");
throw;
}
}
}
