public void ChangeFeedPull()
{
// Connect to cassandra cluster (Cassandra API on Azure Cosmos DB supports only TLSv1.2)
var options = new Cassandra.SSLOptions(SslProtocols.Tls12, true, ValidateServerCertificate);
options.SetHostNameResolver((ipAddress) => CassandraContactPoint);
Cluster cluster = Cluster.Builder().WithCredentials(UserName, Password).WithPort(CassandraPort).AddContactPoint(CassandraContactPoint).WithSSL(options).Build();
session = cluster.Connect();
Setup(); //drop and create keyspace and table
session = cluster.Connect("uprofile");
IMapper mapper = new Mapper(session);
Console.WriteLine("pulling from change feed: ");
//set initial start time for pulling the change feed
DateTime timeBegin = DateTime.UtcNow;
//initialise variable to store the continuation token
byte[] pageState = null;
while (true)
{
try
{
IStatement changeFeedQueryStatement = new SimpleStatement(
$"SELECT * FROM uprofile.user where COSMOS_CHANGEFEED_START_TIME() = '{timeBegin.ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture)}'");
if (pageState != null)
{
changeFeedQueryStatement = changeFeedQueryStatement.SetPagingState(pageState);
}
Console.WriteLine("getting records from change feed at last page state....");
RowSet rowSet = session.Execute(changeFeedQueryStatement);
//store the continuation token here
pageState = rowSet.PagingState;
List <Row> rowList = rowSet.ToList();
if (rowList.Count != 0)
{
for (int i = 0; i < rowList.Count; i++)
{
string value = rowList[i].GetValue <string>("user_name");
int key = rowList[i].GetValue <int>("user_id");
// do something with the data - e.g. compute, forward to another event, function, etc.
// here, we just print the user name field
Console.WriteLine("user_name: " + value);
}
}
else
{
Console.WriteLine("zero documents read");
}
Thread.Sleep(300);
}
catch (Exception e)
{
Console.WriteLine("Exception " + e);
}
}
}
public void Statement_SetPagingState_Null_Does_Not_Disable_AutoPage()
{
var statement = new SimpleStatement();
Assert.True(statement.AutoPage);
statement.SetPagingState(null);
Assert.True(statement.AutoPage);
Assert.Null(statement.PagingState);
}
public void Statement_SetPagingState_Disables_AutoPage()
{
var statement = new SimpleStatement();
Assert.True(statement.AutoPage);
statement.SetPagingState(new byte[] { 1, 2, 3, 4, 5, 6 });
Assert.False(statement.AutoPage);
Assert.NotNull(statement.PagingState);
}
public void Statement_SetPagingState_Null_Does_Not_Disable_AutoPage()
{
var statement = new SimpleStatement();
Assert.True(statement.AutoPage);
statement.SetPagingState(null);
Assert.True(statement.AutoPage);
Assert.Null(statement.PagingState);
}
public void Statement_SetPagingState_Disables_AutoPage()
{
var statement = new SimpleStatement();
Assert.True(statement.AutoPage);
statement.SetPagingState(new byte[] { 1, 2, 3, 4, 5, 6 });
Assert.False(statement.AutoPage);
Assert.NotNull(statement.PagingState);
}
public async Task StartAsync(CancellationToken cancellationToken)
{
Cluster cluster = _cosmosDBCassandraService.GetCluster();
session = cluster.Connect(_keyspace);
//set initial start time for pulling the change feed
DateTime timeBegin = this._startFromBeginning ? DateTime.MinValue.ToUniversalTime() : DateTime.UtcNow;
//initialise variable to store the continuation token
byte[] pageState = null;
_logger.LogInformation(string.Format($"Reading from Cassandra API change feed..."));
while (!cancellationTokenSource.IsCancellationRequested)
{
try
{
IStatement changeFeedQueryStatement = new SimpleStatement(
$"SELECT * FROM " + _keyspace + "." + _table + $" where COSMOS_CHANGEFEED_START_TIME() = '{timeBegin.ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture)}'");
if (pageState != null)
{
changeFeedQueryStatement = changeFeedQueryStatement.SetPagingState(pageState);
}
RowSet rowSet = session.Execute(changeFeedQueryStatement);
pageState = rowSet.PagingState;
TimeSpan wait = _defaultTimeSpan;
if (rowSet.IsFullyFetched)
{
List <Row> rowList = rowSet.ToList();
CqlColumn[] columns = rowSet.Columns;
if (rowList.Count != 0)
{
//convert Cassandra resultset to JArray
List <JArray> rows = new List <JArray>();
for (int i = 0; i < rowList.Count; i++)
{
JArray row = new JArray();
JObject jcolumns = new JObject();
foreach (CqlColumn col in columns)
{
//add column names and values extracted from rowList to JObject
jcolumns.Add(new JProperty(col.Name, rowList[i].GetValue <dynamic>(col.Name)));
}
//add the JObject to the JArray
row.Add(jcolumns);
//add the JArray to the JArray List
rows.Add(row);
//_logger.LogInformation("row: " + row.ToString());
}
_logger.LogInformation("processing change...");
await _executor.TryExecuteAsync(new TriggeredFunctionData()
{
TriggerValue = rows
}, cancellationToken);
wait = TimeSpan.Zero; // If there were changes, we want to capture the next batch right away with no delay
}
}
await Task.Delay(wait, cancellationTokenSource.Token);
}
catch (TaskCanceledException e)
{
_logger.LogWarning(e, "Task cancelled");
}
catch (Exception e)
{
_logger.LogError(e, "Error on change feed cycle");
}
}
}
