public async Task Redis_Should_AddAndGetSimpleItem()
{
var ds = GetValidStream();
var record = new DataStreamRecord()
{
DeviceId = DEVICE_ID,
StreamId = ds.Id,
StreamKey = ds.Key
};
var originalTimeStamp = DateTime.UtcNow.ToJSONString();
record.Timestamp = originalTimeStamp;
record.Data.Add("int1", 100);
record.Data.Add("dec1", 100.12);
record.Data.Add("str1", "hello world");
Assert.IsTrue((await _redisConnector.AddItemAsync(record)).Successful);
var result = await _redisConnector.GetItemsAsync(DEVICE_ID, new Core.Models.UIMetaData.ListRequest());
Assert.IsTrue(result.Successful);
Assert.AreEqual(1, result.Model.Count());
Assert.AreEqual(originalTimeStamp, result.Model.First().Timestamp);
Assert.AreEqual(Convert.ToInt32(100), Convert.ToInt32(result.Model.First()["int1"]));
Assert.AreEqual(Math.Round(100.12, 2), Math.Round(Convert.ToDouble(result.Model.First()["dec1"]), 2));
Assert.AreEqual("hello world", result.Model.First()["str1"]);
}
public async Task Redis_Should_Add_OnUpdateIfItemDoesNotExists()
{
var ds = GetValidStream();
var record = new DataStreamRecord()
{
DeviceId = Guid.NewGuid().ToId(),
StreamId = ds.Id,
StreamKey = ds.Key
};
var originalTimeStamp = DateTime.UtcNow.ToJSONString();
record.Timestamp = originalTimeStamp;
record.Data.Add("int1", 100);
record.Data.Add("dec1", 100.12);
record.Data.Add("str1", "hello world");
Assert.IsTrue((await _redisConnector.UpdateItem(record.Data, new Dictionary <string, object>()
{
{ "deviceId", record.DeviceId }
})).Successful);
var updatedResult = await _redisConnector.GetItemsAsync(record.DeviceId, new Core.Models.UIMetaData.ListRequest());
Assert.IsTrue(updatedResult.Successful);
Assert.AreEqual(1, updatedResult.Model.Count());
Assert.AreEqual(Convert.ToInt32(100), Convert.ToInt32(updatedResult.Model.First()["int1"]));
Assert.AreEqual(Math.Round(100.12, 2), Math.Round(Convert.ToDouble(updatedResult.Model.First()["dec1"]), 2));
Assert.AreEqual("hello world", updatedResult.Model.First()["str1"]);
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item, EntityHeader org, EntityHeader user)
{
var db = _redis.GetDatabase();
await db.SetAddAsync(GetKey(item.DeviceId, org, user), JsonConvert.SerializeObject(item, _camelCaseSettings));
return(InvokeResult.Success);
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var recordId = DateTime.UtcNow.ToInverseTicksRowKey();
item.Data.Add(_stream.TimeStampFieldName, item.GetTimeStampValue(_stream));
item.Data.Add("sortOrder", item.GetTicks());
item.Data.Add("deviceId", item.DeviceId);
item.Data.Add("id", recordId);
item.Data.Add("dataStreamId", _stream.Id);
var result = await _client.IndexAsync(item.Data,
idx => idx
.Index(_stream.ESIndexName)
.Type(_stream.ESTypeName)
.Id(recordId));
if (result.IsValid)
{
return(InvokeResult.Success);
}
else
{
if (result.OriginalException != null)
{
return(InvokeResult.FromError(result.OriginalException.Message));
}
else
{
return(InvokeResult.FromError(result.ServerError.Error.Reason));
}
}
}
public Task <InvokeResult> AddItemAsync(DataStreamRecord item, EntityHeader org, EntityHeader user)
{
item.Data.Add("orgId", org.Id);
item.Data.Add("orgName", org.Text);
item.Data.Add("userId", user.Id);
item.Data.Add("userName", user.Text);
return(AddItemAsync(item));
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var db = _redis.GetDatabase();
var key = GetKey(item.DeviceId);
var json = JsonConvert.SerializeObject(item, _camelCaseSettings);
await db.StringSetAsync(key, json);
return(InvokeResult.Success);
}
public static object GetTimeStampValue(this DataStreamRecord value, DataStream stream)
{
var recordTimeStamp = String.IsNullOrEmpty(value.Timestamp) ? DateTime.UtcNow : value.Timestamp.ToDateTime();
if (stream.DateStorageFormat.Value == DateStorageFormats.Epoch)
{
return(new DateTimeOffset(recordTimeStamp).ToUnixTimeSeconds());
}
else
{
return(recordTimeStamp.ToJSONString());
}
}
public async Task Should_Insert_Point_Array()
{
try
{
var ds = GetStream();
await _dataStreamManager.AddDataStreamAsync(ds, _org, _user);
ds.DbPassword = (await _secureStorage.GetSecretAsync(_org, ds.DBPasswordSecureId, _user)).Result;
var connector = new PointArrayPostgresqlConnector(new AdminLogger(new LogWriter()));
var initResult = await connector.InitAsync(ds);
if (!initResult.Successful)
{
Console.WriteLine(initResult.Errors[0].Details);
throw new InvalidOperationException(initResult.Errors[0].Details);
}
var pointCount = 305;
var pointInterval = 1.0;
var min = 9.0;
var max = 12.0;
var rnd = new Random();
var points = new List <double>();
for (int idx = 0; idx < pointCount; ++idx)
{
var point = (rnd.NextDouble() * (max - min)) + min;
points.Add(point);
}
var secondsFromEpoch = (DateTime.UtcNow.AddSeconds(-(pointCount * pointInterval)) - epoch).TotalSeconds;
var record = new DataStreamRecord();
record.Data.Add("starttimestamp", Convert.ToInt64(secondsFromEpoch));
record.Data.Add("pointcount", pointCount);
record.Data.Add("interval", pointInterval);
record.Data.Add("sensorindex", 1);
record.Data.Add("pointarray", JsonConvert.SerializeObject(points));
record.DeviceId = DEVICE_ID;
await connector.AddItemAsync(record);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}
public async Task <InvokeResult> UpdateItem(Dictionary <string, object> items, Dictionary <string, object> recordFilter)
{
if (!recordFilter.ContainsKey("deviceId"))
{
throw new Exception("Does not contain device id as filter to update item, this is required.");
}
var deviceId = recordFilter["deviceId"].ToString();
var db = _redis.GetDatabase();
var timeStamp = recordFilter.ContainsKey("timestamp") ? recordFilter["timestamp"].ToString() : DateTime.UtcNow.ToJSONString();
var existingRecord = await GetItemsAsync(deviceId, new ListRequest());
if (existingRecord.Model.Count() == 0)
{
var newRecord = new DataStreamRecord();
newRecord.Data = items;
newRecord.DeviceId = deviceId;
newRecord.Timestamp = timeStamp;
await AddItemAsync(newRecord);
}
else
{
var record = existingRecord.Model.First();
foreach (var itm in items)
{
if (record.ContainsKey(itm.Key))
{
record[itm.Key] = itm.Value;
}
else
{
record.Add(itm.Key, itm.Value);
}
}
var newRecord = new DataStreamRecord();
newRecord.Data = record;
newRecord.DeviceId = deviceId;
newRecord.Timestamp = timeStamp;
await AddItemAsync(newRecord);
}
return(InvokeResult.Success);
}
public async Task Redis_Should_AddSimpleItem()
{
var ds = GetValidStream();
var record = new DataStreamRecord()
{
DeviceId = DEVICE_ID,
StreamId = ds.Id,
StreamKey = ds.Key
};
record.Timestamp = DateTime.UtcNow.ToJSONString();
record.Data.Add("int1", 100);
record.Data.Add("dec1", 100.12);
record.Data.Add("str1", "hello world");
Assert.IsTrue((await _redisConnector.AddItemAsync(record)).Successful);
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var recordId = DateTime.UtcNow.ToInverseTicksRowKey();
if (String.IsNullOrEmpty(item.Timestamp))
{
switch (_stream.DateStorageFormat.Value)
{
case DateStorageFormats.Epoch:
item.Data.Add(_stream.TimestampFieldName, DateTimeOffset.Now.ToUnixTimeSeconds());
break;
case DateStorageFormats.ISO8601:
item.Data.Add(_stream.TimestampFieldName, DateTime.UtcNow.ToJSONString());
break;
}
}
item.Data.Add("id", recordId);
item.Data.Add("dataStreamId", _stream.Id);
item.Data.Add(_stream.DeviceIdFieldName, item.DeviceId);
var obj = new PutObjectRequest()
{
BucketName = _stream.S3BucketName,
Key = recordId,
ContentBody = JsonConvert.SerializeObject(item.Data)
};
obj.ContentType = "application/json";
obj.Metadata.Add("title", recordId);
try
{
await _s3Client.PutObjectAsync(obj);
return(InvokeResult.Success);
}
catch (AmazonS3Exception amazonS3Exception)
{
_logger.AddException("AWSS3Connector_AddItem", amazonS3Exception);
return(InvokeResult.FromException("AWSS3Connector_AddItem", amazonS3Exception));
}
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var recordId = DateTime.UtcNow.ToInverseTicksRowKey();
item.Data.Add(_stream.TimeStampFieldName, item.GetTimeStampValue(_stream));
item.Data.Add("sortOrder", item.GetTicks());
item.Data.Add("deviceId", item.DeviceId);
item.Data.Add("id", recordId);
item.Data.Add("dataStreamId", _stream.Id);
var fileName = $"{recordId}.json";
var blob = _container.GetBlockBlobReference(fileName);
blob.Properties.ContentType = "application/json";
var json = JsonConvert.SerializeObject(item.Data);
var numberRetries = 5;
var retryCount = 0;
var completed = false;
while (retryCount++ < numberRetries && !completed)
{
try
{
await blob.UploadTextAsync(json);
}
catch (Exception ex)
{
if (retryCount == numberRetries)
{
_instanceLogger.AddException("AzureBlobConnector_AddItemAsync", ex);
return(InvokeResult.FromException("AzureBlobConnector_AddItemAsync", ex));
}
else
{
_instanceLogger.AddCustomEvent(LagoVista.Core.PlatformSupport.LogLevel.Warning, "AzureBlobConnector_AddItemAsync", "", ex.Message.ToKVP("exceptionMessage"), ex.GetType().Name.ToKVP("exceptionType"), retryCount.ToString().ToKVP("retryCount"));
}
await Task.Delay(retryCount * 250);
}
}
return(InvokeResult.Success);
}
public static DataStreamTSEntity FromDeviceStreamRecord(DataStream stream, DataStreamRecord record)
{
if (String.IsNullOrEmpty(record.Timestamp))
{
record.Timestamp = DateTime.UtcNow.ToJSONString();
}
var tsEntity = new DataStreamTSEntity()
{
PartitionKey = record.DeviceId,
Data = record.Data,
RowKey = record.Timestamp.ToDateTime().ToInverseTicksRowKey(),
};
tsEntity.Timestamp = DateTimeOffset.UtcNow;
tsEntity.Data.Add(stream.TimestampFieldName, record.GetTimeStampValue(stream));
tsEntity.Data.Add(stream.DeviceIdFieldName, record.DeviceId);
return(tsEntity);
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var recordId = DateTime.UtcNow.ToInverseTicksRowKey();
item.Data.Add(_stream.TimeStampFieldName, item.GetTimeStampValue(_stream));
item.Data.Add("sortOrder", item.GetTicks());
item.Data.Add("deviceId", item.DeviceId);
item.Data.Add("id", recordId);
item.Data.Add("dataStreamId", _stream.Id);
var json = JsonConvert.SerializeObject(item.Data);
var buffer = Encoding.UTF8.GetBytes(json);
var eventData = new EventData(buffer);
var numberRetries = 5;
var retryCount = 0;
var completed = false;
while (retryCount++ < numberRetries && !completed)
{
try
{
await _eventHubClient.SendAsync(eventData);
}
catch (Exception ex)
{
if (retryCount == numberRetries)
{
_instanceLogger.AddException("AzureTableStorageConnector_GetItemsAsync", ex);
return(InvokeResult.FromException("AzureBlobConnector_AddItemAsync", ex));
}
else
{
_instanceLogger.AddCustomEvent(LagoVista.Core.PlatformSupport.LogLevel.Warning, "AzureTableStorageConnector_GetItemsAsync", "", ex.Message.ToKVP("exceptionMessage"), ex.GetType().Name.ToKVP("exceptionType"), retryCount.ToString().ToKVP("retryCount"));
}
await Task.Delay(retryCount * 250);
}
}
return(InvokeResult.Success);
}
public async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var fields = String.Empty;
var values = String.Empty;
foreach (var fld in _stream.Fields)
{
/* validation should happen long before this point, however if someone manipulated the value, it could be very, very bad
* with a SQL injection attack, so error on the side of caution and never let it get through.
*/
if (!Validator.Validate(fld).Successful)
{
throw new Exception($"Invalid field name {fld.FieldName}");
}
fields += String.IsNullOrEmpty(fields) ? $"{fld.FieldName}" : $",{fld.FieldName}";
values += String.IsNullOrEmpty(values) ? $"@{fld.FieldName}" : $",@{fld.FieldName}";
}
fields += $",{_stream.DeviceIdFieldName},{_stream.TimeStampFieldName}";
values += $",@{_stream.DeviceIdFieldName},@{_stream.TimeStampFieldName}";
var sql = $"insert into [{_stream.DBTableName}] ({fields}) values ({values})";
using (var cn = new System.Data.SqlClient.SqlConnection(_connectionString))
using (var cmd = new System.Data.SqlClient.SqlCommand(sql, cn))
{
cmd.CommandType = System.Data.CommandType.Text;
foreach (var field in _stream.Fields)
{
object value = System.DBNull.Value;
if (item.Data.ContainsKey(field.FieldName))
{
value = item.Data[field.FieldName];
if (value == null)
{
value = System.DBNull.Value;
}
}
if (value != System.DBNull.Value && field.FieldType.Value == DeviceAdmin.Models.ParameterTypes.GeoLocation)
{
var geoParts = value.ToString().Split(',');
if (geoParts.Count() != 2)
{
return(InvokeResult.FromError($"Attmept to insert invalid geo code {value}"));
}
// Note geo codes ares stored HH.MMMMMM,HH.MMMMMM where lat comes first, SQL expects those to come lon then lat
var parameter = new SqlParameter($"@{field.FieldName}", $"POINT({geoParts[1]} {geoParts[0]})")
{
Direction = ParameterDirection.Input,
};
cmd.Parameters.Add(parameter);
}
else
{
cmd.Parameters.AddWithValue($"@{field.FieldName}", value);
}
}
if (String.IsNullOrEmpty(item.Timestamp))
{
item.Timestamp = DateTime.UtcNow.ToJSONString();
}
cmd.Parameters.AddWithValue($"@{_stream.TimeStampFieldName}", item.Timestamp.ToDateTime());
cmd.Parameters.AddWithValue($"@{_stream.DeviceIdFieldName}", item.DeviceId);
await cn.OpenAsync();
var insertResult = await cmd.ExecuteNonQueryAsync();
}
return(InvokeResult.Success);
}
public Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var tsItem = Models.DataStreamTSEntity.FromDeviceStreamRecord(_stream, item);
return(ExecWithRetry(TableOperation.Insert(tsItem)));
}
public static long GetTicks(this DataStreamRecord value)
{
var recordTimeStamp = String.IsNullOrEmpty(value.Timestamp) ? DateTime.UtcNow : value.Timestamp.ToDateTime();
return(recordTimeStamp.Ticks);
}
public Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
throw new NotImplementedException();
}
public Task <InvokeResult> AddItemAsync(DataStreamRecord item, LagoVista.Core.Models.EntityHeader org, LagoVista.Core.Models.EntityHeader user)
{
throw new NotImplementedException();
}
public async virtual Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var fields = String.Empty;
var values = String.Empty;
foreach (var fld in _stream.Fields)
{
/* validation should happen long before this point, however if someone manipulated the value, it could be very, very bad
* with a SQL injection attack, so error on the side of caution and never let it get through.
*/
if (!Validator.Validate(fld).Successful)
{
throw new Exception($"Invalid field name {fld.FieldName}");
}
if (!fld.IsDatabaseGenerated)
{
fields += String.IsNullOrEmpty(fields) ? $"{fld.FieldName}" : $",{fld.FieldName}";
values += String.IsNullOrEmpty(values) ? $"@{fld.FieldName}" : $",@{fld.FieldName}";
}
}
fields += $",{_stream.DeviceIdFieldName},{_stream.TimestampFieldName}";
values += $",@{_stream.DeviceIdFieldName},@{_stream.TimestampFieldName}";
using (var cn = OpenConnection(_stream.DbName))
using (var cmd = new NpgsqlCommand())
{
cmd.CommandText = $"insert into {_stream.DbSchema}.{_stream.DbTableName} ({fields}) values ({values})";
cmd.CommandType = System.Data.CommandType.Text;
cmd.Connection = cn;
foreach (var field in _stream.Fields)
{
if (!field.IsDatabaseGenerated)
{
object value = System.DBNull.Value;
if (item.Data.ContainsKey(field.FieldName))
{
value = item.Data[field.FieldName];
}
else if (item.Data.ContainsKey(field.Key))
{
value = item.Data[field.Key];
if (value == null)
{
value = System.DBNull.Value;
}
}
if (value != System.DBNull.Value && field.FieldType.Value == DeviceAdmin.Models.ParameterTypes.GeoLocation)
{
var geoParts = value.ToString().Split(',');
if (geoParts.Count() != 2)
{
return(InvokeResult.FromError($"Attmept to insert invalid geo code {value}"));
}
if (!Double.TryParse(geoParts[0], out double lat))
{
return(InvokeResult.FromError($"Attmept to insert invalid geo code {value}"));
}
if (!Double.TryParse(geoParts[1], out double lon))
{
return(InvokeResult.FromError($"Attmept to insert invalid geo code {value}"));
}
cmd.CommandText = cmd.CommandText.Replace($"@{field.FieldName}", $"ST_POINT({lat}, {lon})");
}
else if (value != System.DBNull.Value && field.FieldType.Value == DeviceAdmin.Models.ParameterTypes.DateTime)
{
cmd.Parameters.AddWithValue($"@{field.FieldName}", value.ToString().ToDateTime());
}
else
{
cmd.Parameters.AddWithValue($"@{field.FieldName}", value);
}
}
}
if (String.IsNullOrEmpty(item.Timestamp))
{
item.Timestamp = DateTime.UtcNow.ToJSONString();
}
cmd.Parameters.AddWithValue($"@{_stream.TimestampFieldName}", item.Timestamp.ToDateTime());
cmd.Parameters.AddWithValue($"@{_stream.DeviceIdFieldName}", item.DeviceId);
var insertResult = await cmd.ExecuteNonQueryAsync();
}
return(InvokeResult.Success);
}
public override async Task <InvokeResult> AddItemAsync(DataStreamRecord item)
{
var stream = GetStream();
if (!item.Data.ContainsKey("pointcount"))
{
return(InvokeResult.FromError("Point Array Record must contain a field called pointcount and is the number of points."));
}
if (!item.Data.ContainsKey("sensorindex"))
{
return(InvokeResult.FromError("Point Array Record must contain a field called sensorindex and is the index of the sensor."));
}
if (!item.Data.ContainsKey("interval"))
{
return(InvokeResult.FromError("Point Array Record must contain a field called interval and is the index of the sensor."));
}
if (!item.Data.ContainsKey("pointarray"))
{
return(InvokeResult.FromError("Point Array Record must contain a field called pointarray and is the points from the array."));
}
if (!item.Data.ContainsKey("starttimestamp"))
{
return(InvokeResult.FromError("Point Array Record must contain a field called starttimestamp and is the time stamp for the first point."));
}
try
{
var startTimeStamp = epoch.AddSeconds(Convert.ToUInt64(item.Data["starttimestamp"]));
var pointCount = Convert.ToInt32(item.Data["pointcount"]);
var sensorIndex = Convert.ToInt32(item.Data["sensorindex"]);
var interval = Convert.ToSingle(item.Data["interval"]);
var pointListJson = item.Data["pointarray"].ToString();
var points = JsonConvert.DeserializeObject <List <float> >(pointListJson);
if (pointCount < 1)
{
return(InvokeResult.FromError("Point count must be at least one record."));
}
if (sensorIndex < 0)
{
return(InvokeResult.FromError("Sensor Index must be a postive number."));
}
if (interval < 0)
{
return(InvokeResult.FromError("Interval must be a postiive number."));
}
if (points.Count != pointCount)
{
return(InvokeResult.FromError("Points and point count mismatch."));
}
var insertCount = 0;
using (var cn = OpenConnection(stream.DbName))
using (var cmd = new NpgsqlCommand())
{
cmd.Connection = cn;
var insertSQL = $@"insert into {stream.DbTableName} (device_id, time_stamp, sensor_index, value) values";
var timeStamp = startTimeStamp;
cmd.Parameters.AddWithValue("@device_id", item.DeviceId);
cmd.Parameters.AddWithValue("@sensor_index", sensorIndex);
var batchSize = 100;
var batchCount = pointCount / batchSize;
if (pointCount % batchSize > 0)
{
batchCount++;
}
for (int batch = 0; batch < batchCount; ++batch)
{
var bldr = new StringBuilder(insertSQL);
var startIndex = batch * batchSize;
var lastPoint = Math.Min(startIndex + batchSize, pointCount);
for (int idx = startIndex; idx < lastPoint; ++idx)
{
var valueLine = $"(@device_id, '{timeStamp.ToUniversalTime()}', @sensor_index, {points[idx]})";
bldr.AppendLine(valueLine + ((idx == lastPoint - 1) ? ";" : ","));
timeStamp = timeStamp.AddSeconds(interval);
insertCount++;
}
cmd.CommandText = bldr.ToString();
await cmd.ExecuteNonQueryAsync();
}
if (insertCount != pointCount)
{
return(InvokeResult.FromError("Point count does not match number of values in the array."));
}
cn.Close();
}
return(InvokeResult.Success);
}
catch (Exception ex)
{
return(InvokeResult.FromException("PointArrayPostgresqlConnector_AddItem", ex));
}
}
