/// <summary>
/// Takes in records and writes them out to the Sage instance then sends acks back to the client
/// </summary>
/// <param name="requestStream"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task WriteStream(IAsyncStreamReader <Record> requestStream,
IServerStreamWriter <RecordAck> responseStream, ServerCallContext context)
{
try
{
Logger.Info("Writing records to Sage...");
var schema = _server.WriteSettings.Schema;
var sla = _server.WriteSettings.CommitSLA;
var inCount = 0;
var outCount = 0;
// get next record to publish while connected and configured
while (await requestStream.MoveNext(context.CancellationToken) && _server.Connected &&
_server.WriteConfigured)
{
var record = requestStream.Current;
inCount++;
Logger.Debug($"Got Record: {record.DataJson}");
// send record to source system
// timeout if it takes longer than the sla
var task = Task.Run(() => PutRecord(schema, record));
if (task.Wait(TimeSpan.FromSeconds(sla)))
{
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = task.Result
};
await responseStream.WriteAsync(ack);
if (String.IsNullOrEmpty(task.Result))
{
outCount++;
}
}
else
{
// send timeout ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = "timed out"
};
await responseStream.WriteAsync(ack);
}
}
Logger.Info($"Wrote {outCount} of {inCount} records to Sage.");
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
}
}
public static async Task <string> WriteRecordAsync(IApiClient apiClient, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
var endpoint = EndpointHelper.GetEndpointForSchema(schema);
if (endpoint == null)
{
throw new Exception($"Endpoint {schema.Id} does not exist");
}
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await WriteSemaphoreSlim.WaitAsync();
// write records
var errorMessage = await endpoint.WriteRecordAsync(apiClient, schema, record, responseStream);
if (!string.IsNullOrWhiteSpace(errorMessage))
{
Logger.Error(new Exception(errorMessage), errorMessage);
}
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error writing record {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
WriteSemaphoreSlim.Release();
}
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="connFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecordAsync(IConnectionFactory connectionFactory, Schema schema, Record record, ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
Logger.Debug($"Starting {record.RecordId}");
Stopwatch timer = Stopwatch.StartNew();
try
{
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
await WriteSemaphoreSlim.WaitAsync();
string safeSchemaName = config.SchemaName;
string safeGoldenTableName = config.GoldenTableName;
string safeVersionTableName = config.VersionTableName;
ReplicationTable goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName);
ReplicationTable versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName);
List <string> recordVersionIds = record.Versions.Select(r => r.RecordId).ToList();
// GetNamed Record Data
// TODO: Finish
return(null);
}
catch (Exception ex)
{
Logger.Error(ex, $"Error writing record: {ex.Message}");
RecordAck ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ex.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
return(ex.Message);
}
finally
{
Logger.Debug($"Stopped {record.RecordId}. Time: {timer.ElapsedMilliseconds}");
WriteSemaphoreSlim.Release();
}
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="connFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecord(IConnectionFactory connFactory, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await ReplicationSemaphoreSlim.WaitAsync();
// setup
var safeSchemaName = config.SchemaName;
var safeGoldenTableName = config.GoldenTableName;
var safeVersionTableName = config.VersionTableName;
var goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName);
var versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName);
// transform data
var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList();
var recordData = GetNamedRecordData(schema, record.DataJson);
recordData[Constants.ReplicationRecordId] = record.RecordId;
recordData[Constants.ReplicationVersionIds] = recordVersionIds;
// get previous golden record
List <string> previousRecordVersionIds;
if (await RecordExistsAsync(connFactory, goldenTable, record.RecordId))
{
var recordMap = await GetRecordAsync(connFactory, goldenTable, record.RecordId);
if (recordMap.ContainsKey(Constants.ReplicationVersionIds))
{
previousRecordVersionIds =
JsonConvert.DeserializeObject <List <string> >(recordMap[Constants.ReplicationVersionIds].ToString());
}
else
{
previousRecordVersionIds = recordVersionIds;
}
}
else
{
previousRecordVersionIds = recordVersionIds;
}
// write data
// check if 2 since we always add 2 things to the dictionary
if (recordData.Count == 2)
{
// delete everything for this record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE");
await DeleteRecordAsync(connFactory, goldenTable, record.RecordId);
foreach (var versionId in previousRecordVersionIds)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
await DeleteRecordAsync(connFactory, versionTable, versionId);
}
}
else
{
// update record and remove/add versions
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT");
await UpsertRecordAsync(connFactory, goldenTable, recordData);
// delete missing versions
var missingVersions = previousRecordVersionIds.Except(recordVersionIds);
foreach (var versionId in missingVersions)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
await DeleteRecordAsync(connFactory, versionTable, versionId);
}
// upsert other versions
foreach (var version in record.Versions)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT");
var versionData = GetNamedRecordData(schema, version.DataJson);
versionData[Constants.ReplicationVersionRecordId] = version.RecordId;
versionData[Constants.ReplicationRecordId] = record.RecordId;
await UpsertRecordAsync(connFactory, versionTable, versionData);
}
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
ReplicationSemaphoreSlim.Release();
}
}
/// <summary>
/// Writes records to Sisense
/// </summary>
/// <param name="requestStream"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task WriteStream(IAsyncStreamReader <Record> requestStream,
IServerStreamWriter <RecordAck> responseStream, ServerCallContext context)
{
try
{
Logger.Info("Writing records to Sisense...");
Logger.Info($"API Route: {GetBindingHostedService.ServerAddresses.Addresses.FirstOrDefault()}");
var schema = _server.WriteSettings.Schema;
var sla = _server.WriteSettings.CommitSLA;
var inCount = 0;
var outCount = 0;
// get next record to publish while connected and configured
while (await requestStream.MoveNext(context.CancellationToken) && _server.Connected &&
_server.WriteConfigured)
{
var record = requestStream.Current;
inCount++;
Logger.Debug($"Got record: {record.DataJson}");
if (_server.WriteSettings.IsReplication())
{
var config = JsonConvert.DeserializeObject <ConfigureReplicationFormData>(_server.WriteSettings.Replication.SettingsJson);
// send record to source system
// timeout if it takes longer than the sla
var task = Task.Run(() => Replication.WriteRecord(schema, record, config));
if (task.Wait(TimeSpan.FromSeconds(sla)))
{
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = task.Result
};
await responseStream.WriteAsync(ack);
if (String.IsNullOrEmpty(task.Result))
{
outCount++;
}
}
else
{
// send timeout ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = "timed out"
};
await responseStream.WriteAsync(ack);
}
}
else
{
throw new Exception("Only replication writebacks are supported");
}
}
Logger.Info($"Wrote {outCount} of {inCount} records to Sisense.");
}
catch (Exception e)
{
Logger.Error(e.Message);
throw;
}
}
public static async Task <string> WriteRecordAsync(IConnectionFactory connFactory, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
var conn = connFactory.GetConnection();
try
{
var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await WriteSemaphoreSlim.WaitAsync();
// call stored procedure
var querySb = new StringBuilder($"CALL {schema.Query}(");
foreach (var property in schema.Properties)
{
if (!recordMap.ContainsKey(property.Id))
{
throw new Exception($"{property.Id} is required by the stored procedure and is not mapped on the job.");
}
var rawValue = recordMap[property.Id];
if (rawValue == null || string.IsNullOrWhiteSpace(rawValue.ToString()))
{
querySb.Append("NULL,");
}
else
{
querySb.Append($"'{Utility.Utility.GetSafeString(Utility.Utility.GetSafeString(rawValue.ToString(), "'", "''"))}',");
}
}
querySb.Length--;
querySb.Append(")");
var query = querySb.ToString();
Logger.Debug($"WB querySb: {query}");
await conn.OpenAsync();
var cmd = connFactory.GetCommand(query, conn);
await cmd.ExecuteNonQueryAsync();
await conn.CloseAsync();
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
await conn.CloseAsync();
Logger.Error(e, $"Error writing record {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
WriteSemaphoreSlim.Release();
}
}
/// <summary>
/// Adds and removes records to local replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="clusterFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecord(IClusterFactory clusterFactory, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await ReplicationSemaphoreSlim.WaitAsync();
// setup
var safeShapeName = schema.Name;
var safeGoldenBucketName =
string.Concat(config.GoldenBucketName.Where(c => !char.IsWhiteSpace(c)));
var safeVersionBucketName =
string.Concat(config.VersionBucketName.Where(c => !char.IsWhiteSpace(c)));
var goldenBucket = await clusterFactory.GetBucketAsync(safeGoldenBucketName);
var versionBucket = await clusterFactory.GetBucketAsync(safeVersionBucketName);
// transform data
var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList();
var recordData = GetNamedRecordData(schema, record.DataJson);
recordData[Constants.NaveegoVersionIds] = recordVersionIds;
// get previous golden record
List <string> previousRecordVersionIds;
if (await goldenBucket.ExistsAsync(record.RecordId))
{
var result = await goldenBucket.GetAsync <Dictionary <string, object> >(record.RecordId);
if (result.Value.ContainsKey(Constants.NaveegoVersionIds))
{
previousRecordVersionIds =
JsonConvert.DeserializeObject <List <string> >(
JsonConvert.SerializeObject(result.Value[Constants.NaveegoVersionIds]));
}
else
{
previousRecordVersionIds = recordVersionIds;
}
}
else
{
previousRecordVersionIds = recordVersionIds;
}
// write data
if (recordData.Count == 0)
{
// delete everything for this record
Logger.Debug($"shapeId: {safeShapeName} | recordId: {record.RecordId} - DELETE");
var result = await goldenBucket.RemoveAsync(record.RecordId);
result.EnsureSuccess();
foreach (var versionId in previousRecordVersionIds)
{
Logger.Debug(
$"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
result = await versionBucket.RemoveAsync(versionId);
result.EnsureSuccess();
}
}
else
{
// update record and remove/add versions
Logger.Debug($"shapeId: {safeShapeName} | recordId: {record.RecordId} - UPSERT");
var result = await goldenBucket.UpsertAsync(record.RecordId, recordData);
result.EnsureSuccess();
// delete missing versions
var missingVersions = previousRecordVersionIds.Except(recordVersionIds);
foreach (var versionId in missingVersions)
{
Logger.Debug(
$"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
var versionDeleteResult = await versionBucket.RemoveAsync(versionId);
versionDeleteResult.EnsureSuccess();
}
// upsert other versions
foreach (var version in record.Versions)
{
Logger.Debug(
$"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT");
var versionData = GetNamedRecordData(schema, version.DataJson);
var versionUpsertResult = await versionBucket.UpsertAsync(version.RecordId, versionData);
versionUpsertResult.EnsureSuccess();
}
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
ReplicationSemaphoreSlim.Release();
}
}
public virtual async Task <string> WriteRecordAsync(IApiClient apiClient, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
foreach (var requiredPropertyId in RequiredWritePropertyIds)
{
if (!recordMap.ContainsKey(requiredPropertyId))
{
var errorMessage = $"Record did not contain required property {requiredPropertyId}";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
if (recordMap.ContainsKey(requiredPropertyId) && recordMap[requiredPropertyId] == null)
{
var errorMessage = $"Required property {requiredPropertyId} was NULL";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
}
var postObject = new Dictionary <string, object>();
foreach (var property in schema.Properties)
{
object value = null;
if (recordMap.ContainsKey(property.Id))
{
value = recordMap[property.Id];
}
postObject.TryAdd(property.Id, value);
}
var json = new StringContent(
JsonConvert.SerializeObject(postObject),
Encoding.UTF8,
"application/json"
);
HttpResponseMessage response;
if (!recordMap.ContainsKey(WritePathPropertyId) || recordMap.ContainsKey(WritePathPropertyId) &&
recordMap[WritePathPropertyId] == null)
{
response =
await apiClient.PostAsync($"{BasePath.TrimEnd('/')}", json);
}
else
{
response =
await apiClient.PutAsync($"{BasePath.TrimEnd('/')}/{recordMap[WritePathPropertyId]}", json);
}
if (!response.IsSuccessStatusCode)
{
var errorMessage = await response.Content.ReadAsStringAsync();
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
return("");
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="conn"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecordAsync(SqlDatabaseConnection conn, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await WriteSemaphoreSlim.WaitAsync();
// setup
var safeSchemaName = Constants.SchemaName;
var safeTargetTableName = config.GetGoldenTableName();
var targetTable =
Replication.Replication.GetGoldenReplicationTable(schema, safeSchemaName, safeTargetTableName);
// get record
var recordData = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
recordData[Constants.ReplicationRecordId] = record.RecordId;
recordData[Constants.ReplicationVersionIds] = null;
// write data
if (recordData.Count == 2)
{
// delete record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE");
await Replication.Replication.DeleteRecordAsync(conn, targetTable, record.RecordId);
}
else
{
// add in all default values
foreach (var property in schema.Properties)
{
if (!recordData.ContainsKey(property.Id) && !string.IsNullOrWhiteSpace(property.PublisherMetaJson))
{
Logger.Debug("adding default value");
var columnConfig = JsonConvert.DeserializeObject <WriteColumn>(property.PublisherMetaJson);
recordData[property.Id] = columnConfig.DefaultValue;
}
}
Logger.Debug(JsonConvert.SerializeObject(recordData, Formatting.Indented));
// update record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT");
await Replication.Replication.UpsertRecordAsync(conn, targetTable, recordData);
}
// set triggers for async file write
LastWriteTime = DateTime.Now;
PendingWrites = true;
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
if (e.Message.Contains("library routine called out of sequence"))
{
throw;
}
return(e.Message);
}
finally
{
WriteSemaphoreSlim.Release();
}
}
public override async Task <string> WriteRecordAsync(IApiClient apiClient, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
foreach (var requiredPropertyId in RequiredWritePropertyIds)
{
if (!recordMap.ContainsKey(requiredPropertyId))
{
var errorMessage = $"Record did not contain required property {requiredPropertyId}";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
if (recordMap.ContainsKey(requiredPropertyId) && recordMap[requiredPropertyId] == null)
{
var errorMessage = $"Required property {requiredPropertyId} was NULL";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
}
// get live object
Dictionary <string, object> liveRecord = new Dictionary <string, object>();
var getResponse = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{recordMap[DetailPropertyId]}/{DetailPath.TrimStart('/')}");
if (getResponse.IsSuccessStatusCode)
{
liveRecord =
JsonConvert.DeserializeObject <Dictionary <string, object> >(await getResponse.Content.ReadAsStringAsync());
}
var putObject = new Dictionary <string, object>();
var customFieldObject = new List <CustomField>();
foreach (var property in schema.Properties)
{
if (property.TypeAtSource == Constants.CustomProperty)
{
var customField = new CustomField
{
FieldName = property.Id,
Value = null
};
if (recordMap.ContainsKey(property.Id))
{
customField.Value = recordMap[property.Id];
}
customFieldObject.Add(customField);
}
else
{
object value = null;
if (recordMap.ContainsKey(property.Id))
{
value = recordMap[property.Id];
if (property.Type == PropertyType.Json)
{
if (value is string s)
{
value = JsonConvert.DeserializeObject(s) ?? null;
}
}
}
putObject.Add(property.Id, value);
}
}
putObject.Add("CustomFields", customFieldObject);
if (putObject.ContainsKey("Lists") && putObject["Lists"] != null)
{
JArray j = (JArray)putObject["Lists"];
putObject["Lists"] = j.Select(x => x["ListID"]).ToList();
}
else
{
JArray j = (JArray)liveRecord["Lists"];
putObject["Lists"] = j.Select(x => x["ListID"]).ToList();
}
if (putObject.ContainsKey("Publications") && putObject["Publications"] != null)
{
JArray j = (JArray)putObject["Publications"];
putObject["Publications"] = j.Select(x => x["PublicationID"]).ToList();
}
else
{
JArray j = (JArray)liveRecord["Publications"];
putObject["Publications"] = j.Select(x => x["PublicationID"]).ToList();
}
if (putObject.ContainsKey("Source") && putObject["Source"] != null)
{
JObject j = (JObject)putObject["Source"];
if (j.ContainsKey("SourceID"))
{
putObject["SourceID"] = j["SourceID"];
}
putObject.Remove("Source");
}
else
{
JObject j = (JObject)liveRecord["Source"];
if (j.ContainsKey("SourceID"))
{
putObject["SourceID"] = j["SourceID"];
}
putObject.Remove("Source");
}
var json = new StringContent(
JsonConvert.SerializeObject(putObject),
Encoding.UTF8,
"application/json"
);
var response =
await apiClient.PutAsync($"{BasePath.TrimEnd('/')}/{recordMap[WritePathPropertyId]}", json);
if (response.StatusCode == HttpStatusCode.NotFound)
{
response =
await apiClient.PostAsync($"{BasePath.TrimEnd('/')}", json);
}
if (!response.IsSuccessStatusCode)
{
var errorMessage = await response.Content.ReadAsStringAsync();
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
return("");
}
public override async Task <string> WriteRecordAsync(IApiClient apiClient, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
foreach (var requiredPropertyId in RequiredWritePropertyIds)
{
if (!recordMap.ContainsKey(requiredPropertyId))
{
var errorMessage = $"Record did not contain required property {requiredPropertyId}";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
if (recordMap.ContainsKey(requiredPropertyId) && recordMap[requiredPropertyId] == null)
{
var errorMessage = $"Required property {requiredPropertyId} was NULL";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
}
// only preload conversion dictionary if empty
if (NameToListIdDictionary.IsEmpty)
{
await PreLoadLookup(apiClient);
}
// attempt to add email
var errorString = await AddEmailToList(apiClient, schema, recordMap);
if (!string.IsNullOrWhiteSpace(errorString))
{
errorString = await AddNewList(apiClient, schema, recordMap);
// add email to new list if created
if (string.IsNullOrWhiteSpace(errorString))
{
errorString = await AddEmailToList(apiClient, schema, recordMap);
}
}
if (!string.IsNullOrWhiteSpace(errorString))
{
var errorMessage = errorString;
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
return("");
}
