public static List <string> ValidateReplicationFormData(this ConfigureReplicationFormData data)
{
var errors = new List <string>();
if (string.IsNullOrWhiteSpace(data.SchemaName))
{
errors.Add("Schema name is empty.");
}
if (string.IsNullOrWhiteSpace(data.GoldenTableName))
{
errors.Add("Golden Record table name is empty.");
}
if (string.IsNullOrWhiteSpace(data.VersionTableName))
{
errors.Add("Version Record table name is empty.");
}
if (data.GoldenTableName == data.VersionTableName)
{
errors.Add("Golden Record Table and Version Record table cannot have the same name.");
}
return(errors);
}
public static List <string> ValidateReplicationFormData(this ConfigureReplicationFormData data)
{
var errors = new List <string>();
if (string.IsNullOrWhiteSpace(data.GoldenBucketName))
{
errors.Add("Golden Record bucket name is empty.");
}
else
{
if (data.GoldenBucketName.Length > 100)
{
errors.Add("Golden Record bucket name is too long.");
}
}
if (string.IsNullOrWhiteSpace(data.VersionBucketName))
{
errors.Add("Version Record bucket name is empty.");
}
else
{
if (data.VersionBucketName.Length > 100)
{
errors.Add("Version Record bucket name is too long.");
}
}
return(errors);
}
public DelimitedImportExport(SqlDatabaseConnection sqlDatabaseConnection, string tableName, string schemaName, ConfigureReplicationFormData replicationFormData)
{
if (string.IsNullOrWhiteSpace(tableName))
{
throw new Exception("TableName parameter is required.");
}
if (string.IsNullOrWhiteSpace(schemaName))
{
throw new Exception("SchemaName parameter is required.");
}
if (sqlDatabaseConnection.State == System.Data.ConnectionState.Closed)
{
sqlDatabaseConnection.Open();
}
SQLDatabaseConnection = sqlDatabaseConnection;
TableName = tableName;
SchemaName = schemaName;
Delimiter = replicationFormData.GetDelimiter();
ReplicationFormData = replicationFormData;
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="connFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecord(IConnectionFactory connFactory, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await ReplicationSemaphoreSlim.WaitAsync();
// setup
var safeSchemaName = config.SchemaName;
var safeGoldenTableName = config.GoldenTableName;
var safeVersionTableName = config.VersionTableName;
var goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName);
var versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName);
// transform data
var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList();
var recordData = GetNamedRecordData(schema, record.DataJson);
recordData[Constants.ReplicationRecordId] = record.RecordId;
recordData[Constants.ReplicationVersionIds] = recordVersionIds;
// get previous golden record
List <string> previousRecordVersionIds;
if (await RecordExistsAsync(connFactory, goldenTable, record.RecordId))
{
var recordMap = await GetRecordAsync(connFactory, goldenTable, record.RecordId);
if (recordMap.ContainsKey(Constants.ReplicationVersionIds))
{
previousRecordVersionIds =
JsonConvert.DeserializeObject <List <string> >(recordMap[Constants.ReplicationVersionIds].ToString());
}
else
{
previousRecordVersionIds = recordVersionIds;
}
}
else
{
previousRecordVersionIds = recordVersionIds;
}
// write data
// check if 2 since we always add 2 things to the dictionary
if (recordData.Count == 2)
{
// delete everything for this record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE");
await DeleteRecordAsync(connFactory, goldenTable, record.RecordId);
foreach (var versionId in previousRecordVersionIds)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
await DeleteRecordAsync(connFactory, versionTable, versionId);
}
}
else
{
// update record and remove/add versions
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT");
await UpsertRecordAsync(connFactory, goldenTable, recordData);
// delete missing versions
var missingVersions = previousRecordVersionIds.Except(recordVersionIds);
foreach (var versionId in missingVersions)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
await DeleteRecordAsync(connFactory, versionTable, versionId);
}
// upsert other versions
foreach (var version in record.Versions)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT");
var versionData = GetNamedRecordData(schema, version.DataJson);
versionData[Constants.ReplicationVersionRecordId] = version.RecordId;
versionData[Constants.ReplicationRecordId] = record.RecordId;
await UpsertRecordAsync(connFactory, versionTable, versionData);
}
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
ReplicationSemaphoreSlim.Release();
}
}
public static void DeleteFileAtPath(string path, ConfigureReplicationFormData config, Settings settings, bool deleteRemote)
{
DeleteFileAtPath(path, config.FileWriteMode, settings, deleteRemote);
}
/// <summary>
/// Adds and removes records to local replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="clusterFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecord(IClusterFactory clusterFactory, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await ReplicationSemaphoreSlim.WaitAsync();
// setup
var safeShapeName = schema.Name;
var safeGoldenBucketName =
string.Concat(config.GoldenBucketName.Where(c => !char.IsWhiteSpace(c)));
var safeVersionBucketName =
string.Concat(config.VersionBucketName.Where(c => !char.IsWhiteSpace(c)));
var goldenBucket = await clusterFactory.GetBucketAsync(safeGoldenBucketName);
var versionBucket = await clusterFactory.GetBucketAsync(safeVersionBucketName);
// transform data
var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList();
var recordData = GetNamedRecordData(schema, record.DataJson);
recordData[Constants.NaveegoVersionIds] = recordVersionIds;
// get previous golden record
List <string> previousRecordVersionIds;
if (await goldenBucket.ExistsAsync(record.RecordId))
{
var result = await goldenBucket.GetAsync <Dictionary <string, object> >(record.RecordId);
if (result.Value.ContainsKey(Constants.NaveegoVersionIds))
{
previousRecordVersionIds =
JsonConvert.DeserializeObject <List <string> >(
JsonConvert.SerializeObject(result.Value[Constants.NaveegoVersionIds]));
}
else
{
previousRecordVersionIds = recordVersionIds;
}
}
else
{
previousRecordVersionIds = recordVersionIds;
}
// write data
if (recordData.Count == 0)
{
// delete everything for this record
Logger.Debug($"shapeId: {safeShapeName} | recordId: {record.RecordId} - DELETE");
var result = await goldenBucket.RemoveAsync(record.RecordId);
result.EnsureSuccess();
foreach (var versionId in previousRecordVersionIds)
{
Logger.Debug(
$"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
result = await versionBucket.RemoveAsync(versionId);
result.EnsureSuccess();
}
}
else
{
// update record and remove/add versions
Logger.Debug($"shapeId: {safeShapeName} | recordId: {record.RecordId} - UPSERT");
var result = await goldenBucket.UpsertAsync(record.RecordId, recordData);
result.EnsureSuccess();
// delete missing versions
var missingVersions = previousRecordVersionIds.Except(recordVersionIds);
foreach (var versionId in missingVersions)
{
Logger.Debug(
$"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
var versionDeleteResult = await versionBucket.RemoveAsync(versionId);
versionDeleteResult.EnsureSuccess();
}
// upsert other versions
foreach (var version in record.Versions)
{
Logger.Debug(
$"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT");
var versionData = GetNamedRecordData(schema, version.DataJson);
var versionUpsertResult = await versionBucket.UpsertAsync(version.RecordId, versionData);
versionUpsertResult.EnsureSuccess();
}
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
ReplicationSemaphoreSlim.Release();
}
}
/// <summary>
/// Adds and removes records to local replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <returns>Error message string</returns>
public static string WriteRecord(Schema schema, Record record, ConfigureReplicationFormData config)
{
try
{
Directory.CreateDirectory(Path);
var recordData = GetNamedRecordData(schema, record);
// var safeShapeName = string.Concat(schema.Name.Where(c => !char.IsWhiteSpace(c)));
var safeShapeName = string.Concat(config.ShapeName.Where(c => !char.IsWhiteSpace(c)));
using (var db = new LiteDatabase($"{Path}/SisenseReplication.db"))
{
var goldenRecords = db.GetCollection <ReplicationGoldenRecord>($"{safeShapeName}_golden_records");
var versions = db.GetCollection <ReplicationVersionRecord>($"{safeShapeName}_versions");
// get and check previous record
var previousRecord = goldenRecords.FindOne(r => r.RecordId == record.RecordId);
if (previousRecord == null)
{
// set previous record to current record
previousRecord = new ReplicationGoldenRecord
{
RecordId = record.RecordId,
VersionRecordIds = record.Versions.Select(v => v.RecordId).ToList(),
Data = recordData
};
}
if (recordData.Count == 0)
{
// delete everything for this record
Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} - DELETE");
goldenRecords.Delete(r => r.RecordId == record.RecordId);
foreach (var versionId in previousRecord.VersionRecordIds)
{
Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
versions.Delete(r => r.VersionRecordId == versionId);
}
}
else
{
// update record and remove/add versions
Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} - UPSERT");
var replicationRecord = new ReplicationGoldenRecord
{
RecordId = record.RecordId,
VersionRecordIds = record.Versions.Select(v => v.RecordId).ToList(),
Data = recordData
};
goldenRecords.Upsert(replicationRecord);
// delete missing versions
var missingVersions = previousRecord.VersionRecordIds.Except(replicationRecord.VersionRecordIds);
foreach (var versionId in missingVersions)
{
Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
versions.Delete(r => r.VersionRecordId == versionId);
}
// upsert other versions
foreach (var version in record.Versions)
{
Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT");
var versionRecord = new ReplicationVersionRecord
{
VersionRecordId = version.RecordId,
GoldenRecordId = record.RecordId,
Data = JsonConvert.DeserializeObject <Dictionary <string, object> >(version.DataJson)
};
versions.Upsert(versionRecord);
}
}
// update shapes
var shapes = db.GetCollection <ShapeNameObject>("shapes");
var existingShape = shapes.FindOne(s => s.ShapeName == safeShapeName);
if (existingShape == null)
{
var shapeNameObject = new ShapeNameObject
{
ShapeName = safeShapeName
};
shapes.Upsert(shapeNameObject);
// update the Sisense service
var sisenseConfig = GenerateSisenseConfig();
AddSisenseService(sisenseConfig);
}
}
return("");
}
catch (Exception e)
{
Logger.Error(e.Message);
throw;
}
}
/// <summary>
/// Adds and removes records to local database
/// Adds and updates available shapes
/// </summary>
/// <param name="client"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <returns></returns>
public static async Task <string> WriteRecord(RequestHelper client, Schema schema, Record record,
ConfigureReplicationFormData config)
{
try
{
var recordData = GetNamedRecordData(schema, record);
var databaseName = string.Concat(config.DatabaseName.Where(c => !char.IsWhiteSpace(c)));
var primaryKey = config.PrimaryKey;
//generate record id for CouchDB
var recordId = primaryKey.Equals("auto generate unique id")
? record.RecordId
: recordData[primaryKey].ToString();
Logger.Info($"receive record: {recordId}");
//remove "_id" and "_rev" fields from record data
if (recordData.ContainsKey("_id"))
{
recordData.Remove("_id");
}
if (recordData.ContainsKey("_rev"))
{
recordData.Remove("_rev");
}
// find previous record in database, return null if it doesn't exist
var findQuery = $"{{\"selector\":{{\"_id\": {{\"$eq\":\"{recordId}\"}}}}}}";
var previousRecordResponse = await client.PostAsync($"{databaseName}/_find", new StringContent(
findQuery, Encoding.UTF8,
"application/json"));
previousRecordResponse.EnsureSuccessStatusCode();
var documents = JObject.Parse(await previousRecordResponse.Content.ReadAsStringAsync())["docs"];
// get and check previous record
Logger.Info("get previous record");
var previousRecord = documents.ToList().FirstOrDefault();
Logger.Info($"previous record: {previousRecord}");
if (previousRecord == null && recordData.Count != 0)
{
// set previous record to current record
Logger.Info($"databaseName:{databaseName} | recordId: {record.RecordId} - INSERT");
var createDocUri = $"/{databaseName}/{recordId}";
await client.PutAsync(createDocUri,
new StringContent(JsonConvert.SerializeObject(recordData), Encoding.UTF8, "application/json"));
return("");
}
if (previousRecord != null)
{
if (recordData.Count == 0)
{
Logger.Info($"databaseName:{databaseName} | recordId: {record.RecordId} - DELETE");
var deleteDocUri =
$"/{databaseName}/{recordId}?{previousRecord["rev"]}";
await client.DeleteAsync(deleteDocUri);
}
else
{
// update record and remove/add version
Logger.Info($"shapeId; {databaseName} | recordId: {record.RecordId} - UPSERT");
var reviseDocUri = $"/{databaseName}/{recordId}";
// add _rev to recordData
recordData.Add("_rev", previousRecord["rev"]);
await client.PutAsync(reviseDocUri,
new StringContent(JsonConvert.SerializeObject(recordData), Encoding.UTF8,
"application/json"));
}
}
return("");
}
catch (Exception e)
{
Logger.Error(e, e.Message);
throw;
}
}
public IImportExportFile MakeImportExportFile(SqlDatabaseConnection sqlDatabaseConnection,
ConfigureReplicationFormData replicationFormData, string tableName, string schemaName)
{
return(new DelimitedImportExport(sqlDatabaseConnection, tableName, schemaName, replicationFormData));
}
public static void WriteToDisk(IImportExportFile goldenImportExport,
IImportExportFile versionImportExport, ConfigureReplicationFormData config, Settings settings, bool forceWrite = false)
{
// check if 5 seconds have passed since last write to disk
if (forceWrite || (DateTime.Now - LastWriteTime).TotalSeconds >= 5 && PendingWrites)
{
// write out to disk
goldenImportExport.ExportTable(config.GetLocalGoldenFilePath());
versionImportExport.ExportTable(config.GetLocalVersionFilePath());
PendingWrites = false;
// write to Remote
if (config.FileWriteMode != Constants.FileModeLocal)
{
var localGoldenFileName = config.GetLocalGoldenFilePath();
var remoteGoldenFileName = config.GetRemoteGoldenFilePath();
var localVersionFileName = config.GetLocalVersionFilePath();
var remoteVersionFileName = config.GetRemoteVersionFilePath();
switch (config.FileWriteMode)
{
case Constants.FileModeFtp:
using (var client = Utility.Utility.GetFtpClient())
{
try
{
var status = client.UploadFile(localGoldenFileName, remoteGoldenFileName);
if (status == FtpStatus.Failed)
{
throw new Exception($"Could not write file to remote {remoteGoldenFileName}");
}
status = client.UploadFile(localVersionFileName, remoteVersionFileName);
if (status == FtpStatus.Failed)
{
throw new Exception($"Could not write file to remote {remoteVersionFileName}");
}
}
finally
{
client.Disconnect();
}
}
break;
case Constants.FileModeSftp:
using (var client = Utility.Utility.GetSftpClient())
{
try
{
var goldenFileStream =
Utility.Utility.GetStream(localGoldenFileName, Constants.FileModeLocal, true);
client.UploadFile(goldenFileStream.Stream, remoteGoldenFileName);
goldenFileStream.Close();
var versionFileStream =
Utility.Utility.GetStream(localVersionFileName, Constants.FileModeLocal, true);
client.UploadFile(versionFileStream.Stream, remoteVersionFileName);
versionFileStream.Close();
}
catch
{
throw new Exception(
$"Could not write files to remote {remoteGoldenFileName} {remoteVersionFileName}");
}
finally
{
client.Disconnect();
}
}
break;
}
}
}
}
public IImportExportFile MakeImportExportFile(SqlDatabaseConnection sqlDatabaseConnection,
ConfigureReplicationFormData replicationFormData, string tableName, string schemaName)
{
throw new System.NotImplementedException();
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="conn"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecordAsync(SqlDatabaseConnection conn, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await WriteSemaphoreSlim.WaitAsync();
// setup
var safeSchemaName = Constants.SchemaName;
var safeTargetTableName = config.GetGoldenTableName();
var targetTable =
Replication.Replication.GetGoldenReplicationTable(schema, safeSchemaName, safeTargetTableName);
// get record
var recordData = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
recordData[Constants.ReplicationRecordId] = record.RecordId;
recordData[Constants.ReplicationVersionIds] = null;
// write data
if (recordData.Count == 2)
{
// delete record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE");
await Replication.Replication.DeleteRecordAsync(conn, targetTable, record.RecordId);
}
else
{
// add in all default values
foreach (var property in schema.Properties)
{
if (!recordData.ContainsKey(property.Id) && !string.IsNullOrWhiteSpace(property.PublisherMetaJson))
{
Logger.Debug("adding default value");
var columnConfig = JsonConvert.DeserializeObject <WriteColumn>(property.PublisherMetaJson);
recordData[property.Id] = columnConfig.DefaultValue;
}
}
Logger.Debug(JsonConvert.SerializeObject(recordData, Formatting.Indented));
// update record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT");
await Replication.Replication.UpsertRecordAsync(conn, targetTable, recordData);
}
// set triggers for async file write
LastWriteTime = DateTime.Now;
PendingWrites = true;
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
if (e.Message.Contains("library routine called out of sequence"))
{
throw;
}
return(e.Message);
}
finally
{
WriteSemaphoreSlim.Release();
}
}
public static List <string> ValidateReplicationFormData(this ConfigureReplicationFormData data,
Settings settings)
{
var errors = new List <string>();
if (string.IsNullOrWhiteSpace(data.GoldenRecordFileDirectory))
{
errors.Add("Golden Record file directory is empty.");
}
if (string.IsNullOrWhiteSpace(data.GoldenRecordFileName))
{
errors.Add("Golden Record file name is empty.");
}
if (string.IsNullOrWhiteSpace(data.VersionRecordFileDirectory))
{
errors.Add("Version Record file directory is empty.");
}
if (string.IsNullOrWhiteSpace(data.VersionRecordFileName))
{
errors.Add("Version Record file name is empty.");
}
if (string.IsNullOrWhiteSpace(data.FileWriteMode))
{
errors.Add("File Write Mode is empty.");
}
if (string.IsNullOrWhiteSpace(data.NullValue))
{
data.NullValue = "";
}
if (string.IsNullOrWhiteSpace(data.CustomHeader))
{
data.CustomHeader = "";
}
if (data.FileWriteMode != Constants.FileModeLocal)
{
var goldenTestFileName = "golden_test.txt";
var remoteGoldenDirectory = data.GetRemoteGoldenDirectory();
var remoteGoldenTestFileName = Path.Join(remoteGoldenDirectory, goldenTestFileName);
var localGoldenTestDirectory = data.GetLocalGoldenDirectory();
var localGoldenTestFileName = Path.Join(localGoldenTestDirectory, goldenTestFileName);
Directory.CreateDirectory(localGoldenTestDirectory);
var goldenTestFile = new StreamWriter(localGoldenTestFileName);
goldenTestFile.WriteLine("test");
goldenTestFile.Close();
var versionTestFileName = "version_test.txt";
var remoteVersionDirectory = data.GetRemoteVersionDirectory();
var remoteVersionTestFileName = Path.Join(remoteVersionDirectory, versionTestFileName);
var localVersionTestDirectory = data.GetLocalVersionDirectory();
var localVersionTestFileName = Path.Join(localVersionTestDirectory, versionTestFileName);
Directory.CreateDirectory(localVersionTestDirectory);
var testFile = new StreamWriter(localVersionTestFileName);
testFile.WriteLine("test");
testFile.Close();
switch (data.FileWriteMode)
{
case Constants.FileModeFtp:
using (var client = Utility.Utility.GetFtpClient())
{
try
{
if (!client.DirectoryExists(remoteGoldenDirectory))
{
errors.Add($"{remoteGoldenDirectory} is not a directory on remote FTP");
}
else
{
var status = client.UploadFile(localGoldenTestFileName, remoteGoldenTestFileName);
if (status == FtpStatus.Failed)
{
errors.Add($"Could not write to golden directory {remoteGoldenDirectory}");
}
}
Utility.Utility.DeleteFileAtPath(localGoldenTestFileName, data, settings, true);
if (!client.DirectoryExists(remoteVersionDirectory))
{
errors.Add($"{remoteVersionDirectory} is not a directory on remote FTP");
}
else
{
var status = client.UploadFile(localVersionTestFileName, remoteVersionTestFileName);
if (status == FtpStatus.Failed)
{
errors.Add(
$"Could not write to version directory {remoteVersionDirectory}");
}
}
Utility.Utility.DeleteFileAtPath(localVersionTestFileName, data, settings, true);
}
finally
{
client.Disconnect();
}
}
break;
case Constants.FileModeSftp:
using (var client = Utility.Utility.GetSftpClient())
{
try
{
try
{
if (!client.Exists(remoteGoldenDirectory))
{
errors.Add($"{remoteGoldenDirectory} is not a directory on remote FTP");
}
else
{
var fileStream = Utility.Utility.GetStream(localGoldenTestFileName, Constants.FileModeLocal, true);
client.UploadFile(fileStream.Stream, remoteGoldenTestFileName);
fileStream.Close();
Utility.Utility.DeleteFileAtPath(localGoldenTestFileName, data, settings, true);
}
}
catch
{
errors.Add($"Could not write to golden directory {remoteGoldenDirectory}");
}
try
{
if (!client.Exists(remoteVersionDirectory))
{
errors.Add($"{remoteVersionDirectory} is not a directory on remote FTP");
}
else
{
var fileStream = Utility.Utility.GetStream(localVersionTestFileName, Constants.FileModeLocal, true);
client.UploadFile(fileStream.Stream, remoteVersionTestFileName);
fileStream.Close();
Utility.Utility.DeleteFileAtPath(localVersionTestFileName, data, settings, true);
}
}
catch
{
errors.Add($"Could not write to version directory {remoteVersionDirectory}");
}
}
finally
{
client.Disconnect();
}
}
break;
}
}
return(errors);
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="connFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecordAsync(IConnectionFactory connectionFactory, Schema schema, Record record, ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
Logger.Debug($"Starting {record.RecordId}");
Stopwatch timer = Stopwatch.StartNew();
try
{
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
await WriteSemaphoreSlim.WaitAsync();
string safeSchemaName = config.SchemaName;
string safeGoldenTableName = config.GoldenTableName;
string safeVersionTableName = config.VersionTableName;
ReplicationTable goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName);
ReplicationTable versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName);
List <string> recordVersionIds = record.Versions.Select(r => r.RecordId).ToList();
// GetNamed Record Data
// TODO: Finish
return(null);
}
catch (Exception ex)
{
Logger.Error(ex, $"Error writing record: {ex.Message}");
RecordAck ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ex.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
return(ex.Message);
}
finally
{
Logger.Debug($"Stopped {record.RecordId}. Time: {timer.ElapsedMilliseconds}");
WriteSemaphoreSlim.Release();
}
}
