public static List <string> ValidateReplicationFormData(this ConfigureReplicationFormData data) { var errors = new List <string>(); if (string.IsNullOrWhiteSpace(data.SchemaName)) { errors.Add("Schema name is empty."); } if (string.IsNullOrWhiteSpace(data.GoldenTableName)) { errors.Add("Golden Record table name is empty."); } if (string.IsNullOrWhiteSpace(data.VersionTableName)) { errors.Add("Version Record table name is empty."); } if (data.GoldenTableName == data.VersionTableName) { errors.Add("Golden Record Table and Version Record table cannot have the same name."); } return(errors); }
public static List <string> ValidateReplicationFormData(this ConfigureReplicationFormData data) { var errors = new List <string>(); if (string.IsNullOrWhiteSpace(data.GoldenBucketName)) { errors.Add("Golden Record bucket name is empty."); } else { if (data.GoldenBucketName.Length > 100) { errors.Add("Golden Record bucket name is too long."); } } if (string.IsNullOrWhiteSpace(data.VersionBucketName)) { errors.Add("Version Record bucket name is empty."); } else { if (data.VersionBucketName.Length > 100) { errors.Add("Version Record bucket name is too long."); } } return(errors); }
public DelimitedImportExport(SqlDatabaseConnection sqlDatabaseConnection, string tableName, string schemaName, ConfigureReplicationFormData replicationFormData) { if (string.IsNullOrWhiteSpace(tableName)) { throw new Exception("TableName parameter is required."); } if (string.IsNullOrWhiteSpace(schemaName)) { throw new Exception("SchemaName parameter is required."); } if (sqlDatabaseConnection.State == System.Data.ConnectionState.Closed) { sqlDatabaseConnection.Open(); } SQLDatabaseConnection = sqlDatabaseConnection; TableName = tableName; SchemaName = schemaName; Delimiter = replicationFormData.GetDelimiter(); ReplicationFormData = replicationFormData; }
/// <summary> /// Adds and removes records to replication db /// Adds and updates available shapes /// </summary> /// <param name="connFactory"></param> /// <param name="schema"></param> /// <param name="record"></param> /// <param name="config"></param> /// <param name="responseStream"></param> /// <returns>Error message string</returns> public static async Task <string> WriteRecord(IConnectionFactory connFactory, Schema schema, Record record, ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream) { // debug Logger.Debug($"Starting timer for {record.RecordId}"); var timer = Stopwatch.StartNew(); try { // debug Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented)); // semaphore await ReplicationSemaphoreSlim.WaitAsync(); // setup var safeSchemaName = config.SchemaName; var safeGoldenTableName = config.GoldenTableName; var safeVersionTableName = config.VersionTableName; var goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName); var versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName); // transform data var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList(); var recordData = GetNamedRecordData(schema, record.DataJson); recordData[Constants.ReplicationRecordId] = record.RecordId; recordData[Constants.ReplicationVersionIds] = recordVersionIds; // get previous golden record List <string> previousRecordVersionIds; if (await RecordExistsAsync(connFactory, goldenTable, record.RecordId)) { var recordMap = await GetRecordAsync(connFactory, goldenTable, record.RecordId); if (recordMap.ContainsKey(Constants.ReplicationVersionIds)) { previousRecordVersionIds = JsonConvert.DeserializeObject <List <string> >(recordMap[Constants.ReplicationVersionIds].ToString()); } else { previousRecordVersionIds = recordVersionIds; } } else { previousRecordVersionIds = recordVersionIds; } // write data // check if 2 since we always add 2 things to the dictionary if (recordData.Count == 2) { // delete everything for this record Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE"); await DeleteRecordAsync(connFactory, goldenTable, record.RecordId); foreach (var versionId in previousRecordVersionIds) { Logger.Debug( $"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE"); await DeleteRecordAsync(connFactory, versionTable, versionId); } } else { // update record and remove/add versions Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT"); await UpsertRecordAsync(connFactory, goldenTable, recordData); // delete missing versions var missingVersions = previousRecordVersionIds.Except(recordVersionIds); foreach (var versionId in missingVersions) { Logger.Debug( $"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE"); await DeleteRecordAsync(connFactory, versionTable, versionId); } // upsert other versions foreach (var version in record.Versions) { Logger.Debug( $"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT"); var versionData = GetNamedRecordData(schema, version.DataJson); versionData[Constants.ReplicationVersionRecordId] = version.RecordId; versionData[Constants.ReplicationRecordId] = record.RecordId; await UpsertRecordAsync(connFactory, versionTable, versionData); } } var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = "" }; await responseStream.WriteAsync(ack); timer.Stop(); Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}"); return(""); } catch (Exception e) { Logger.Error(e, $"Error replicating records {e.Message}"); // send ack var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = e.Message }; await responseStream.WriteAsync(ack); timer.Stop(); Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}"); return(e.Message); } finally { ReplicationSemaphoreSlim.Release(); } }
public static void DeleteFileAtPath(string path, ConfigureReplicationFormData config, Settings settings, bool deleteRemote) { DeleteFileAtPath(path, config.FileWriteMode, settings, deleteRemote); }
/// <summary> /// Adds and removes records to local replication db /// Adds and updates available shapes /// </summary> /// <param name="clusterFactory"></param> /// <param name="schema"></param> /// <param name="record"></param> /// <param name="config"></param> /// <param name="responseStream"></param> /// <returns>Error message string</returns> public static async Task <string> WriteRecord(IClusterFactory clusterFactory, Schema schema, Record record, ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream) { // debug Logger.Debug($"Starting timer for {record.RecordId}"); var timer = Stopwatch.StartNew(); try { // debug Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented)); // semaphore await ReplicationSemaphoreSlim.WaitAsync(); // setup var safeShapeName = schema.Name; var safeGoldenBucketName = string.Concat(config.GoldenBucketName.Where(c => !char.IsWhiteSpace(c))); var safeVersionBucketName = string.Concat(config.VersionBucketName.Where(c => !char.IsWhiteSpace(c))); var goldenBucket = await clusterFactory.GetBucketAsync(safeGoldenBucketName); var versionBucket = await clusterFactory.GetBucketAsync(safeVersionBucketName); // transform data var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList(); var recordData = GetNamedRecordData(schema, record.DataJson); recordData[Constants.NaveegoVersionIds] = recordVersionIds; // get previous golden record List <string> previousRecordVersionIds; if (await goldenBucket.ExistsAsync(record.RecordId)) { var result = await goldenBucket.GetAsync <Dictionary <string, object> >(record.RecordId); if (result.Value.ContainsKey(Constants.NaveegoVersionIds)) { previousRecordVersionIds = JsonConvert.DeserializeObject <List <string> >( JsonConvert.SerializeObject(result.Value[Constants.NaveegoVersionIds])); } else { previousRecordVersionIds = recordVersionIds; } } else { previousRecordVersionIds = recordVersionIds; } // write data if (recordData.Count == 0) { // delete everything for this record Logger.Debug($"shapeId: {safeShapeName} | recordId: {record.RecordId} - DELETE"); var result = await goldenBucket.RemoveAsync(record.RecordId); result.EnsureSuccess(); foreach (var versionId in previousRecordVersionIds) { Logger.Debug( $"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE"); result = await versionBucket.RemoveAsync(versionId); result.EnsureSuccess(); } } else { // update record and remove/add versions Logger.Debug($"shapeId: {safeShapeName} | recordId: {record.RecordId} - UPSERT"); var result = await goldenBucket.UpsertAsync(record.RecordId, recordData); result.EnsureSuccess(); // delete missing versions var missingVersions = previousRecordVersionIds.Except(recordVersionIds); foreach (var versionId in missingVersions) { Logger.Debug( $"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE"); var versionDeleteResult = await versionBucket.RemoveAsync(versionId); versionDeleteResult.EnsureSuccess(); } // upsert other versions foreach (var version in record.Versions) { Logger.Debug( $"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT"); var versionData = GetNamedRecordData(schema, version.DataJson); var versionUpsertResult = await versionBucket.UpsertAsync(version.RecordId, versionData); versionUpsertResult.EnsureSuccess(); } } var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = "" }; await responseStream.WriteAsync(ack); timer.Stop(); Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}"); return(""); } catch (Exception e) { Logger.Error(e, $"Error replicating records {e.Message}"); // send ack var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = e.Message }; await responseStream.WriteAsync(ack); timer.Stop(); Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}"); return(e.Message); } finally { ReplicationSemaphoreSlim.Release(); } }
/// <summary> /// Adds and removes records to local replication db /// Adds and updates available shapes /// </summary> /// <param name="schema"></param> /// <param name="record"></param> /// <param name="config"></param> /// <returns>Error message string</returns> public static string WriteRecord(Schema schema, Record record, ConfigureReplicationFormData config) { try { Directory.CreateDirectory(Path); var recordData = GetNamedRecordData(schema, record); // var safeShapeName = string.Concat(schema.Name.Where(c => !char.IsWhiteSpace(c))); var safeShapeName = string.Concat(config.ShapeName.Where(c => !char.IsWhiteSpace(c))); using (var db = new LiteDatabase($"{Path}/SisenseReplication.db")) { var goldenRecords = db.GetCollection <ReplicationGoldenRecord>($"{safeShapeName}_golden_records"); var versions = db.GetCollection <ReplicationVersionRecord>($"{safeShapeName}_versions"); // get and check previous record var previousRecord = goldenRecords.FindOne(r => r.RecordId == record.RecordId); if (previousRecord == null) { // set previous record to current record previousRecord = new ReplicationGoldenRecord { RecordId = record.RecordId, VersionRecordIds = record.Versions.Select(v => v.RecordId).ToList(), Data = recordData }; } if (recordData.Count == 0) { // delete everything for this record Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} - DELETE"); goldenRecords.Delete(r => r.RecordId == record.RecordId); foreach (var versionId in previousRecord.VersionRecordIds) { Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE"); versions.Delete(r => r.VersionRecordId == versionId); } } else { // update record and remove/add versions Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} - UPSERT"); var replicationRecord = new ReplicationGoldenRecord { RecordId = record.RecordId, VersionRecordIds = record.Versions.Select(v => v.RecordId).ToList(), Data = recordData }; goldenRecords.Upsert(replicationRecord); // delete missing versions var missingVersions = previousRecord.VersionRecordIds.Except(replicationRecord.VersionRecordIds); foreach (var versionId in missingVersions) { Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE"); versions.Delete(r => r.VersionRecordId == versionId); } // upsert other versions foreach (var version in record.Versions) { Logger.Info($"shapeId: {safeShapeName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT"); var versionRecord = new ReplicationVersionRecord { VersionRecordId = version.RecordId, GoldenRecordId = record.RecordId, Data = JsonConvert.DeserializeObject <Dictionary <string, object> >(version.DataJson) }; versions.Upsert(versionRecord); } } // update shapes var shapes = db.GetCollection <ShapeNameObject>("shapes"); var existingShape = shapes.FindOne(s => s.ShapeName == safeShapeName); if (existingShape == null) { var shapeNameObject = new ShapeNameObject { ShapeName = safeShapeName }; shapes.Upsert(shapeNameObject); // update the Sisense service var sisenseConfig = GenerateSisenseConfig(); AddSisenseService(sisenseConfig); } } return(""); } catch (Exception e) { Logger.Error(e.Message); throw; } }
/// <summary> /// Adds and removes records to local database /// Adds and updates available shapes /// </summary> /// <param name="client"></param> /// <param name="schema"></param> /// <param name="record"></param> /// <param name="config"></param> /// <returns></returns> public static async Task <string> WriteRecord(RequestHelper client, Schema schema, Record record, ConfigureReplicationFormData config) { try { var recordData = GetNamedRecordData(schema, record); var databaseName = string.Concat(config.DatabaseName.Where(c => !char.IsWhiteSpace(c))); var primaryKey = config.PrimaryKey; //generate record id for CouchDB var recordId = primaryKey.Equals("auto generate unique id") ? record.RecordId : recordData[primaryKey].ToString(); Logger.Info($"receive record: {recordId}"); //remove "_id" and "_rev" fields from record data if (recordData.ContainsKey("_id")) { recordData.Remove("_id"); } if (recordData.ContainsKey("_rev")) { recordData.Remove("_rev"); } // find previous record in database, return null if it doesn't exist var findQuery = $"{{\"selector\":{{\"_id\": {{\"$eq\":\"{recordId}\"}}}}}}"; var previousRecordResponse = await client.PostAsync($"{databaseName}/_find", new StringContent( findQuery, Encoding.UTF8, "application/json")); previousRecordResponse.EnsureSuccessStatusCode(); var documents = JObject.Parse(await previousRecordResponse.Content.ReadAsStringAsync())["docs"]; // get and check previous record Logger.Info("get previous record"); var previousRecord = documents.ToList().FirstOrDefault(); Logger.Info($"previous record: {previousRecord}"); if (previousRecord == null && recordData.Count != 0) { // set previous record to current record Logger.Info($"databaseName:{databaseName} | recordId: {record.RecordId} - INSERT"); var createDocUri = $"/{databaseName}/{recordId}"; await client.PutAsync(createDocUri, new StringContent(JsonConvert.SerializeObject(recordData), Encoding.UTF8, "application/json")); return(""); } if (previousRecord != null) { if (recordData.Count == 0) { Logger.Info($"databaseName:{databaseName} | recordId: {record.RecordId} - DELETE"); var deleteDocUri = $"/{databaseName}/{recordId}?{previousRecord["rev"]}"; await client.DeleteAsync(deleteDocUri); } else { // update record and remove/add version Logger.Info($"shapeId; {databaseName} | recordId: {record.RecordId} - UPSERT"); var reviseDocUri = $"/{databaseName}/{recordId}"; // add _rev to recordData recordData.Add("_rev", previousRecord["rev"]); await client.PutAsync(reviseDocUri, new StringContent(JsonConvert.SerializeObject(recordData), Encoding.UTF8, "application/json")); } } return(""); } catch (Exception e) { Logger.Error(e, e.Message); throw; } }
public IImportExportFile MakeImportExportFile(SqlDatabaseConnection sqlDatabaseConnection, ConfigureReplicationFormData replicationFormData, string tableName, string schemaName) { return(new DelimitedImportExport(sqlDatabaseConnection, tableName, schemaName, replicationFormData)); }
public static void WriteToDisk(IImportExportFile goldenImportExport, IImportExportFile versionImportExport, ConfigureReplicationFormData config, Settings settings, bool forceWrite = false) { // check if 5 seconds have passed since last write to disk if (forceWrite || (DateTime.Now - LastWriteTime).TotalSeconds >= 5 && PendingWrites) { // write out to disk goldenImportExport.ExportTable(config.GetLocalGoldenFilePath()); versionImportExport.ExportTable(config.GetLocalVersionFilePath()); PendingWrites = false; // write to Remote if (config.FileWriteMode != Constants.FileModeLocal) { var localGoldenFileName = config.GetLocalGoldenFilePath(); var remoteGoldenFileName = config.GetRemoteGoldenFilePath(); var localVersionFileName = config.GetLocalVersionFilePath(); var remoteVersionFileName = config.GetRemoteVersionFilePath(); switch (config.FileWriteMode) { case Constants.FileModeFtp: using (var client = Utility.Utility.GetFtpClient()) { try { var status = client.UploadFile(localGoldenFileName, remoteGoldenFileName); if (status == FtpStatus.Failed) { throw new Exception($"Could not write file to remote {remoteGoldenFileName}"); } status = client.UploadFile(localVersionFileName, remoteVersionFileName); if (status == FtpStatus.Failed) { throw new Exception($"Could not write file to remote {remoteVersionFileName}"); } } finally { client.Disconnect(); } } break; case Constants.FileModeSftp: using (var client = Utility.Utility.GetSftpClient()) { try { var goldenFileStream = Utility.Utility.GetStream(localGoldenFileName, Constants.FileModeLocal, true); client.UploadFile(goldenFileStream.Stream, remoteGoldenFileName); goldenFileStream.Close(); var versionFileStream = Utility.Utility.GetStream(localVersionFileName, Constants.FileModeLocal, true); client.UploadFile(versionFileStream.Stream, remoteVersionFileName); versionFileStream.Close(); } catch { throw new Exception( $"Could not write files to remote {remoteGoldenFileName} {remoteVersionFileName}"); } finally { client.Disconnect(); } } break; } } } }
public IImportExportFile MakeImportExportFile(SqlDatabaseConnection sqlDatabaseConnection, ConfigureReplicationFormData replicationFormData, string tableName, string schemaName) { throw new System.NotImplementedException(); }
/// <summary> /// Adds and removes records to replication db /// Adds and updates available shapes /// </summary> /// <param name="conn"></param> /// <param name="schema"></param> /// <param name="record"></param> /// <param name="config"></param> /// <param name="responseStream"></param> /// <returns>Error message string</returns> public static async Task <string> WriteRecordAsync(SqlDatabaseConnection conn, Schema schema, Record record, ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream) { // debug Logger.Debug($"Starting timer for {record.RecordId}"); var timer = Stopwatch.StartNew(); try { // debug Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented)); // semaphore await WriteSemaphoreSlim.WaitAsync(); // setup var safeSchemaName = Constants.SchemaName; var safeTargetTableName = config.GetGoldenTableName(); var targetTable = Replication.Replication.GetGoldenReplicationTable(schema, safeSchemaName, safeTargetTableName); // get record var recordData = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson); recordData[Constants.ReplicationRecordId] = record.RecordId; recordData[Constants.ReplicationVersionIds] = null; // write data if (recordData.Count == 2) { // delete record Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE"); await Replication.Replication.DeleteRecordAsync(conn, targetTable, record.RecordId); } else { // add in all default values foreach (var property in schema.Properties) { if (!recordData.ContainsKey(property.Id) && !string.IsNullOrWhiteSpace(property.PublisherMetaJson)) { Logger.Debug("adding default value"); var columnConfig = JsonConvert.DeserializeObject <WriteColumn>(property.PublisherMetaJson); recordData[property.Id] = columnConfig.DefaultValue; } } Logger.Debug(JsonConvert.SerializeObject(recordData, Formatting.Indented)); // update record Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT"); await Replication.Replication.UpsertRecordAsync(conn, targetTable, recordData); } // set triggers for async file write LastWriteTime = DateTime.Now; PendingWrites = true; var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = "" }; await responseStream.WriteAsync(ack); timer.Stop(); Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}"); return(""); } catch (Exception e) { Logger.Error(e, $"Error replicating records {e.Message}"); // send ack var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = e.Message }; await responseStream.WriteAsync(ack); timer.Stop(); Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}"); if (e.Message.Contains("library routine called out of sequence")) { throw; } return(e.Message); } finally { WriteSemaphoreSlim.Release(); } }
public static List <string> ValidateReplicationFormData(this ConfigureReplicationFormData data, Settings settings) { var errors = new List <string>(); if (string.IsNullOrWhiteSpace(data.GoldenRecordFileDirectory)) { errors.Add("Golden Record file directory is empty."); } if (string.IsNullOrWhiteSpace(data.GoldenRecordFileName)) { errors.Add("Golden Record file name is empty."); } if (string.IsNullOrWhiteSpace(data.VersionRecordFileDirectory)) { errors.Add("Version Record file directory is empty."); } if (string.IsNullOrWhiteSpace(data.VersionRecordFileName)) { errors.Add("Version Record file name is empty."); } if (string.IsNullOrWhiteSpace(data.FileWriteMode)) { errors.Add("File Write Mode is empty."); } if (string.IsNullOrWhiteSpace(data.NullValue)) { data.NullValue = ""; } if (string.IsNullOrWhiteSpace(data.CustomHeader)) { data.CustomHeader = ""; } if (data.FileWriteMode != Constants.FileModeLocal) { var goldenTestFileName = "golden_test.txt"; var remoteGoldenDirectory = data.GetRemoteGoldenDirectory(); var remoteGoldenTestFileName = Path.Join(remoteGoldenDirectory, goldenTestFileName); var localGoldenTestDirectory = data.GetLocalGoldenDirectory(); var localGoldenTestFileName = Path.Join(localGoldenTestDirectory, goldenTestFileName); Directory.CreateDirectory(localGoldenTestDirectory); var goldenTestFile = new StreamWriter(localGoldenTestFileName); goldenTestFile.WriteLine("test"); goldenTestFile.Close(); var versionTestFileName = "version_test.txt"; var remoteVersionDirectory = data.GetRemoteVersionDirectory(); var remoteVersionTestFileName = Path.Join(remoteVersionDirectory, versionTestFileName); var localVersionTestDirectory = data.GetLocalVersionDirectory(); var localVersionTestFileName = Path.Join(localVersionTestDirectory, versionTestFileName); Directory.CreateDirectory(localVersionTestDirectory); var testFile = new StreamWriter(localVersionTestFileName); testFile.WriteLine("test"); testFile.Close(); switch (data.FileWriteMode) { case Constants.FileModeFtp: using (var client = Utility.Utility.GetFtpClient()) { try { if (!client.DirectoryExists(remoteGoldenDirectory)) { errors.Add($"{remoteGoldenDirectory} is not a directory on remote FTP"); } else { var status = client.UploadFile(localGoldenTestFileName, remoteGoldenTestFileName); if (status == FtpStatus.Failed) { errors.Add($"Could not write to golden directory {remoteGoldenDirectory}"); } } Utility.Utility.DeleteFileAtPath(localGoldenTestFileName, data, settings, true); if (!client.DirectoryExists(remoteVersionDirectory)) { errors.Add($"{remoteVersionDirectory} is not a directory on remote FTP"); } else { var status = client.UploadFile(localVersionTestFileName, remoteVersionTestFileName); if (status == FtpStatus.Failed) { errors.Add( $"Could not write to version directory {remoteVersionDirectory}"); } } Utility.Utility.DeleteFileAtPath(localVersionTestFileName, data, settings, true); } finally { client.Disconnect(); } } break; case Constants.FileModeSftp: using (var client = Utility.Utility.GetSftpClient()) { try { try { if (!client.Exists(remoteGoldenDirectory)) { errors.Add($"{remoteGoldenDirectory} is not a directory on remote FTP"); } else { var fileStream = Utility.Utility.GetStream(localGoldenTestFileName, Constants.FileModeLocal, true); client.UploadFile(fileStream.Stream, remoteGoldenTestFileName); fileStream.Close(); Utility.Utility.DeleteFileAtPath(localGoldenTestFileName, data, settings, true); } } catch { errors.Add($"Could not write to golden directory {remoteGoldenDirectory}"); } try { if (!client.Exists(remoteVersionDirectory)) { errors.Add($"{remoteVersionDirectory} is not a directory on remote FTP"); } else { var fileStream = Utility.Utility.GetStream(localVersionTestFileName, Constants.FileModeLocal, true); client.UploadFile(fileStream.Stream, remoteVersionTestFileName); fileStream.Close(); Utility.Utility.DeleteFileAtPath(localVersionTestFileName, data, settings, true); } } catch { errors.Add($"Could not write to version directory {remoteVersionDirectory}"); } } finally { client.Disconnect(); } } break; } } return(errors); }
/// <summary> /// Adds and removes records to replication db /// Adds and updates available shapes /// </summary> /// <param name="connFactory"></param> /// <param name="schema"></param> /// <param name="record"></param> /// <param name="config"></param> /// <param name="responseStream"></param> /// <returns>Error message string</returns> public static async Task <string> WriteRecordAsync(IConnectionFactory connectionFactory, Schema schema, Record record, ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream) { Logger.Debug($"Starting {record.RecordId}"); Stopwatch timer = Stopwatch.StartNew(); try { Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented)); await WriteSemaphoreSlim.WaitAsync(); string safeSchemaName = config.SchemaName; string safeGoldenTableName = config.GoldenTableName; string safeVersionTableName = config.VersionTableName; ReplicationTable goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName); ReplicationTable versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName); List <string> recordVersionIds = record.Versions.Select(r => r.RecordId).ToList(); // GetNamed Record Data // TODO: Finish return(null); } catch (Exception ex) { Logger.Error(ex, $"Error writing record: {ex.Message}"); RecordAck ack = new RecordAck { CorrelationId = record.CorrelationId, Error = ex.Message }; await responseStream.WriteAsync(ack); timer.Stop(); return(ex.Message); } finally { Logger.Debug($"Stopped {record.RecordId}. Time: {timer.ElapsedMilliseconds}"); WriteSemaphoreSlim.Release(); } }