public void CopyTable(TransferParameters transferParameters, string sourceCnStr, ConnectParameters targetConnector, string referenceJson) { _targetConnector = targetConnector; _sourceConnectString = sourceCnStr; _referenceJson = referenceJson; _transferParameters = transferParameters; string destCnStr = targetConnector.ConnectionString; string table = transferParameters.Table; SqlConnection sourceConn = new SqlConnection(sourceCnStr); SqlConnection destinationConn = new SqlConnection(destCnStr); try { sourceConn.Open(); destinationConn.Open(); BulkCopy(sourceConn, destinationConn, transferParameters); } catch (Exception ex) { Exception error = new Exception($"Sorry! Error copying table: {table}; {ex}"); throw error; } finally { sourceConn.Close(); destinationConn.Close(); } }
internal static void SaveCommunityCookies(CookieContainer cookies) { TransferParameters transferParams = GetTransferParameters(); if (transferParams == null) { transferParams = new TransferParameters(); } foreach (Cookie cookie in cookies.GetCookies(new Uri("https://steamcommunity.com/"))) { if (cookie.Name == "steamRememberLogin") { string[] bits = WebUtility.UrlDecode(cookie.Value).Split(new[] { "||" }, 2, StringSplitOptions.None); transferParams.SteamId = ulong.Parse(bits[0]); transferParams.RememberLoginToken = bits[1]; } else if (cookie.Name.StartsWith("steamMachineAuth")) { string steamId = cookie.Name.Replace("steamMachineAuth", ""); transferParams.SteamId = ulong.Parse(steamId); transferParams.WebCookie = cookie.Value; } } SaveTransferParameters(transferParams); }
public static async Task <IActionResult> CopyData( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("TransferData: Starting"); try { string storageAccount = Common.Helpers.Common.GetStorageKey(req); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); TransferParameters transParm = JsonConvert.DeserializeObject <TransferParameters>(requestBody); if (transParm == null) { log.LogError("TransferData: error missing transfer parameters"); return(new BadRequestObjectResult("Error missing transfer parameters")); } DataTransfer dt = new DataTransfer(storageAccount); await dt.CopyFiles(transParm); } catch (Exception ex) { log.LogError($"TransferData: Error getting rules: {ex}"); return(new BadRequestObjectResult($"Error getting rules: {ex}")); } log.LogInformation("TransferData: Complete"); return(new OkObjectResult("OK")); }
public void DataLakeUploader_UploadDownloadSingleSegment() { var frontEnd = new InMemoryFrontEnd(); var up = new TransferParameters( inputFilePath: _smallFilePath, targetStreamPath: "1", perFileThreadCount: ThreadCount, accountName: "foo", isResume: false, maxSegmentLength: 4 * 1024 * 1024, localMetadataLocation: Path.GetTempPath()); File.WriteAllBytes(_smallFilePath, _smallFileData); var uploader = new DataLakeStoreTransferClient(up, frontEnd); uploader.Execute(); VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData); up = new TransferParameters( inputFilePath: "1", targetStreamPath: _downloadFilePath, perFileThreadCount: ThreadCount, accountName: "foo", isResume: false, isOverwrite: true, isDownload: true, maxSegmentLength: 4 * 1024 * 1024, localMetadataLocation: Path.GetTempPath()); // now download uploader = new DataLakeStoreTransferClient(up, frontEnd); uploader.Execute(); VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData); }
public ActionResult Delete(TransferParameters transferParameters) { ConnectParameters connector = Common.GetConnectParameters(connectionString, container, transferParameters.TargetName); DbUtilities dbConn = new DbUtilities(); string table = transferParameters.Table; try { dbConn.OpenConnection(connector); if (String.IsNullOrEmpty(table)) { return(BadRequest()); } dbConn.DBDelete(table); } catch (Exception) { return(BadRequest()); } string message = $"{table} has been cleared"; dbConn.CloseConnection(); return(Ok(message)); }
private void InsertQueryData(SqlConnection source, TransferParameters transferParameters) { string query = transferParameters.TransferQuery; string queryType = transferParameters.QueryType; string sql; if (queryType == "File") { sql = $"DECLARE @Array NVARCHAR(MAX) = '{query}' " + "INSERT INTO #PDOList ( UWI ) " + "SELECT * FROM STRING_SPLIT(@Array, ',')"; } else { sql = $"INSERT INTO #PDOList (UWI) SELECT UWI FROM WELL {query} "; } using (SqlCommand cmd = new SqlCommand(sql, source)) { try { cmd.CommandTimeout = 3000; cmd.ExecuteNonQuery(); } catch (SqlException ex) { Exception error = new Exception("Error inserting into table: ", ex); throw error; } } }
private void CopyTable(TransferParameters transferParameters) { ConnectParameters destination = Common.GetConnectParameters(connectionString, container, transferParameters.TargetName); string destCnStr = destination.ConnectionString; ConnectParameters source = Common.GetConnectParameters(connectionString, container, transferParameters.SourceName); string sourceCnStr = source.ConnectionString; string table = transferParameters.Table; SqlConnection sourceConn = new SqlConnection(sourceCnStr); SqlConnection destinationConn = new SqlConnection(destCnStr); try { sourceConn.Open(); destinationConn.Open(); BulkCopy(sourceConn, destinationConn, table); } catch (Exception ex) { Exception error = new Exception($"Sorry! Error copying table: {table}; {ex}"); throw error; } finally { sourceConn.Close(); destinationConn.Close(); } }
public async Task Copy(TransferParameters transferParameters) { var response = await httpService.Post(copyUrl, transferParameters); if (!response.Success) { throw new ApplicationException(await response.GetBody()); } }
public static async Task <string> DeleteDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"DeleteDataTransfer: Starting deleting"); DataTransfer dt = new DataTransfer(pipe.StorageAccount); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); await dt.DeleteTable(parms.TargetName, parms.Table); log.LogInformation($"DeleteDataTransfer: Complete deleting {parms.Table}"); return($"DeleteDataTransfer Complete"); }
public async Task Copy(TransferParameters transferParameters) { string url = baseUrl.BuildFunctionUrl("TransferData", $"", apiKey); var response = await httpService.Post(url, transferParameters); if (!response.Success) { throw new ApplicationException(await response.GetBody()); } }
public static async Task <string> DataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"DataTransfer: Starting"); DataTransfer dt = new DataTransfer(pipe.StorageAccount); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); await dt.CopyFiles(parms); log.LogInformation($"DataTransfer: Complete copying {parms.Table}"); return($"OK"); }
public static async Task <List <string> > InitDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log) { log.LogInformation($"InitDataTransfer: Starting"); DataTransfer dt = new DataTransfer(pipe.StorageAccount); TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>(); List <string> files = await dt.GetFiles(parms.SourceName); log.LogInformation($"InitDataTransfer: Complete"); return(files); }
public async Task CopyRemote(TransferParameters transferParameters) { string url = baseUrl.BuildFunctionUrl("TransferRemote", $"", apiKey); Console.WriteLine($"Copy: {url}"); var response = await httpService.Post(url, transferParameters); if (!response.Success) { throw new ApplicationException(await response.GetBody()); } }
public static Transfer[] GetTransfers(TransferParameters parameters, out int rowsCount) { List <Transfer> l = new List <Transfer>(); using (SqlCommand cmd = new SqlCommand()) { cmd.CommandType = CommandType.StoredProcedure; cmd.Connection = new SqlConnection(cstr.con); cmd.Connection.Open(); cmd.CommandText = "GetTransfers"; /*cmd.Parameters.AddWithValue("status", parameters.Status); * cmd.Parameters.AddWithValue("page", parameters.CurrentPage); * cmd.Parameters.AddWithValue("pageLength", parameters.PageLength); * SqlParameter rowsCountParam = cmd.Parameters.Add("rowsCount", SqlDbType.Int); * rowsCountParam.Direction = ParameterDirection.InputOutput;*/ SqlDataReader r = cmd.ExecuteReader(); if (r.HasRows) { while (r.Read()) { Transfer c = new Transfer(); if (r["id"] != DBNull.Value) { c.Id = Convert.ToInt32(r["id"]); } if (r["senderOTP"] != DBNull.Value) { c.SenderOTP = Convert.ToString(r["senderOTP"]); } if (r["meter_id"] != DBNull.Value) { c.MeterId = Convert.ToInt32(r["meter_id"]); } if (r["amount"] != DBNull.Value) { c.Amount = Convert.ToDecimal(r["amount"]); } l.Add(c); } } r.Close(); cmd.Connection.Close(); //rowsCount = Convert.ToInt32(rowsCountParam.Value); rowsCount = l.Count; } return(l.ToArray()); }
public void FillCircle(TransferParameters transferParameters) { if (_JsInteropExecutionMode == 0) { BCCCJsInterop.Draw_Circle(canvasID, transferParameters); } else { SB_Append(" ctx1.arc(" + transferParameters.x + ", " + transferParameters.y + ", " + transferParameters.r + ", " + transferParameters.sAngle + ", " + transferParameters.eAngle + ");"); } }
private DoLoginResponse GetDoLoginResponse() { DoLoginResponse doLoginResponse = new DoLoginResponse(); TransferParameters transferParameters = new TransferParameters(); transferParameters.Auth = "b9wr1irasauno6rjngp3z2cfwf3qk3an"; transferParameters.RememberLogin = true; transferParameters.SteamId = "76561198028630048"; transferParameters.Token = "7ACRVJ9F3D1ETEN9HKFPWSEXZ3X020KRBEZ6HDA7"; transferParameters.TokenSecure = "O8W437FVTL29F9I31C6H48R0MBNAVIH5ILUMV4FW"; doLoginResponse.TransferParameters = transferParameters; return(doLoginResponse); }
public async Task <ActionResult <string> > CopyData(TransferParameters transferParameters) { try { string tmpConnString = Request.Headers["AzureStorageConnection"]; DataTransfer dt = new DataTransfer(tmpConnString); await dt.CopyFiles(transferParameters); } catch (Exception ex) { return(BadRequest(ex.ToString())); } string message = $"{transferParameters.Table} has been copied"; return(Ok(message)); }
public async Task <ActionResult <string> > CopyRemote(TransferParameters transferParameters) { try { string tmpConnString = Request.Headers["AzureStorageConnection"]; DataTransfer dt = new DataTransfer(tmpConnString); dt.CopyRemote(transferParameters); } catch (Exception) { return(BadRequest("Problems with data transfer queue")); } string response = $"{transferParameters.Table} has started on remote computer"; return(Ok(response)); }
internal static void SaveTransferParameters(TransferParameters transferParams) { TransferParameters newParams = new TransferParameters { SteamId = transferParams.SteamId, WebCookie = transferParams.WebCookie, RememberLoginToken = transferParams.RememberLoginToken }; string serialized = JsonConvert.SerializeObject(newParams); byte[] blob = Encoding.UTF8.GetBytes(serialized); byte[] cryptedBlob = ProtectedData.Protect(blob, GetStrongNameKey(), DataProtectionScope.CurrentUser); string cryptedParams = Convert.ToBase64String(cryptedBlob); Properties.Settings.Default.Credentials = cryptedParams; Properties.Settings.Default.Save(); }
public ActionResult Copy(TransferParameters transferParameters) { string message = ""; string table = transferParameters.Table; try { CopyTable(transferParameters); } catch (Exception ex) { return(BadRequest(ex.ToString())); } message = $"{table} has been copied"; return(Ok(message)); }
/// <summary> /// Verifies the file was successfully uploaded. /// </summary> /// <param name="up">The upload parameters.</param> /// <param name="frontEnd">The front end.</param> private void VerifyFileUploadedSuccessfully(TransferParameters up, InMemoryFrontEnd frontEnd) { if (up.IsRecursive) { var fileList = new Dictionary <string, byte[]> { { string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_largeFilePath)), _largeFileData }, { string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_smallFilePath)), _smallFileData }, { string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_downloadFilePath)), _smallFileData } }; VerifyFileUploadedSuccessfully(fileList, frontEnd, up.IsDownload); } else { VerifyFileUploadedSuccessfully(up, frontEnd, _largeFileData); } }
internal static CookieContainer GetCommunityCookies() { TransferParameters transferParams = GetTransferParameters(); if (transferParams == null) { return(null); } CookieContainer cookies = new CookieContainer(); // 1. Persistent login cookie cookies.Add(new Cookie("steamRememberLogin", WebUtility.UrlEncode(string.Format("{0}||{1}", transferParams.SteamId, transferParams.RememberLoginToken)), "/", "steamcommunity.com")); // 2. Machine auth token cookies.Add(new Cookie(string.Format("steamMachineAuth{0}", transferParams.SteamId), transferParams.WebCookie, "/", "steamcommunity.com") { Secure = true }); return(cookies); }
public async Task CopyFiles(TransferParameters parms) { try { ConnectParameters sourceConnector = await Common.GetConnectParameters(_azureConnectionString, parms.SourceName); ConnectParameters targetConnector = await Common.GetConnectParameters(_azureConnectionString, parms.TargetName); string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); string referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json"); targetConnector.DataAccessDefinition = dataAccessDefinition; if (sourceConnector.SourceType == "DataBase") { DatabaseLoader dl = new DatabaseLoader(); dl.CopyTable(parms, sourceConnector.ConnectionString, targetConnector, referenceJson); } else if (sourceConnector.SourceType == "File") { if (sourceConnector.DataType == "Logs") { LASLoader ls = new LASLoader(_fileStorage); await ls.LoadLASFile(sourceConnector, targetConnector, parms.Table, referenceJson); } else { CSVLoader cl = new CSVLoader(_fileStorage); await cl.LoadCSVFile(sourceConnector, targetConnector, parms.Table); } } else { Exception error = new Exception($"DataTransfer: Not a valid source type for {sourceConnector.SourceName}"); throw error; } } catch (Exception ex) { Exception error = new Exception($"DataTransfer: Problems transfer files/tables, {ex}"); throw error; } }
private void BulkCopy(SqlConnection source, SqlConnection destination, TransferParameters transferParameters) { string sql = ""; string table = transferParameters.Table; string query = ""; if (CopyTables.dictionary[table] == "TABLE") { query = transferParameters.TransferQuery; } if (string.IsNullOrEmpty(query)) { sql = $"select * from {table}"; } else { CreateTempTable(source); InsertQueryData(source, transferParameters); sql = $"select * from {table} where UWI in (select UWI from #PDOList)"; } ProcessReferenceTables(table, source, destination); using (SqlCommand cmd = new SqlCommand(sql, source)) { try { cmd.CommandTimeout = 3600; SqlDataReader reader = cmd.ExecuteReader(); SqlBulkCopy bulkData = new SqlBulkCopy(destination); bulkData.DestinationTableName = table; bulkData.BulkCopyTimeout = 1000; bulkData.WriteToServer(reader); bulkData.Close(); } catch (SqlException ex) { Exception error = new Exception($"Sorry! Error copying table: {table}; {ex}"); throw error; } } }
public void UploadMetadataGenerator_AlignSegmentsToRecordBoundaries() { //We keep creating a file, by appending a number of bytes to it (taken from FileLengthsInMB). //At each iteration, we append a new blob of data, and then run the whole test on the entire file var rnd = new Random(0); string folderPath = string.Format(@"{0}\uploadtest", Path.GetTempPath()); string filePath = Path.Combine(folderPath, "verifymetadata.txt"); try { if (!Directory.Exists(folderPath)) { Directory.CreateDirectory(folderPath); } if (File.Exists(filePath)) { File.Delete(filePath); } foreach (var lengthMB in FileLengthsMB) { var appendLength = (int)(lengthMB * 1024 * 1024); AppendToFile(filePath, appendLength, rnd, 0, MaxAppendLength); string metadataFilePath = filePath + ".metadata.txt"; var up = new TransferParameters(filePath, filePath, null, isBinary: false, maxSegmentLength: 4 * 1024 * 1024); var mg = new TransferMetadataGenerator(up, new InMemoryFrontEnd()); var metadata = mg.CreateNewMetadata(metadataFilePath); VerifySegmentsAreOnRecordBoundaries(metadata, filePath); } } finally { if (Directory.Exists(folderPath)) { Directory.Delete(folderPath, true); } } }
public async Task GetTransferConnector(string message) { try { TransferParameters transParms = JsonConvert.DeserializeObject <TransferParameters>(message); _targetConnector = GetConnectionString(transParms.TargetName); string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json"); _targetConnector.DataAccessDefinition = dataAccessDefinition; target = _targetConnector.ConnectionString; _logger.LogInformation($"Target connect string: {target}"); ConnectParameters sourceConnector = GetConnectionString(transParms.SourceName); source = sourceConnector.ConnectionString; _logger.LogInformation($"Source connect string: {source}"); transferQuery = transParms.TransferQuery; queryType = transParms.QueryType; _referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json"); } catch (Exception ex) { _logger.LogInformation($"Error getting connector info {ex.ToString()}"); } }
public void FillGauge(string color, TransferParameters transferParameters) { if (_JsInteropExecutionMode == 0) { BCCCJsInterop.Draw_Gauge(canvasID, color, transferParameters); } else { SB_Append(" var gradient = ctx1.createRadialGradient(0, 0, 0, 0, 0, " + transferParameters.r + ");"); SB_Append(" gradient.addColorStop(0, 'white');"); SB_Append(" gradient.addColorStop(1, \"" + color + "\");"); SB_Append(" ctx1.beginPath();"); SB_Append(" ctx1.moveTo(0, 0);"); SB_Append(" ctx1.arc(" + transferParameters.x + ", " + transferParameters.y + ", " + transferParameters.r + ", " + transferParameters.sAngle + ", " + transferParameters.eAngle + ");"); SB_Append(" ctx1.fillStyle = gradient;"); SB_Append(" ctx1.closePath();"); SB_Append(" ctx1.fill();"); } }
public void CopyRemote(TransferParameters parms) { string message = JsonConvert.SerializeObject(parms); _queueService.InsertMessage(queueName, message); }
public static Task <bool> Draw_Gauge(string canvasID, string color, TransferParameters transferParameters) { return(JSRuntime.Current.InvokeAsync <bool>("JsInteropClockCanvas.Draw_Gauge", new { canvasID, color, transferParameters })); }
public static Task <bool> Run(TransferParameters _params) { return(JSRuntime.Current.InvokeAsync <bool>( "JsInterop2.Run", _params)); }