public void CopyTable(TransferParameters transferParameters, string sourceCnStr, ConnectParameters targetConnector, string referenceJson)
{
_targetConnector = targetConnector;
_sourceConnectString = sourceCnStr;
_referenceJson = referenceJson;
_transferParameters = transferParameters;
string destCnStr = targetConnector.ConnectionString;
string table = transferParameters.Table;
SqlConnection sourceConn = new SqlConnection(sourceCnStr);
SqlConnection destinationConn = new SqlConnection(destCnStr);
try
{
sourceConn.Open();
destinationConn.Open();
BulkCopy(sourceConn, destinationConn, transferParameters);
}
catch (Exception ex)
{
Exception error = new Exception($"Sorry! Error copying table: {table}; {ex}");
throw error;
}
finally
{
sourceConn.Close();
destinationConn.Close();
}
}
internal static void SaveCommunityCookies(CookieContainer cookies)
{
TransferParameters transferParams = GetTransferParameters();
if (transferParams == null)
{
transferParams = new TransferParameters();
}
foreach (Cookie cookie in cookies.GetCookies(new Uri("https://steamcommunity.com/")))
{
if (cookie.Name == "steamRememberLogin")
{
string[] bits = WebUtility.UrlDecode(cookie.Value).Split(new[] { "||" }, 2, StringSplitOptions.None);
transferParams.SteamId = ulong.Parse(bits[0]);
transferParams.RememberLoginToken = bits[1];
}
else if (cookie.Name.StartsWith("steamMachineAuth"))
{
string steamId = cookie.Name.Replace("steamMachineAuth", "");
transferParams.SteamId = ulong.Parse(steamId);
transferParams.WebCookie = cookie.Value;
}
}
SaveTransferParameters(transferParams);
}
public static async Task <IActionResult> CopyData(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("TransferData: Starting");
try
{
string storageAccount = Common.Helpers.Common.GetStorageKey(req);
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
TransferParameters transParm = JsonConvert.DeserializeObject <TransferParameters>(requestBody);
if (transParm == null)
{
log.LogError("TransferData: error missing transfer parameters");
return(new BadRequestObjectResult("Error missing transfer parameters"));
}
DataTransfer dt = new DataTransfer(storageAccount);
await dt.CopyFiles(transParm);
}
catch (Exception ex)
{
log.LogError($"TransferData: Error getting rules: {ex}");
return(new BadRequestObjectResult($"Error getting rules: {ex}"));
}
log.LogInformation("TransferData: Complete");
return(new OkObjectResult("OK"));
}
public void DataLakeUploader_UploadDownloadSingleSegment()
{
var frontEnd = new InMemoryFrontEnd();
var up = new TransferParameters(
inputFilePath: _smallFilePath,
targetStreamPath: "1",
perFileThreadCount: ThreadCount,
accountName: "foo",
isResume: false,
maxSegmentLength: 4 * 1024 * 1024,
localMetadataLocation: Path.GetTempPath());
File.WriteAllBytes(_smallFilePath, _smallFileData);
var uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
up = new TransferParameters(
inputFilePath: "1",
targetStreamPath: _downloadFilePath,
perFileThreadCount: ThreadCount,
accountName: "foo",
isResume: false,
isOverwrite: true,
isDownload: true,
maxSegmentLength: 4 * 1024 * 1024,
localMetadataLocation: Path.GetTempPath());
// now download
uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
}
public ActionResult Delete(TransferParameters transferParameters)
{
ConnectParameters connector = Common.GetConnectParameters(connectionString, container,
transferParameters.TargetName);
DbUtilities dbConn = new DbUtilities();
string table = transferParameters.Table;
try
{
dbConn.OpenConnection(connector);
if (String.IsNullOrEmpty(table))
{
return(BadRequest());
}
dbConn.DBDelete(table);
}
catch (Exception)
{
return(BadRequest());
}
string message = $"{table} has been cleared";
dbConn.CloseConnection();
return(Ok(message));
}
private void InsertQueryData(SqlConnection source, TransferParameters transferParameters)
{
string query = transferParameters.TransferQuery;
string queryType = transferParameters.QueryType;
string sql;
if (queryType == "File")
{
sql = $"DECLARE @Array NVARCHAR(MAX) = '{query}' " +
"INSERT INTO #PDOList ( UWI ) " +
"SELECT * FROM STRING_SPLIT(@Array, ',')";
}
else
{
sql = $"INSERT INTO #PDOList (UWI) SELECT UWI FROM WELL {query} ";
}
using (SqlCommand cmd = new SqlCommand(sql, source))
{
try
{
cmd.CommandTimeout = 3000;
cmd.ExecuteNonQuery();
}
catch (SqlException ex)
{
Exception error = new Exception("Error inserting into table: ", ex);
throw error;
}
}
}
private void CopyTable(TransferParameters transferParameters)
{
ConnectParameters destination = Common.GetConnectParameters(connectionString, container,
transferParameters.TargetName);
string destCnStr = destination.ConnectionString;
ConnectParameters source = Common.GetConnectParameters(connectionString, container,
transferParameters.SourceName);
string sourceCnStr = source.ConnectionString;
string table = transferParameters.Table;
SqlConnection sourceConn = new SqlConnection(sourceCnStr);
SqlConnection destinationConn = new SqlConnection(destCnStr);
try
{
sourceConn.Open();
destinationConn.Open();
BulkCopy(sourceConn, destinationConn, table);
}
catch (Exception ex)
{
Exception error = new Exception($"Sorry! Error copying table: {table}; {ex}");
throw error;
}
finally
{
sourceConn.Close();
destinationConn.Close();
}
}
public async Task Copy(TransferParameters transferParameters)
{
var response = await httpService.Post(copyUrl, transferParameters);
if (!response.Success)
{
throw new ApplicationException(await response.GetBody());
}
}
public static async Task <string> DeleteDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"DeleteDataTransfer: Starting deleting");
DataTransfer dt = new DataTransfer(pipe.StorageAccount);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
await dt.DeleteTable(parms.TargetName, parms.Table);
log.LogInformation($"DeleteDataTransfer: Complete deleting {parms.Table}");
return($"DeleteDataTransfer Complete");
}
public async Task Copy(TransferParameters transferParameters)
{
string url = baseUrl.BuildFunctionUrl("TransferData", $"", apiKey);
var response = await httpService.Post(url, transferParameters);
if (!response.Success)
{
throw new ApplicationException(await response.GetBody());
}
}
public static async Task <string> DataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"DataTransfer: Starting");
DataTransfer dt = new DataTransfer(pipe.StorageAccount);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
await dt.CopyFiles(parms);
log.LogInformation($"DataTransfer: Complete copying {parms.Table}");
return($"OK");
}
public static async Task <List <string> > InitDataTransfer([ActivityTrigger] DataOpParameters pipe, ILogger log)
{
log.LogInformation($"InitDataTransfer: Starting");
DataTransfer dt = new DataTransfer(pipe.StorageAccount);
TransferParameters parms = JObject.Parse(pipe.JsonParameters).ToObject <TransferParameters>();
List <string> files = await dt.GetFiles(parms.SourceName);
log.LogInformation($"InitDataTransfer: Complete");
return(files);
}
public async Task CopyRemote(TransferParameters transferParameters)
{
string url = baseUrl.BuildFunctionUrl("TransferRemote", $"", apiKey);
Console.WriteLine($"Copy: {url}");
var response = await httpService.Post(url, transferParameters);
if (!response.Success)
{
throw new ApplicationException(await response.GetBody());
}
}
public static Transfer[] GetTransfers(TransferParameters parameters, out int rowsCount)
{
List <Transfer> l = new List <Transfer>();
using (SqlCommand cmd = new SqlCommand())
{
cmd.CommandType = CommandType.StoredProcedure;
cmd.Connection = new SqlConnection(cstr.con);
cmd.Connection.Open();
cmd.CommandText = "GetTransfers";
/*cmd.Parameters.AddWithValue("status", parameters.Status);
* cmd.Parameters.AddWithValue("page", parameters.CurrentPage);
* cmd.Parameters.AddWithValue("pageLength", parameters.PageLength);
* SqlParameter rowsCountParam = cmd.Parameters.Add("rowsCount", SqlDbType.Int);
* rowsCountParam.Direction = ParameterDirection.InputOutput;*/
SqlDataReader r = cmd.ExecuteReader();
if (r.HasRows)
{
while (r.Read())
{
Transfer c = new Transfer();
if (r["id"] != DBNull.Value)
{
c.Id = Convert.ToInt32(r["id"]);
}
if (r["senderOTP"] != DBNull.Value)
{
c.SenderOTP = Convert.ToString(r["senderOTP"]);
}
if (r["meter_id"] != DBNull.Value)
{
c.MeterId = Convert.ToInt32(r["meter_id"]);
}
if (r["amount"] != DBNull.Value)
{
c.Amount = Convert.ToDecimal(r["amount"]);
}
l.Add(c);
}
}
r.Close();
cmd.Connection.Close();
//rowsCount = Convert.ToInt32(rowsCountParam.Value);
rowsCount = l.Count;
}
return(l.ToArray());
}
public void FillCircle(TransferParameters transferParameters)
{
if (_JsInteropExecutionMode == 0)
{
BCCCJsInterop.Draw_Circle(canvasID, transferParameters);
}
else
{
SB_Append(" ctx1.arc(" + transferParameters.x +
", " + transferParameters.y +
", " + transferParameters.r +
", " + transferParameters.sAngle +
", " + transferParameters.eAngle + ");");
}
}
private DoLoginResponse GetDoLoginResponse()
{
DoLoginResponse doLoginResponse = new DoLoginResponse();
TransferParameters transferParameters = new TransferParameters();
transferParameters.Auth = "b9wr1irasauno6rjngp3z2cfwf3qk3an";
transferParameters.RememberLogin = true;
transferParameters.SteamId = "76561198028630048";
transferParameters.Token =
"7ACRVJ9F3D1ETEN9HKFPWSEXZ3X020KRBEZ6HDA7";
transferParameters.TokenSecure =
"O8W437FVTL29F9I31C6H48R0MBNAVIH5ILUMV4FW";
doLoginResponse.TransferParameters = transferParameters;
return(doLoginResponse);
}
public async Task <ActionResult <string> > CopyData(TransferParameters transferParameters)
{
try
{
string tmpConnString = Request.Headers["AzureStorageConnection"];
DataTransfer dt = new DataTransfer(tmpConnString);
await dt.CopyFiles(transferParameters);
}
catch (Exception ex)
{
return(BadRequest(ex.ToString()));
}
string message = $"{transferParameters.Table} has been copied";
return(Ok(message));
}
public async Task <ActionResult <string> > CopyRemote(TransferParameters transferParameters)
{
try
{
string tmpConnString = Request.Headers["AzureStorageConnection"];
DataTransfer dt = new DataTransfer(tmpConnString);
dt.CopyRemote(transferParameters);
}
catch (Exception)
{
return(BadRequest("Problems with data transfer queue"));
}
string response = $"{transferParameters.Table} has started on remote computer";
return(Ok(response));
}
internal static void SaveTransferParameters(TransferParameters transferParams)
{
TransferParameters newParams = new TransferParameters
{
SteamId = transferParams.SteamId,
WebCookie = transferParams.WebCookie,
RememberLoginToken = transferParams.RememberLoginToken
};
string serialized = JsonConvert.SerializeObject(newParams);
byte[] blob = Encoding.UTF8.GetBytes(serialized);
byte[] cryptedBlob = ProtectedData.Protect(blob, GetStrongNameKey(), DataProtectionScope.CurrentUser);
string cryptedParams = Convert.ToBase64String(cryptedBlob);
Properties.Settings.Default.Credentials = cryptedParams;
Properties.Settings.Default.Save();
}
public ActionResult Copy(TransferParameters transferParameters)
{
string message = "";
string table = transferParameters.Table;
try
{
CopyTable(transferParameters);
}
catch (Exception ex)
{
return(BadRequest(ex.ToString()));
}
message = $"{table} has been copied";
return(Ok(message));
}
/// <summary>
/// Verifies the file was successfully uploaded.
/// </summary>
/// <param name="up">The upload parameters.</param>
/// <param name="frontEnd">The front end.</param>
private void VerifyFileUploadedSuccessfully(TransferParameters up, InMemoryFrontEnd frontEnd)
{
if (up.IsRecursive)
{
var fileList = new Dictionary <string, byte[]>
{
{ string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_largeFilePath)), _largeFileData },
{ string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_smallFilePath)), _smallFileData },
{ string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_downloadFilePath)), _smallFileData }
};
VerifyFileUploadedSuccessfully(fileList, frontEnd, up.IsDownload);
}
else
{
VerifyFileUploadedSuccessfully(up, frontEnd, _largeFileData);
}
}
internal static CookieContainer GetCommunityCookies()
{
TransferParameters transferParams = GetTransferParameters();
if (transferParams == null)
{
return(null);
}
CookieContainer cookies = new CookieContainer();
// 1. Persistent login cookie
cookies.Add(new Cookie("steamRememberLogin", WebUtility.UrlEncode(string.Format("{0}||{1}", transferParams.SteamId, transferParams.RememberLoginToken)), "/", "steamcommunity.com"));
// 2. Machine auth token
cookies.Add(new Cookie(string.Format("steamMachineAuth{0}", transferParams.SteamId), transferParams.WebCookie, "/", "steamcommunity.com")
{
Secure = true
});
return(cookies);
}
public async Task CopyFiles(TransferParameters parms)
{
try
{
ConnectParameters sourceConnector = await Common.GetConnectParameters(_azureConnectionString, parms.SourceName);
ConnectParameters targetConnector = await Common.GetConnectParameters(_azureConnectionString, parms.TargetName);
string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
string referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json");
targetConnector.DataAccessDefinition = dataAccessDefinition;
if (sourceConnector.SourceType == "DataBase")
{
DatabaseLoader dl = new DatabaseLoader();
dl.CopyTable(parms, sourceConnector.ConnectionString, targetConnector, referenceJson);
}
else if (sourceConnector.SourceType == "File")
{
if (sourceConnector.DataType == "Logs")
{
LASLoader ls = new LASLoader(_fileStorage);
await ls.LoadLASFile(sourceConnector, targetConnector, parms.Table, referenceJson);
}
else
{
CSVLoader cl = new CSVLoader(_fileStorage);
await cl.LoadCSVFile(sourceConnector, targetConnector, parms.Table);
}
}
else
{
Exception error = new Exception($"DataTransfer: Not a valid source type for {sourceConnector.SourceName}");
throw error;
}
}
catch (Exception ex)
{
Exception error = new Exception($"DataTransfer: Problems transfer files/tables, {ex}");
throw error;
}
}
private void BulkCopy(SqlConnection source, SqlConnection destination, TransferParameters transferParameters)
{
string sql = "";
string table = transferParameters.Table;
string query = "";
if (CopyTables.dictionary[table] == "TABLE")
{
query = transferParameters.TransferQuery;
}
if (string.IsNullOrEmpty(query))
{
sql = $"select * from {table}";
}
else
{
CreateTempTable(source);
InsertQueryData(source, transferParameters);
sql = $"select * from {table} where UWI in (select UWI from #PDOList)";
}
ProcessReferenceTables(table, source, destination);
using (SqlCommand cmd = new SqlCommand(sql, source))
{
try
{
cmd.CommandTimeout = 3600;
SqlDataReader reader = cmd.ExecuteReader();
SqlBulkCopy bulkData = new SqlBulkCopy(destination);
bulkData.DestinationTableName = table;
bulkData.BulkCopyTimeout = 1000;
bulkData.WriteToServer(reader);
bulkData.Close();
}
catch (SqlException ex)
{
Exception error = new Exception($"Sorry! Error copying table: {table}; {ex}");
throw error;
}
}
}
public void UploadMetadataGenerator_AlignSegmentsToRecordBoundaries()
{
//We keep creating a file, by appending a number of bytes to it (taken from FileLengthsInMB).
//At each iteration, we append a new blob of data, and then run the whole test on the entire file
var rnd = new Random(0);
string folderPath = string.Format(@"{0}\uploadtest", Path.GetTempPath());
string filePath = Path.Combine(folderPath, "verifymetadata.txt");
try
{
if (!Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
if (File.Exists(filePath))
{
File.Delete(filePath);
}
foreach (var lengthMB in FileLengthsMB)
{
var appendLength = (int)(lengthMB * 1024 * 1024);
AppendToFile(filePath, appendLength, rnd, 0, MaxAppendLength);
string metadataFilePath = filePath + ".metadata.txt";
var up = new TransferParameters(filePath, filePath, null, isBinary: false,
maxSegmentLength: 4 * 1024 * 1024);
var mg = new TransferMetadataGenerator(up, new InMemoryFrontEnd());
var metadata = mg.CreateNewMetadata(metadataFilePath);
VerifySegmentsAreOnRecordBoundaries(metadata, filePath);
}
}
finally
{
if (Directory.Exists(folderPath))
{
Directory.Delete(folderPath, true);
}
}
}
public async Task GetTransferConnector(string message)
{
try
{
TransferParameters transParms = JsonConvert.DeserializeObject <TransferParameters>(message);
_targetConnector = GetConnectionString(transParms.TargetName);
string dataAccessDefinition = await _fileStorage.ReadFile("connectdefinition", "PPDMDataAccess.json");
_targetConnector.DataAccessDefinition = dataAccessDefinition;
target = _targetConnector.ConnectionString;
_logger.LogInformation($"Target connect string: {target}");
ConnectParameters sourceConnector = GetConnectionString(transParms.SourceName);
source = sourceConnector.ConnectionString;
_logger.LogInformation($"Source connect string: {source}");
transferQuery = transParms.TransferQuery;
queryType = transParms.QueryType;
_referenceJson = await _fileStorage.ReadFile("connectdefinition", "PPDMReferenceTables.json");
}
catch (Exception ex)
{
_logger.LogInformation($"Error getting connector info {ex.ToString()}");
}
}
public void FillGauge(string color, TransferParameters transferParameters)
{
if (_JsInteropExecutionMode == 0)
{
BCCCJsInterop.Draw_Gauge(canvasID, color, transferParameters);
}
else
{
SB_Append(" var gradient = ctx1.createRadialGradient(0, 0, 0, 0, 0, " + transferParameters.r + ");");
SB_Append(" gradient.addColorStop(0, 'white');");
SB_Append(" gradient.addColorStop(1, \"" + color + "\");");
SB_Append(" ctx1.beginPath();");
SB_Append(" ctx1.moveTo(0, 0);");
SB_Append(" ctx1.arc(" + transferParameters.x + ", "
+ transferParameters.y + ", "
+ transferParameters.r + ", "
+ transferParameters.sAngle +
", " + transferParameters.eAngle + ");");
SB_Append(" ctx1.fillStyle = gradient;");
SB_Append(" ctx1.closePath();");
SB_Append(" ctx1.fill();");
}
}
public void CopyRemote(TransferParameters parms)
{
string message = JsonConvert.SerializeObject(parms);
_queueService.InsertMessage(queueName, message);
}
public static Task <bool> Draw_Gauge(string canvasID, string color, TransferParameters transferParameters)
{
return(JSRuntime.Current.InvokeAsync <bool>("JsInteropClockCanvas.Draw_Gauge",
new { canvasID, color, transferParameters }));
}
public static Task <bool> Run(TransferParameters _params)
{
return(JSRuntime.Current.InvokeAsync <bool>(
"JsInterop2.Run", _params));
}