/// <summary>
/// Used for receiving files
/// </summary>
/// <param name="fromFilePath"></param>
/// <returns></returns>
public Stream Get(string fromFilePath)
{
try
{
try
{
Connect();
return(_sftp.OpenRead(fromFilePath));
}
catch
{
ReConnect();
return(_sftp.OpenRead(fromFilePath));
}
}
catch (Exception ex)
{
throw ExceptionHandling.HandleComponentException(
EventLogEventIDs.UnableToGetFile,
System.Reflection.MethodBase.GetCurrentMethod(),
new SftpException("Unable to get file " + fromFilePath, ex));
}
finally
{
RaiseOnDisconnect();
}
}
public StreamTransferInfo GetFileStream(string file)
{
_setStatus(true);
// download files
return(new StreamTransferInfo
{
FileName = file,
FileStream = client.OpenRead(file)
});
}
public async Task TransferProductsData()
{
sftpClient.Connect();
using (var sftpStream = sftpClient.OpenRead("/root/data/products.csv"))
{
var products = sftpStream.ReadCsv <ProductItem>();
await awsS3API.WriteParquet(products, "products.parquet");
}
sftpClient.Disconnect();
}
public async Task <string> DownloadFileToBlobAsync(string remoteFilePath, string localFilePath, BlobContainerClient blobContainerClient)
{
try
{
BlobClient blobClient = blobContainerClient.GetBlobClient(localFilePath);
Connect();
var blobInfo = await blobClient.UploadAsync(_sftpClient.OpenRead(remoteFilePath)); //stream file to Azure Blob
var download = await blobClient.DownloadAsync(); //Download blob
string text;
using (StreamReader streamReader = new StreamReader(download.Value.Content))
{
text = streamReader.ReadToEnd(); //get file content
}
_logger.LogInformation($"Finished transferring file [{localFilePath}] from [{remoteFilePath}]");
return(text);
}
catch (Exception exception)
{
_logger.LogError(exception, $"Failed in transferring file [{localFilePath}] from [{remoteFilePath}]");
}
return(null);
}
public StatusCode Download(Models.FileTransportInfo transportInformation, out System.IO.Stream st)
{
st = null;
StatusCode retVal = init();
if (retVal != StatusCode.SUCCEED_STATUS)
{
return(retVal);
}
try
{
streamingClient = new SftpClient(connectionInformation);
streamingClient.Connect();
st = streamingClient.OpenRead(transportInformation.SourceFullName);
}
catch (Renci.SshNet.Common.SftpPathNotFoundException spnex)
{
return(new Common.CommonStatusCode(Common.CommonStatusCode.SFTP_FILE_NOT_FOUND, new object[] { connectionInformation.Host, transportInformation.SourceFolderName, transportInformation.SourceFileName }, spnex, Config, ApplicationID));
}
catch (Exception ex)
{
return(new Common.CommonStatusCode(Common.CommonStatusCode.SFTP_DOWNLOAD_ERROR, new object[] { connectionInformation.Host, transportInformation.SourceFolderName, transportInformation.SourceFileName }, ex, Config, ApplicationID));
}
finally
{
}
return(retVal);
}
public Task <Stream> OpenReadStreamAsync(CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
ThrowIfDisposed();
ConnectIfDisconnected();
return(Task.FromResult <Stream>(_client.OpenRead(_file.FullName)));
}
public async Task <Stream> OpenRead(string fileName, DirectoryEnum directoryEnum = DirectoryEnum.Published)
{
if (await Exists(fileName, directoryEnum))
{
return(_sftpClient.OpenRead(GetFilePath(fileName, directoryEnum)) as Stream);
}
return(null);
}
public async Task <ActionResult> LoadFTPFiles(FTPCredentials credentials, Guid requestId, IEnumerable <string> paths)
{
List <Document> uploadedDocuments = new List <Document>();
using (var sftp = new SftpClient(credentials.Address, credentials.Port, credentials.Login, credentials.Password))
{
try
{
sftp.Connect();
foreach (var path in paths)
{
var fileInfo = sftp.Get(path);
using (var sSource = sftp.OpenRead(path))
{
using (var db = new DataContext())
{
var document = new Document
{
CreatedOn = DateTime.UtcNow,
FileName = fileInfo.Name,
ItemID = requestId,
Length = fileInfo.Length,
MimeType = FileEx.GetMimeTypeByExtension(fileInfo.Name),
Name = fileInfo.Name,
Viewable = false,
Kind = DocumentKind.User
};
db.Documents.Add(document);
uploadedDocuments.Add(document);
await db.SaveChangesAsync();
var docStream = new DocumentStream(db, document.ID);
await sSource.CopyToAsync(docStream);
}
}
}
}
catch (Exception ex)
{
return(new HttpStatusCodeResult((int)HttpStatusCode.BadRequest, ex.Message));
}
finally
{
if (sftp.IsConnected)
{
sftp.Disconnect();
}
}
}
Response.StatusCode = (int)HttpStatusCode.OK;
Response.ContentType = "application/json";
return(Json(uploadedDocuments.Select(d => new { d.ID, d.FileName, d.MimeType, Size = d.Length }), JsonRequestBehavior.AllowGet));
}
public override async Task <byte[]> ReadFileAsync(string path, CancellationToken cancellationToken = default)
{
await GetFileAsync(path, cancellationToken);
try
{
using var fileStream = client.OpenRead(PrependRootPath(path));
var fileContents = new byte[fileStream.Length];
await fileStream.ReadAsync(fileContents, 0, (int)fileStream.Length, cancellationToken);
return(fileContents);
}
catch (Exception exception)
{
throw Exception(exception);
}
}
public async Task <byte[]> ReadFile(SftpClient sftp, string path, CancellationToken ct)
{
logger.LogDebug("Opening {path} for reading", path);
await using var readFs = sftp.OpenRead(path);
await using var ms = new MemoryStream();
await readFs.CopyToAsync(ms, ct);
return(ms.ToArray());
}
public override void OpenFileForReading(string name)
{
if (fileStream != null)
{
throw new Exception("Can't happen");
}
fileName = FullPathWithName(name);
fileStream = sftpClient.OpenRead(WireEncodedString(fileName));
fileLastWriteTimeUtc = DateTime.MinValue;
}
public void DownloadToLocalFile(string filePath)
{
var connectionInfo = _connectionInfo.CreateConnectionInfo();
using (var client = new SftpClient(connectionInfo))
{
client.Connect();
using (FileStream fs = File.OpenWrite(filePath))
client.OpenRead(Path.Combine(_path, Name)).CopyTo(fs);
}
}
/// <summary>
/// 读取SSH服务器Server配置文件
/// </summary>
/// <param name="path">Server配置文件路径</param>
public void ReadFromSSH(string path, SftpClient client)
{
MemoryStream stream = new MemoryStream();
client.OpenRead(path).CopyTo(stream);
stream.Seek(0, SeekOrigin.Begin);
m_Setting = new IniHelper(stream, false);
try { SettingToFields(); }
catch (Exception) { throw; }
}
public static SshWebResponse GetDownloadResponse(SftpClient client, string path)
{
var response = new SshWebResponse();
if (client.Exists(path))
{
response.stream = client.OpenRead(path);
return(response);
}
path = path.TrimStart('/');
if (client.Exists(path))
{
response.stream = client.OpenRead(path);
return(response);
}
return(response);
}
public async Task GetDataFromResponseAndWriteLocalDisk(string sourceUrl, SftpClient sftp, string protocol)
{
long totalSize = 0;
DateTime downloadStarted = DateTime.Now;
DateTime downloadEnded = DateTime.Now;
double elapsedTime = 0;
double downloadSpeed = 0;
int PercentProgress = 0;
try
{
string fileName = GetFileNameFromUrl(sourceUrl);
string path = _appSettings.DownloadedFileLocation + fileName;
if (!Directory.Exists(_appSettings.DownloadedFileLocation))
{
Directory.CreateDirectory(_appSettings.DownloadedFileLocation);
}
if (File.Exists(path))
{
fileName = Guid.NewGuid().ToString() + fileName;
}
FileStream fileStream = File.Create(_appSettings.DownloadedFileLocation + fileName);
int bufferSize = _appSettings.BufferSize;
if (bufferSize > int.MaxValue)
{
bufferSize = int.MaxValue;
}
byte[] buffer = new byte[bufferSize];
int bytesRead;
var sftpFileStream = sftp.OpenRead(path);
totalSize = sftpFileStream.Length;
downloadStarted = DateTime.Now;
while ((bytesRead = await sftpFileStream.ReadAsync(buffer, 0, bufferSize)) != 0)
{
fileStream.Write(buffer, 0, bytesRead);
PercentProgress = Convert.ToInt32((fileStream.Length * 100) / sftpFileStream.Length);
}
downloadEnded = DateTime.Now;
elapsedTime = (downloadEnded - downloadStarted).TotalSeconds; // Total time needed in seconds
downloadSpeed = Utiles.ConvertBytesToMegabytes(totalSize) / elapsedTime; // Calculation for Mbps
}
catch (Exception ex)
{
throw ex;
}
finally
{
var entity = DomainObjectToEntity(sourceUrl, protocol, totalSize, _appSettings.DownloadedFileLocation, downloadStarted, downloadEnded, elapsedTime, downloadSpeed, PercentProgress);
_repository.Add(entity);
_unitOfWork.Commit();
}
}
internal Stream OpenRead(string path)
{
if (_sftpClient is NoAuthenticationSftpClient noAuthenticationSftpClient)
{
MemoryStream ms = new MemoryStream();
noAuthenticationSftpClient.RunCommand(new DownloadFile(path, ms));
return(ms);
}
else
{
return(_sftpClient.OpenRead(path));
}
}
/// <summary>
/// SFTP读取文件
/// </summary>
/// <param name="remotePath">远程路径</param>
/// <param name="localPath">本地路径</param>
public SftpFileStream ReadFile(string remotePath)
{
try
{
Connect();
var byt = sftp.OpenRead(remotePath);
return(byt);
}
catch (Exception ex)
{
//TxtLog.WriteTxt(CommonMethod.GetProgramName(), string.Format("SFTP文件获取失败，原因：{0}", ex.Message));
throw new Exception(string.Format("SFTP文件获取失败，原因：{0}", ex.Message));
}
}
/// <summary>
/// Open SFTP connection to server and download specified file to string
/// </summary>
/// <param name="filePath">Filename to download</param>
/// <returns>File contents string</returns>
internal async Task <string> ReadRemoteFile(string filePath, bool refreshHttp = false, bool invokeEmHttp = false)
{
string output = null;
SftpFileStream remoteFileStream = null;
// Check if we need to update the http page before getting our file
if (refreshHttp)
{
await RefreshHttpPage($"http://{Host}", "/Docker");
}
// Check if we need to invoke the emhttpd script before getting our file
if (invokeEmHttp)
{
await InvokeEmHttp();
}
await Task.Run(async() =>
{
try
{
if (!_sftpClient.IsConnected)
{
_sftpClient.Connect();
}
remoteFileStream = _sftpClient.OpenRead(filePath);
}
catch (Exception ex)
{
Debug.WriteLine($"[ERROR] [UnraidApiClient.ReadRemoteFile(string {filePath})] EXCEPTION: {ex.Message}");
}
finally
{
if (remoteFileStream != null)
{
output = await new StreamReader(remoteFileStream).ReadToEndAsync();
}
}
});
//Debug.WriteLine($"[DEBUG] [UnraidApiClient.ReadRemoteFile(string {filePath})] RETURNED: {output}");
return(output);
}
public SftpFileStream GetRemoteFile(string filename)
{
// Server credentials
var host = "host";
var port = 22;
var username = "******";
var password = "******";
sftp = new SftpClient(host, port, username, password);
// Connect to the SFTP server
sftp.Connect();
// Read the file in question
file = sftp.OpenRead(filename);
return(file);
}
/// <inheritdoc/>
/// <remarks>SFTP offers something called symbolic links, but they are not always enabled and are not actually a real copy.</remarks>
public void CopyFile(string fileName, string newFileName)
{
HandleExceptions(() =>
{
if (_sftpClient.Exists(newFileName))
{
throw new RepositoryConfigurationException(ConfigurationExceptionCategory.FileExists, this, String.Format(RepositoryExceptionMessage.FileAlreadyExists_1, newFileName));
}
using (Stream inputFileStream = _sftpClient.OpenRead(fileName))
using (Stream outputFileStream = _sftpClient.Open(newFileName, FileMode.CreateNew, FileAccess.Write))
{
inputFileStream.CopyTo(outputFileStream);
}
_sftpClient.RenameFile(fileName, newFileName);
}, fileName);
}
public HResult GetFileDataCallback(int commandId, string relativePath, ulong byteOffset, uint length, Guid dataStreamId, byte[] contentId, byte[] providerId, uint triggeringProcessId, string triggeringProcessImageFileName)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return(HResult.Ok);
}
else
{
var fullPath = GetFullPath(relativePath);
if (sftpClient.Exists(fullPath))
{
var file = sftpClient.Get(fullPath);
uint desiredBufferSize = (uint)Math.Min(64 * 1024, file.Length);
using (var reader = sftpClient.OpenRead(fullPath)) {
using (var writeBuffer = virtualization.CreateWriteBuffer(byteOffset, desiredBufferSize, out var alignedWriteOffset, out var alignedBufferSize)) {
while (alignedWriteOffset < (ulong)file.Length)
{
writeBuffer.Stream.Seek(0, SeekOrigin.Begin);
var currentBufferSize = Math.Min(desiredBufferSize, (ulong)file.Length - alignedWriteOffset);
var buffer = new byte[currentBufferSize];
reader.Read(buffer, 0, (int)currentBufferSize);
writeBuffer.Stream.Write(buffer, 0, (int)currentBufferSize);
var hr = virtualization.WriteFileData(dataStreamId, writeBuffer, alignedWriteOffset, (uint)currentBufferSize);
if (hr != HResult.Ok)
{
return(HResult.InternalError);
}
alignedWriteOffset += (ulong)currentBufferSize;
}
}
}
return(HResult.Ok);
}
else
{
return(HResult.FileNotFound);
}
}
}
public SFTPStream(Uri uri, ConnectionInfo connectionInfo, StreamMode streamMode)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (connectionInfo == null)
{
throw new ArgumentNullException(nameof(connectionInfo));
}
_uri = uri;
_client = new SftpClient(connectionInfo);
_client.Connect();
String adjustedLocalPath = uri.LocalPath.Substring(1);
_stream = streamMode == StreamMode.Read ? _client.OpenRead(adjustedLocalPath) : _client.Create(adjustedLocalPath);
}
public void DownloadFile(string remotePath, string localPath, AsyncCallback callback)
{
SftpFileStream inSt = null;
FileStream outSt = null;
try
{
SftpFile file = sftp.Get(remotePath);
long fileSize = file.Length;
inSt = sftp.OpenRead(remotePath);
outSt = new FileStream(localPath, FileMode.OpenOrCreate, FileAccess.ReadWrite);
byte[] buf = new byte[8092];
int len = 0;
long count = 0;
while ((len = inSt.Read(buf, 0, 8092)) > 0)
{
outSt.Write(buf, 0, len);
count += len;
if (callback != null) // 下载进度回调通知
{
DownloadAsyncResult result = new DownloadAsyncResult(count, fileSize);
callback.Invoke(result);
}
}
}
finally
{
if (inSt != null)
{
inSt.Close();
}
if (outSt != null)
{
outSt.Close();
}
}
}
public void OpenReadTest()
{
ConnectionInfo connectionInfo = null; // TODO: Initialize to an appropriate value
SftpClient target = new SftpClient(connectionInfo); // TODO: Initialize to an appropriate value
string path = string.Empty; // TODO: Initialize to an appropriate value
SftpFileStream expected = null; // TODO: Initialize to an appropriate value
SftpFileStream actual;
actual = target.OpenRead(path);
Assert.AreEqual(expected, actual);
Assert.Inconclusive("Verify the correctness of this test method.");
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > myFunc, Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
var connectionInfo = await SetUpSftpAsync(config, myFunc);
_sftpClient = new SftpClient(connectionInfo);
using (_sftpClient)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpConnectPolicy).Execute(() => _sftpClient.Connect());
var listOfFiles = _sftpClient.ListDirectory(config.Ftp.Path).Where(f => !f.IsDirectory)
.Select(f => f)
.ToList();
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath))
{
var folderExisted = _sftpClient.Exists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_sftpClient.CreateDirectory(config.Ftp.ArchivedPath);
}
}
var resolvedStorageAccountKey =
await myFunc.Invoke(config.AzureBlobStorage.StorageKeyVault);
foreach (var item in listOfFiles)
{
bool copied = false;
bool isArchived = false;
try
{
var fileName = item.Name;
using (var istream = _sftpClient.OpenRead(item.FullName))
{
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath))
{
var fileNameSplit = item.Name.Split('.');
var newFileName =
fileNameSplit[0] + "_" + DateTime.UtcNow.ToString("MMddyyyy_HHmmss") + "." +
fileNameSplit[1];
var path = config.Ftp.ArchivedPath + "/" + newFileName;
item.MoveTo(path);
isArchived = true;
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = fileName,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
response.FileData.Add(fres);
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"{nameof(ActorSftpClient)} failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
}
}
}
await Stop();
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
_logger.LogError(e, $"{nameof(ActorSftpClient)} failed to start creating the client of {config.Ftp.Host} : {e.Message}");
}
return(response);
}
public async Task <ActionResult> LoadFTPFiles(FTPCredentials credentials, Guid requestID, Guid?taskID, string authToken, string comments, IEnumerable <string> paths, DTO.Enums.TaskItemTypes?taskItemType)
{
List <Lpp.Dns.DTO.ExtendedDocumentDTO> documents = new List <Lpp.Dns.DTO.ExtendedDocumentDTO>();
using (var web = new System.Net.Http.HttpClient())
using (var sftp = new SftpClient(credentials.Address, credentials.Port, credentials.Login, credentials.Password))
{
web.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", authToken);
web.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("multipart/form-data"));
try
{
sftp.Connect();
HttpResponseMessage response = new HttpResponseMessage();
foreach (var p in paths)
{
var path = p;
var fileInfo = sftp.Get(path);
using (MultipartFormDataContent container = new MultipartFormDataContent())
using (var ftpSource = sftp.OpenRead(path))
{
string filename = System.IO.Path.GetFileName(fileInfo.Name);
container.Add(new StreamContent(ftpSource), "files", filename);
container.Add(new StringContent(System.IO.Path.GetFileNameWithoutExtension(filename)), "documentName");
container.Add(new StringContent(requestID.ToString()), "requestID");
if (taskID.HasValue)
{
container.Add(new StringContent(taskID.Value.ToString()), "taskID");
if (taskItemType.HasValue)
{
container.Add(new StringContent(taskItemType.Value.ToString("D")), "taskItemType");
}
}
if (!string.IsNullOrWhiteSpace(comments))
{
container.Add(new StringContent(comments), "comments");
}
response = await web.PostAsync(WebConfigurationManager.AppSettings["ServiceUrl"] + "/documents/upload", container);
string body = await response.Content.ReadAsStringAsync();
if (response.IsSuccessStatusCode)
{
Lpp.Utilities.BaseResponse <DTO.ExtendedDocumentDTO> savedDocument = Newtonsoft.Json.JsonConvert.DeserializeObject <Lpp.Utilities.BaseResponse <DTO.ExtendedDocumentDTO> >(body);
if (savedDocument.results != null && savedDocument.results.Length > 0)
{
documents.AddRange(savedDocument.results);
}
}
else
{
Response.StatusCode = (int)response.StatusCode;
return(Json(new { success = false, content = body }, "text/plain"));
}
}
}
}
catch (Exception ex)
{
Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest;
return(Json(new { success = false, content = ex.Message }, "text/plain"));
}
finally
{
if (sftp.IsConnected)
{
sftp.Disconnect();
}
}
}
return(Json(new { success = true, content = Newtonsoft.Json.JsonConvert.SerializeObject(documents) }, "text/plain"));
}
public static async Task <SftpFileStream> OpenReadAsync(this SftpClient client, string path)
{
return(await Task.Factory.StartNew(() => client.OpenRead(path)));
}
public SftpFileStream OpenRead(string path)
{
return(_client.OpenRead(path));
}
public static async Task <EtlSettings> ReadEtlSampleData(this EtlSettings etlSettings, int lines = 20)
{
etlSettings.Sample = new DataSample();
switch (etlSettings.SourceType)
{
case EtlSourceEnum.SFTP:
{
var sftp = etlSettings.SFTPSource;
var nameRegex = new Regex(sftp.PathRegex);
using (var sftpClient = new SftpClient(sftp.Host, sftp.Username, sftp.Password))
{
sftpClient.Connect();
var files = sftpClient.ListDirectory(sftp.BasePath);
files = files.Where(f => nameRegex.IsMatch(f.FullName)).ToList();
var first = files.FirstOrDefault();
if (first != null)
{
switch (etlSettings.FileType)
{
case EtlFileType.CSV:
{
using (var sftpStream = sftpClient.OpenRead(first.FullName))
{
etlSettings.ReadFromCSVFile(sftpStream, lines);
}
}
break;
}
}
sftpClient.Disconnect();
}
}
break;
case EtlSourceEnum.S3BucketCheck:
{
var s3 = etlSettings.S3CheckSource;
var awsS3API = new AWSS3API(new AWSS3Options()
{
Key = s3.Key,
Secret = s3.Secret,
Bucket = s3.BucketName,
Region = s3.Region,
});
var objects = await awsS3API.ListAllObjectsInBucket(s3.BucketName, s3.Prefix);
var nameRegex = new Regex(s3.PathRegex);
objects = objects.Where(f => nameRegex.IsMatch(f.Key)).ToList();
var first = objects.FirstOrDefault();
if (first != null)
{
switch (etlSettings.FileType)
{
case EtlFileType.CSV:
{
using (var s3Stream = await awsS3API.OpenReadAsync(first.Key, first.BucketName))
{
etlSettings.ReadFromCSVFile(s3Stream, lines);
}
}
break;
}
}
}
break;
case EtlSourceEnum.S3BucketEvent:
{
var s3 = etlSettings.S3EventSource;
var awsS3API = new AWSS3API(new AWSS3Options()
{
Key = s3.Key,
Secret = s3.Secret,
Bucket = s3.BucketName,
Region = s3.Region,
});
if (await awsS3API.FileExists(s3.ExamplePath, s3.BucketName))
{
switch (etlSettings.FileType)
{
case EtlFileType.CSV:
{
using (var s3Stream = await awsS3API.OpenReadAsync(s3.ExamplePath, s3.BucketName))
{
etlSettings.ReadFromCSVFile(s3Stream, lines);
}
}
break;
}
}
}
break;
case EtlSourceEnum.GoogleAnalytics:
{
await etlSettings.GetBigQueryResultSampleByDate(lines);
}
break;
case EtlSourceEnum.AmazonAthena:
{
await etlSettings.GetAthenaQueryResultSampleByDate(lines);
}
break;
case EtlSourceEnum.AmazonAthenaPipes:
{
// need to compile the query
await etlSettings.ParseAthenaQueryPipes();
}
break;
}
// make the sample data smaller
foreach (var row in etlSettings.Sample.Rows.ToList())
{
row.Items = row.Items.Select(item => item.Length < 100 ? item : item.Substring(0, 50) + "..." + item.Substring(item.Length - 50)).ToList();
}
return(etlSettings);
}
public static async Task <List <string> > TransferData(this EtlSettings etlSettings, AWSAthenaAPI awsAthenaAPI, GenericLogger logger = null, DateTime?useDate = null)
{
var result = new List <string>();
logger?.Log?.Invoke($"ETL Mode: {etlSettings.SourceType}");
switch (etlSettings.SourceType)
{
case EtlSourceEnum.SFTP:
{
var sftp = etlSettings.SFTPSource;
var nameRegex = new Regex(sftp.PathRegex);
var dateRegex = new Regex(sftp.DateKeyRegex);
using (var sftpClient = new SftpClient(sftp.Host, sftp.Username, sftp.Password))
{
sftpClient.Connect();
var files = sftpClient.ListDirectory(sftp.BasePath);
files = files
.Where(f => nameRegex.IsMatch(f.FullName) && dateRegex.IsMatch(f.Name))
.OrderByDescending(f => f.Name)
.ToList();
// find in the target to work out if there is the corresponding parquet file
var targetS3 = etlSettings.CreateTargetS3API();
SftpFile first = null;
foreach (var file in files)
{
Console.WriteLine($"Check File: {file.FullName}");
var s3Key = etlSettings.TargetFlagFile(file.Name);
if (!await targetS3.FileExists(s3Key))
{
first = file;
break;
}
}
// transfer that file
if (first != null)
{
Console.WriteLine($"Transfer File: {first.FullName}");
var dateKey = first.Name.MakeRegexExtraction(dateRegex);
using (var sftpStream = sftpClient.OpenRead(first.FullName))
{
result = await etlSettings.TransferCsvStream(awsAthenaAPI, sftpStream, dateKey, first.Name, false);
}
}
sftpClient.Disconnect();
}
}
break;
case EtlSourceEnum.S3BucketCheck:
{
}
break;
case EtlSourceEnum.S3BucketEvent:
{
var sourceAwsS3Api = new AWSS3API(new AWSS3Options()
{
Key = etlSettings.S3EventSource.Key,
Secret = etlSettings.S3EventSource.Secret,
Bucket = etlSettings.S3EventSource.BucketName,
Region = etlSettings.S3EventSource.Region
});
var s3Event = etlSettings.S3EventSource;
var nameRegex = new Regex(s3Event.PathRegex);
var keyRegex = new Regex(s3Event.FileNameRegex);
// do nothing if it does not match the path pattern
if (!nameRegex.IsMatch(s3Event.ExamplePath) || (!keyRegex.IsMatch(s3Event.ExamplePath)))
{
return(result);
}
// generate dateKey
var dateKey = DateTime.UtcNow.ToString("yyyyMMdd");
Regex dateRegex = null;
if (!s3Event.UseEventDateAsDateKey)
{
dateRegex = new Regex(s3Event.DateKeyRegex);
if (!dateRegex.IsMatch(s3Event.ExamplePath))
{
return(result);
}
dateKey = s3Event.ExamplePath.MakeRegexExtraction(dateRegex);
}
// generate file name
var filename = s3Event.ExamplePath.MakeRegexExtraction(keyRegex);
// it will overwrite by default we need to workout datekey first of all
var prefixUpToDate = etlSettings.MakeTargetS3Prefix(dateKey, filename, true);
// check files that should be deleted
var targetAwsS3Api = etlSettings.CreateTargetS3API();
var oldObjects = await targetAwsS3Api.ListAllObjectsInBucket(prefix : prefixUpToDate);
// delete the files with those prefix
foreach (var oldObj in oldObjects)
{
await targetAwsS3Api.Delete(oldObj.Key);
}
// open file stream and transfer data
using (var awsS3Stream = await sourceAwsS3Api.OpenReadAsync(s3Event.ExamplePath))
{
result = await etlSettings.TransferCsvStream(awsAthenaAPI, awsS3Stream, dateKey, filename, true);
}
}
break;
case EtlSourceEnum.GoogleAnalytics:
{
result = await etlSettings.TransferBigQueryResultByDate(awsAthenaAPI, useDate);
}
break;
case EtlSourceEnum.AmazonAthena:
{
result = await etlSettings.TransferAthenaQueryResultByDate(awsAthenaAPI, useDate);
}
break;
case EtlSourceEnum.AmazonAthenaPipes:
{
await etlSettings.RunAthenaQueryPipes(useDate);
}
break;
}
return(result);
}
private bool RunSFTPFileTransfer()
{
try
{
LoggerSFTPService.FileCreation("(2)in RunSFTPFileTransfer()", logPath);
string Host = ConfigurationManager.AppSettings["SFTPHost"];
int Port = Convert.ToInt32(ConfigurationManager.AppSettings["SFTPPort"]);
string Username = ConfigurationManager.AppSettings["SFTPUsername"];
string Password = ConfigurationManager.AppSettings["SFTPPassword"];
string remoteDirectory = ConfigurationManager.AppSettings["remoteDirectory"];
string localDirectory = ConfigurationManager.AppSettings["LocalDirectory"];
string fileNameStartWith = ConfigurationManager.AppSettings["remoteDrectryFileNameStrat"];
int fileCreateDay = Convert.ToInt32(ConfigurationManager.AppSettings["fileCreateDay"]);
var EmailAddressListString = ConfigurationManager.AppSettings["RecipientsEmailAddressList"];
string[] EmailArray = EmailAddressListString.Split(',');
var EmailSubject = ConfigurationManager.AppSettings["EmailSubject"];
var EmailBody = ConfigurationManager.AppSettings["EmailBody"];
LoggerSFTPService.FileCreation("(2)in making EmailList", logPath);
using (var sftp = new SftpClient(Host, Port, Username, Password))
{
LoggerSFTPService.FileCreation("(3)in sftp before client create", logPath);
sftp.Connect();
LoggerSFTPService.FileCreation("(4)in sftp client create success", logPath);
var files = sftp.ListDirectory(remoteDirectory);
LoggerSFTPService.FileCreation("(5) files list from remote by SSH.Number of file:" + files.Count() + "", logPath);
foreach (var file in files)
{
string remoteFileName = file.Name;
var dd = file.LastWriteTime.Date;
// if ((!file.Name.StartsWith(".")) && ((file.LastWriteTime.Date == DateTime.Today))
if (file.Name.StartsWith(fileNameStartWith) && file.LastWriteTime.Date == DateTime.Today.AddDays(fileCreateDay))//MobileRechargeErrorCodeDetails
{
using (Stream file1 = File.OpenWrite(localDirectory + remoteFileName))
{
LoggerSFTPService.FileCreation("(6) file name:" + remoteFileName + " to send email", logPath);
sftp.DownloadFile(remoteDirectory + remoteFileName, file1);
LoggerSFTPService.FileCreation("(7) file download in desire folder: " + localDirectory + remoteFileName + "", logPath);
if (EmailArray.Count() > 0)
{
foreach (var email in EmailArray)
{
LoggerSFTPService.FileCreation("(8) in loop of sending email ", logPath);
var strimFile = sftp.OpenRead(remoteDirectory + remoteFileName);
LoggerSFTPService.FileCreation("(9) in loop of strim strem file ", logPath);
SendEmailT(EmailSubject, EmailBody, email, strimFile);
}
}
}
}
}
sftp.Disconnect();
LoggerSFTPService.FileCreation("sftp Disconnect successfully", logPath);
}
}
catch (Exception ex)
{
LoggerSFTPService.FileCreation("sftp Exception:" + ex.Message.ToString() + "", logPath);
Console.WriteLine(ex.Message.ToString());
}
return(true);
}
