/// <summary> /// Used for receiving files /// </summary> /// <param name="fromFilePath"></param> /// <returns></returns> public Stream Get(string fromFilePath) { try { try { Connect(); return(_sftp.OpenRead(fromFilePath)); } catch { ReConnect(); return(_sftp.OpenRead(fromFilePath)); } } catch (Exception ex) { throw ExceptionHandling.HandleComponentException( EventLogEventIDs.UnableToGetFile, System.Reflection.MethodBase.GetCurrentMethod(), new SftpException("Unable to get file " + fromFilePath, ex)); } finally { RaiseOnDisconnect(); } }
public StreamTransferInfo GetFileStream(string file) { _setStatus(true); // download files return(new StreamTransferInfo { FileName = file, FileStream = client.OpenRead(file) }); }
public async Task TransferProductsData() { sftpClient.Connect(); using (var sftpStream = sftpClient.OpenRead("/root/data/products.csv")) { var products = sftpStream.ReadCsv <ProductItem>(); await awsS3API.WriteParquet(products, "products.parquet"); } sftpClient.Disconnect(); }
public async Task <string> DownloadFileToBlobAsync(string remoteFilePath, string localFilePath, BlobContainerClient blobContainerClient) { try { BlobClient blobClient = blobContainerClient.GetBlobClient(localFilePath); Connect(); var blobInfo = await blobClient.UploadAsync(_sftpClient.OpenRead(remoteFilePath)); //stream file to Azure Blob var download = await blobClient.DownloadAsync(); //Download blob string text; using (StreamReader streamReader = new StreamReader(download.Value.Content)) { text = streamReader.ReadToEnd(); //get file content } _logger.LogInformation($"Finished transferring file [{localFilePath}] from [{remoteFilePath}]"); return(text); } catch (Exception exception) { _logger.LogError(exception, $"Failed in transferring file [{localFilePath}] from [{remoteFilePath}]"); } return(null); }
public StatusCode Download(Models.FileTransportInfo transportInformation, out System.IO.Stream st) { st = null; StatusCode retVal = init(); if (retVal != StatusCode.SUCCEED_STATUS) { return(retVal); } try { streamingClient = new SftpClient(connectionInformation); streamingClient.Connect(); st = streamingClient.OpenRead(transportInformation.SourceFullName); } catch (Renci.SshNet.Common.SftpPathNotFoundException spnex) { return(new Common.CommonStatusCode(Common.CommonStatusCode.SFTP_FILE_NOT_FOUND, new object[] { connectionInformation.Host, transportInformation.SourceFolderName, transportInformation.SourceFileName }, spnex, Config, ApplicationID)); } catch (Exception ex) { return(new Common.CommonStatusCode(Common.CommonStatusCode.SFTP_DOWNLOAD_ERROR, new object[] { connectionInformation.Host, transportInformation.SourceFolderName, transportInformation.SourceFileName }, ex, Config, ApplicationID)); } finally { } return(retVal); }
public Task <Stream> OpenReadStreamAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); ConnectIfDisconnected(); return(Task.FromResult <Stream>(_client.OpenRead(_file.FullName))); }
public async Task <Stream> OpenRead(string fileName, DirectoryEnum directoryEnum = DirectoryEnum.Published) { if (await Exists(fileName, directoryEnum)) { return(_sftpClient.OpenRead(GetFilePath(fileName, directoryEnum)) as Stream); } return(null); }
public async Task <ActionResult> LoadFTPFiles(FTPCredentials credentials, Guid requestId, IEnumerable <string> paths) { List <Document> uploadedDocuments = new List <Document>(); using (var sftp = new SftpClient(credentials.Address, credentials.Port, credentials.Login, credentials.Password)) { try { sftp.Connect(); foreach (var path in paths) { var fileInfo = sftp.Get(path); using (var sSource = sftp.OpenRead(path)) { using (var db = new DataContext()) { var document = new Document { CreatedOn = DateTime.UtcNow, FileName = fileInfo.Name, ItemID = requestId, Length = fileInfo.Length, MimeType = FileEx.GetMimeTypeByExtension(fileInfo.Name), Name = fileInfo.Name, Viewable = false, Kind = DocumentKind.User }; db.Documents.Add(document); uploadedDocuments.Add(document); await db.SaveChangesAsync(); var docStream = new DocumentStream(db, document.ID); await sSource.CopyToAsync(docStream); } } } } catch (Exception ex) { return(new HttpStatusCodeResult((int)HttpStatusCode.BadRequest, ex.Message)); } finally { if (sftp.IsConnected) { sftp.Disconnect(); } } } Response.StatusCode = (int)HttpStatusCode.OK; Response.ContentType = "application/json"; return(Json(uploadedDocuments.Select(d => new { d.ID, d.FileName, d.MimeType, Size = d.Length }), JsonRequestBehavior.AllowGet)); }
public override async Task <byte[]> ReadFileAsync(string path, CancellationToken cancellationToken = default) { await GetFileAsync(path, cancellationToken); try { using var fileStream = client.OpenRead(PrependRootPath(path)); var fileContents = new byte[fileStream.Length]; await fileStream.ReadAsync(fileContents, 0, (int)fileStream.Length, cancellationToken); return(fileContents); } catch (Exception exception) { throw Exception(exception); } }
public async Task <byte[]> ReadFile(SftpClient sftp, string path, CancellationToken ct) { logger.LogDebug("Opening {path} for reading", path); await using var readFs = sftp.OpenRead(path); await using var ms = new MemoryStream(); await readFs.CopyToAsync(ms, ct); return(ms.ToArray()); }
public override void OpenFileForReading(string name) { if (fileStream != null) { throw new Exception("Can't happen"); } fileName = FullPathWithName(name); fileStream = sftpClient.OpenRead(WireEncodedString(fileName)); fileLastWriteTimeUtc = DateTime.MinValue; }
public void DownloadToLocalFile(string filePath) { var connectionInfo = _connectionInfo.CreateConnectionInfo(); using (var client = new SftpClient(connectionInfo)) { client.Connect(); using (FileStream fs = File.OpenWrite(filePath)) client.OpenRead(Path.Combine(_path, Name)).CopyTo(fs); } }
/// <summary> /// 读取SSH服务器Server配置文件 /// </summary> /// <param name="path">Server配置文件路径</param> public void ReadFromSSH(string path, SftpClient client) { MemoryStream stream = new MemoryStream(); client.OpenRead(path).CopyTo(stream); stream.Seek(0, SeekOrigin.Begin); m_Setting = new IniHelper(stream, false); try { SettingToFields(); } catch (Exception) { throw; } }
public static SshWebResponse GetDownloadResponse(SftpClient client, string path) { var response = new SshWebResponse(); if (client.Exists(path)) { response.stream = client.OpenRead(path); return(response); } path = path.TrimStart('/'); if (client.Exists(path)) { response.stream = client.OpenRead(path); return(response); } return(response); }
public async Task GetDataFromResponseAndWriteLocalDisk(string sourceUrl, SftpClient sftp, string protocol) { long totalSize = 0; DateTime downloadStarted = DateTime.Now; DateTime downloadEnded = DateTime.Now; double elapsedTime = 0; double downloadSpeed = 0; int PercentProgress = 0; try { string fileName = GetFileNameFromUrl(sourceUrl); string path = _appSettings.DownloadedFileLocation + fileName; if (!Directory.Exists(_appSettings.DownloadedFileLocation)) { Directory.CreateDirectory(_appSettings.DownloadedFileLocation); } if (File.Exists(path)) { fileName = Guid.NewGuid().ToString() + fileName; } FileStream fileStream = File.Create(_appSettings.DownloadedFileLocation + fileName); int bufferSize = _appSettings.BufferSize; if (bufferSize > int.MaxValue) { bufferSize = int.MaxValue; } byte[] buffer = new byte[bufferSize]; int bytesRead; var sftpFileStream = sftp.OpenRead(path); totalSize = sftpFileStream.Length; downloadStarted = DateTime.Now; while ((bytesRead = await sftpFileStream.ReadAsync(buffer, 0, bufferSize)) != 0) { fileStream.Write(buffer, 0, bytesRead); PercentProgress = Convert.ToInt32((fileStream.Length * 100) / sftpFileStream.Length); } downloadEnded = DateTime.Now; elapsedTime = (downloadEnded - downloadStarted).TotalSeconds; // Total time needed in seconds downloadSpeed = Utiles.ConvertBytesToMegabytes(totalSize) / elapsedTime; // Calculation for Mbps } catch (Exception ex) { throw ex; } finally { var entity = DomainObjectToEntity(sourceUrl, protocol, totalSize, _appSettings.DownloadedFileLocation, downloadStarted, downloadEnded, elapsedTime, downloadSpeed, PercentProgress); _repository.Add(entity); _unitOfWork.Commit(); } }
internal Stream OpenRead(string path) { if (_sftpClient is NoAuthenticationSftpClient noAuthenticationSftpClient) { MemoryStream ms = new MemoryStream(); noAuthenticationSftpClient.RunCommand(new DownloadFile(path, ms)); return(ms); } else { return(_sftpClient.OpenRead(path)); } }
/// <summary> /// SFTP读取文件 /// </summary> /// <param name="remotePath">远程路径</param> /// <param name="localPath">本地路径</param> public SftpFileStream ReadFile(string remotePath) { try { Connect(); var byt = sftp.OpenRead(remotePath); return(byt); } catch (Exception ex) { //TxtLog.WriteTxt(CommonMethod.GetProgramName(), string.Format("SFTP文件获取失败,原因:{0}", ex.Message)); throw new Exception(string.Format("SFTP文件获取失败,原因:{0}", ex.Message)); } }
/// <summary> /// Open SFTP connection to server and download specified file to string /// </summary> /// <param name="filePath">Filename to download</param> /// <returns>File contents string</returns> internal async Task <string> ReadRemoteFile(string filePath, bool refreshHttp = false, bool invokeEmHttp = false) { string output = null; SftpFileStream remoteFileStream = null; // Check if we need to update the http page before getting our file if (refreshHttp) { await RefreshHttpPage($"http://{Host}", "/Docker"); } // Check if we need to invoke the emhttpd script before getting our file if (invokeEmHttp) { await InvokeEmHttp(); } await Task.Run(async() => { try { if (!_sftpClient.IsConnected) { _sftpClient.Connect(); } remoteFileStream = _sftpClient.OpenRead(filePath); } catch (Exception ex) { Debug.WriteLine($"[ERROR] [UnraidApiClient.ReadRemoteFile(string {filePath})] EXCEPTION: {ex.Message}"); } finally { if (remoteFileStream != null) { output = await new StreamReader(remoteFileStream).ReadToEndAsync(); } } }); //Debug.WriteLine($"[DEBUG] [UnraidApiClient.ReadRemoteFile(string {filePath})] RETURNED: {output}"); return(output); }
public SftpFileStream GetRemoteFile(string filename) { // Server credentials var host = "host"; var port = 22; var username = "******"; var password = "******"; sftp = new SftpClient(host, port, username, password); // Connect to the SFTP server sftp.Connect(); // Read the file in question file = sftp.OpenRead(filename); return(file); }
/// <inheritdoc/> /// <remarks>SFTP offers something called symbolic links, but they are not always enabled and are not actually a real copy.</remarks> public void CopyFile(string fileName, string newFileName) { HandleExceptions(() => { if (_sftpClient.Exists(newFileName)) { throw new RepositoryConfigurationException(ConfigurationExceptionCategory.FileExists, this, String.Format(RepositoryExceptionMessage.FileAlreadyExists_1, newFileName)); } using (Stream inputFileStream = _sftpClient.OpenRead(fileName)) using (Stream outputFileStream = _sftpClient.Open(newFileName, FileMode.CreateNew, FileAccess.Write)) { inputFileStream.CopyTo(outputFileStream); } _sftpClient.RenameFile(fileName, newFileName); }, fileName); }
public HResult GetFileDataCallback(int commandId, string relativePath, ulong byteOffset, uint length, Guid dataStreamId, byte[] contentId, byte[] providerId, uint triggeringProcessId, string triggeringProcessImageFileName) { if (string.IsNullOrWhiteSpace(relativePath)) { return(HResult.Ok); } else { var fullPath = GetFullPath(relativePath); if (sftpClient.Exists(fullPath)) { var file = sftpClient.Get(fullPath); uint desiredBufferSize = (uint)Math.Min(64 * 1024, file.Length); using (var reader = sftpClient.OpenRead(fullPath)) { using (var writeBuffer = virtualization.CreateWriteBuffer(byteOffset, desiredBufferSize, out var alignedWriteOffset, out var alignedBufferSize)) { while (alignedWriteOffset < (ulong)file.Length) { writeBuffer.Stream.Seek(0, SeekOrigin.Begin); var currentBufferSize = Math.Min(desiredBufferSize, (ulong)file.Length - alignedWriteOffset); var buffer = new byte[currentBufferSize]; reader.Read(buffer, 0, (int)currentBufferSize); writeBuffer.Stream.Write(buffer, 0, (int)currentBufferSize); var hr = virtualization.WriteFileData(dataStreamId, writeBuffer, alignedWriteOffset, (uint)currentBufferSize); if (hr != HResult.Ok) { return(HResult.InternalError); } alignedWriteOffset += (ulong)currentBufferSize; } } } return(HResult.Ok); } else { return(HResult.FileNotFound); } } }
public SFTPStream(Uri uri, ConnectionInfo connectionInfo, StreamMode streamMode) { if (uri == null) { throw new ArgumentNullException(nameof(uri)); } if (connectionInfo == null) { throw new ArgumentNullException(nameof(connectionInfo)); } _uri = uri; _client = new SftpClient(connectionInfo); _client.Connect(); String adjustedLocalPath = uri.LocalPath.Substring(1); _stream = streamMode == StreamMode.Read ? _client.OpenRead(adjustedLocalPath) : _client.Create(adjustedLocalPath); }
public void DownloadFile(string remotePath, string localPath, AsyncCallback callback) { SftpFileStream inSt = null; FileStream outSt = null; try { SftpFile file = sftp.Get(remotePath); long fileSize = file.Length; inSt = sftp.OpenRead(remotePath); outSt = new FileStream(localPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); byte[] buf = new byte[8092]; int len = 0; long count = 0; while ((len = inSt.Read(buf, 0, 8092)) > 0) { outSt.Write(buf, 0, len); count += len; if (callback != null) // 下载进度回调通知 { DownloadAsyncResult result = new DownloadAsyncResult(count, fileSize); callback.Invoke(result); } } } finally { if (inSt != null) { inSt.Close(); } if (outSt != null) { outSt.Close(); } } }
public void OpenReadTest() { ConnectionInfo connectionInfo = null; // TODO: Initialize to an appropriate value SftpClient target = new SftpClient(connectionInfo); // TODO: Initialize to an appropriate value string path = string.Empty; // TODO: Initialize to an appropriate value SftpFileStream expected = null; // TODO: Initialize to an appropriate value SftpFileStream actual; actual = target.OpenRead(path); Assert.AreEqual(expected, actual); Assert.Inconclusive("Verify the correctness of this test method."); }
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > myFunc, Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc) { var response = new FtpClientResponse { ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>() }; try { var connectionInfo = await SetUpSftpAsync(config, myFunc); _sftpClient = new SftpClient(connectionInfo); using (_sftpClient) { _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpConnectPolicy).Execute(() => _sftpClient.Connect()); var listOfFiles = _sftpClient.ListDirectory(config.Ftp.Path).Where(f => !f.IsDirectory) .Select(f => f) .ToList(); if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath)) { var folderExisted = _sftpClient.Exists(config.Ftp.ArchivedPath); if (!folderExisted) { _sftpClient.CreateDirectory(config.Ftp.ArchivedPath); } } var resolvedStorageAccountKey = await myFunc.Invoke(config.AzureBlobStorage.StorageKeyVault); foreach (var item in listOfFiles) { bool copied = false; bool isArchived = false; try { var fileName = item.Name; using (var istream = _sftpClient.OpenRead(item.FullName)) { MemoryStream memoryStream = new MemoryStream(); istream.CopyTo(memoryStream); memoryStream.Position = 0; copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName, fileName, config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey); if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath)) { var fileNameSplit = item.Name.Split('.'); var newFileName = fileNameSplit[0] + "_" + DateTime.UtcNow.ToString("MMddyyyy_HHmmss") + "." + fileNameSplit[1]; var path = config.Ftp.ArchivedPath + "/" + newFileName; item.MoveTo(path); isArchived = true; } var fres = new FtpClientResponseFile { Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus, FileName = fileName, ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError) }; response.FileData.Add(fres); } } catch (Exception ex) { _logger.LogError(ex, $"{nameof(ActorSftpClient)} failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}"); } } } await Stop(); } catch (Exception e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; _logger.LogError(e, $"{nameof(ActorSftpClient)} failed to start creating the client of {config.Ftp.Host} : {e.Message}"); } return(response); }
public async Task <ActionResult> LoadFTPFiles(FTPCredentials credentials, Guid requestID, Guid?taskID, string authToken, string comments, IEnumerable <string> paths, DTO.Enums.TaskItemTypes?taskItemType) { List <Lpp.Dns.DTO.ExtendedDocumentDTO> documents = new List <Lpp.Dns.DTO.ExtendedDocumentDTO>(); using (var web = new System.Net.Http.HttpClient()) using (var sftp = new SftpClient(credentials.Address, credentials.Port, credentials.Login, credentials.Password)) { web.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", authToken); web.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("multipart/form-data")); try { sftp.Connect(); HttpResponseMessage response = new HttpResponseMessage(); foreach (var p in paths) { var path = p; var fileInfo = sftp.Get(path); using (MultipartFormDataContent container = new MultipartFormDataContent()) using (var ftpSource = sftp.OpenRead(path)) { string filename = System.IO.Path.GetFileName(fileInfo.Name); container.Add(new StreamContent(ftpSource), "files", filename); container.Add(new StringContent(System.IO.Path.GetFileNameWithoutExtension(filename)), "documentName"); container.Add(new StringContent(requestID.ToString()), "requestID"); if (taskID.HasValue) { container.Add(new StringContent(taskID.Value.ToString()), "taskID"); if (taskItemType.HasValue) { container.Add(new StringContent(taskItemType.Value.ToString("D")), "taskItemType"); } } if (!string.IsNullOrWhiteSpace(comments)) { container.Add(new StringContent(comments), "comments"); } response = await web.PostAsync(WebConfigurationManager.AppSettings["ServiceUrl"] + "/documents/upload", container); string body = await response.Content.ReadAsStringAsync(); if (response.IsSuccessStatusCode) { Lpp.Utilities.BaseResponse <DTO.ExtendedDocumentDTO> savedDocument = Newtonsoft.Json.JsonConvert.DeserializeObject <Lpp.Utilities.BaseResponse <DTO.ExtendedDocumentDTO> >(body); if (savedDocument.results != null && savedDocument.results.Length > 0) { documents.AddRange(savedDocument.results); } } else { Response.StatusCode = (int)response.StatusCode; return(Json(new { success = false, content = body }, "text/plain")); } } } } catch (Exception ex) { Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest; return(Json(new { success = false, content = ex.Message }, "text/plain")); } finally { if (sftp.IsConnected) { sftp.Disconnect(); } } } return(Json(new { success = true, content = Newtonsoft.Json.JsonConvert.SerializeObject(documents) }, "text/plain")); }
public static async Task <SftpFileStream> OpenReadAsync(this SftpClient client, string path) { return(await Task.Factory.StartNew(() => client.OpenRead(path))); }
public SftpFileStream OpenRead(string path) { return(_client.OpenRead(path)); }
public static async Task <EtlSettings> ReadEtlSampleData(this EtlSettings etlSettings, int lines = 20) { etlSettings.Sample = new DataSample(); switch (etlSettings.SourceType) { case EtlSourceEnum.SFTP: { var sftp = etlSettings.SFTPSource; var nameRegex = new Regex(sftp.PathRegex); using (var sftpClient = new SftpClient(sftp.Host, sftp.Username, sftp.Password)) { sftpClient.Connect(); var files = sftpClient.ListDirectory(sftp.BasePath); files = files.Where(f => nameRegex.IsMatch(f.FullName)).ToList(); var first = files.FirstOrDefault(); if (first != null) { switch (etlSettings.FileType) { case EtlFileType.CSV: { using (var sftpStream = sftpClient.OpenRead(first.FullName)) { etlSettings.ReadFromCSVFile(sftpStream, lines); } } break; } } sftpClient.Disconnect(); } } break; case EtlSourceEnum.S3BucketCheck: { var s3 = etlSettings.S3CheckSource; var awsS3API = new AWSS3API(new AWSS3Options() { Key = s3.Key, Secret = s3.Secret, Bucket = s3.BucketName, Region = s3.Region, }); var objects = await awsS3API.ListAllObjectsInBucket(s3.BucketName, s3.Prefix); var nameRegex = new Regex(s3.PathRegex); objects = objects.Where(f => nameRegex.IsMatch(f.Key)).ToList(); var first = objects.FirstOrDefault(); if (first != null) { switch (etlSettings.FileType) { case EtlFileType.CSV: { using (var s3Stream = await awsS3API.OpenReadAsync(first.Key, first.BucketName)) { etlSettings.ReadFromCSVFile(s3Stream, lines); } } break; } } } break; case EtlSourceEnum.S3BucketEvent: { var s3 = etlSettings.S3EventSource; var awsS3API = new AWSS3API(new AWSS3Options() { Key = s3.Key, Secret = s3.Secret, Bucket = s3.BucketName, Region = s3.Region, }); if (await awsS3API.FileExists(s3.ExamplePath, s3.BucketName)) { switch (etlSettings.FileType) { case EtlFileType.CSV: { using (var s3Stream = await awsS3API.OpenReadAsync(s3.ExamplePath, s3.BucketName)) { etlSettings.ReadFromCSVFile(s3Stream, lines); } } break; } } } break; case EtlSourceEnum.GoogleAnalytics: { await etlSettings.GetBigQueryResultSampleByDate(lines); } break; case EtlSourceEnum.AmazonAthena: { await etlSettings.GetAthenaQueryResultSampleByDate(lines); } break; case EtlSourceEnum.AmazonAthenaPipes: { // need to compile the query await etlSettings.ParseAthenaQueryPipes(); } break; } // make the sample data smaller foreach (var row in etlSettings.Sample.Rows.ToList()) { row.Items = row.Items.Select(item => item.Length < 100 ? item : item.Substring(0, 50) + "..." + item.Substring(item.Length - 50)).ToList(); } return(etlSettings); }
public static async Task <List <string> > TransferData(this EtlSettings etlSettings, AWSAthenaAPI awsAthenaAPI, GenericLogger logger = null, DateTime?useDate = null) { var result = new List <string>(); logger?.Log?.Invoke($"ETL Mode: {etlSettings.SourceType}"); switch (etlSettings.SourceType) { case EtlSourceEnum.SFTP: { var sftp = etlSettings.SFTPSource; var nameRegex = new Regex(sftp.PathRegex); var dateRegex = new Regex(sftp.DateKeyRegex); using (var sftpClient = new SftpClient(sftp.Host, sftp.Username, sftp.Password)) { sftpClient.Connect(); var files = sftpClient.ListDirectory(sftp.BasePath); files = files .Where(f => nameRegex.IsMatch(f.FullName) && dateRegex.IsMatch(f.Name)) .OrderByDescending(f => f.Name) .ToList(); // find in the target to work out if there is the corresponding parquet file var targetS3 = etlSettings.CreateTargetS3API(); SftpFile first = null; foreach (var file in files) { Console.WriteLine($"Check File: {file.FullName}"); var s3Key = etlSettings.TargetFlagFile(file.Name); if (!await targetS3.FileExists(s3Key)) { first = file; break; } } // transfer that file if (first != null) { Console.WriteLine($"Transfer File: {first.FullName}"); var dateKey = first.Name.MakeRegexExtraction(dateRegex); using (var sftpStream = sftpClient.OpenRead(first.FullName)) { result = await etlSettings.TransferCsvStream(awsAthenaAPI, sftpStream, dateKey, first.Name, false); } } sftpClient.Disconnect(); } } break; case EtlSourceEnum.S3BucketCheck: { } break; case EtlSourceEnum.S3BucketEvent: { var sourceAwsS3Api = new AWSS3API(new AWSS3Options() { Key = etlSettings.S3EventSource.Key, Secret = etlSettings.S3EventSource.Secret, Bucket = etlSettings.S3EventSource.BucketName, Region = etlSettings.S3EventSource.Region }); var s3Event = etlSettings.S3EventSource; var nameRegex = new Regex(s3Event.PathRegex); var keyRegex = new Regex(s3Event.FileNameRegex); // do nothing if it does not match the path pattern if (!nameRegex.IsMatch(s3Event.ExamplePath) || (!keyRegex.IsMatch(s3Event.ExamplePath))) { return(result); } // generate dateKey var dateKey = DateTime.UtcNow.ToString("yyyyMMdd"); Regex dateRegex = null; if (!s3Event.UseEventDateAsDateKey) { dateRegex = new Regex(s3Event.DateKeyRegex); if (!dateRegex.IsMatch(s3Event.ExamplePath)) { return(result); } dateKey = s3Event.ExamplePath.MakeRegexExtraction(dateRegex); } // generate file name var filename = s3Event.ExamplePath.MakeRegexExtraction(keyRegex); // it will overwrite by default we need to workout datekey first of all var prefixUpToDate = etlSettings.MakeTargetS3Prefix(dateKey, filename, true); // check files that should be deleted var targetAwsS3Api = etlSettings.CreateTargetS3API(); var oldObjects = await targetAwsS3Api.ListAllObjectsInBucket(prefix : prefixUpToDate); // delete the files with those prefix foreach (var oldObj in oldObjects) { await targetAwsS3Api.Delete(oldObj.Key); } // open file stream and transfer data using (var awsS3Stream = await sourceAwsS3Api.OpenReadAsync(s3Event.ExamplePath)) { result = await etlSettings.TransferCsvStream(awsAthenaAPI, awsS3Stream, dateKey, filename, true); } } break; case EtlSourceEnum.GoogleAnalytics: { result = await etlSettings.TransferBigQueryResultByDate(awsAthenaAPI, useDate); } break; case EtlSourceEnum.AmazonAthena: { result = await etlSettings.TransferAthenaQueryResultByDate(awsAthenaAPI, useDate); } break; case EtlSourceEnum.AmazonAthenaPipes: { await etlSettings.RunAthenaQueryPipes(useDate); } break; } return(result); }
private bool RunSFTPFileTransfer() { try { LoggerSFTPService.FileCreation("(2)in RunSFTPFileTransfer()", logPath); string Host = ConfigurationManager.AppSettings["SFTPHost"]; int Port = Convert.ToInt32(ConfigurationManager.AppSettings["SFTPPort"]); string Username = ConfigurationManager.AppSettings["SFTPUsername"]; string Password = ConfigurationManager.AppSettings["SFTPPassword"]; string remoteDirectory = ConfigurationManager.AppSettings["remoteDirectory"]; string localDirectory = ConfigurationManager.AppSettings["LocalDirectory"]; string fileNameStartWith = ConfigurationManager.AppSettings["remoteDrectryFileNameStrat"]; int fileCreateDay = Convert.ToInt32(ConfigurationManager.AppSettings["fileCreateDay"]); var EmailAddressListString = ConfigurationManager.AppSettings["RecipientsEmailAddressList"]; string[] EmailArray = EmailAddressListString.Split(','); var EmailSubject = ConfigurationManager.AppSettings["EmailSubject"]; var EmailBody = ConfigurationManager.AppSettings["EmailBody"]; LoggerSFTPService.FileCreation("(2)in making EmailList", logPath); using (var sftp = new SftpClient(Host, Port, Username, Password)) { LoggerSFTPService.FileCreation("(3)in sftp before client create", logPath); sftp.Connect(); LoggerSFTPService.FileCreation("(4)in sftp client create success", logPath); var files = sftp.ListDirectory(remoteDirectory); LoggerSFTPService.FileCreation("(5) files list from remote by SSH.Number of file:" + files.Count() + "", logPath); foreach (var file in files) { string remoteFileName = file.Name; var dd = file.LastWriteTime.Date; // if ((!file.Name.StartsWith(".")) && ((file.LastWriteTime.Date == DateTime.Today)) if (file.Name.StartsWith(fileNameStartWith) && file.LastWriteTime.Date == DateTime.Today.AddDays(fileCreateDay))//MobileRechargeErrorCodeDetails { using (Stream file1 = File.OpenWrite(localDirectory + remoteFileName)) { LoggerSFTPService.FileCreation("(6) file name:" + remoteFileName + " to send email", logPath); sftp.DownloadFile(remoteDirectory + remoteFileName, file1); LoggerSFTPService.FileCreation("(7) file download in desire folder: " + localDirectory + remoteFileName + "", logPath); if (EmailArray.Count() > 0) { foreach (var email in EmailArray) { LoggerSFTPService.FileCreation("(8) in loop of sending email ", logPath); var strimFile = sftp.OpenRead(remoteDirectory + remoteFileName); LoggerSFTPService.FileCreation("(9) in loop of strim strem file ", logPath); SendEmailT(EmailSubject, EmailBody, email, strimFile); } } } } } sftp.Disconnect(); LoggerSFTPService.FileCreation("sftp Disconnect successfully", logPath); } } catch (Exception ex) { LoggerSFTPService.FileCreation("sftp Exception:" + ex.Message.ToString() + "", logPath); Console.WriteLine(ex.Message.ToString()); } return(true); }