private void WriteFileToLocalStorage(FtpListItem file, string fileName, string localPath, string remotePath, string archiveFilePath, bool archive) { Stream ftpStream = null; FileStream fileStream = null; try { string destinationPath = string.Format(@"{0}\{1}", localPath, file.Name); ftpStream = ftp.OpenRead(file.FullName); if (ftpStream.Length > 0) { fileStream = File.Create(destinationPath, (int)ftpStream.Length); byte[] buffer = new byte[200 * 1024]; int count; while ((count = ftpStream.Read(buffer, 0, buffer.Length)) > 0) { fileStream.Write(buffer, 0, count); } } if (archive && !string.IsNullOrEmpty(archiveFilePath)) { ArchiveFile(remotePath, archiveFilePath, fileName); } } catch (Exception ex) { throw ex; } finally { if (ftpStream != null) { ftpStream.Flush(); ftpStream.Close(); } if (fileStream != null) { fileStream.Flush(); fileStream.Close(); } } }
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack, Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc) { var response = new FtpClientResponse { ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>() }; try { if (_ftpClient == null) { await CreateFtpClientAsync(config, securePasswordCallBack); } else { if (!_ftpClient.IsConnected) { await CreateFtpClientAsync(config, securePasswordCallBack); } } var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(f => f.Type == FtpFileSystemObjectType.File), new Polly.Context($"LIST_FILE_{config.Ftp.Path}")); //config.Ftp.ArchivedPath != config.Ftp.Path is delete mode if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path) { var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath); if (!folderExisted) { _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER")); } } var resolvedStorageAccountKey = await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault); Parallel.ForEach( listOfFiles, new ParallelOptions { MaxDegreeOfParallelism = config.Parellelism }, item => { try { var fileName = item.Name; using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(item.FullName), new Polly.Context($"READ_FILE_{item.FullName}"))) { bool isArchived = false; MemoryStream memoryStream = new MemoryStream(); istream.CopyTo(memoryStream); memoryStream.Position = 0; //_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information); var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName, fileName, config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey); if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path) { isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE")); } else if (config.Ftp.ArchivedPath == config.Ftp.Path) { //delete mode isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => { _ftpClient.DeleteFile(item.FullName); return(true); }, new Polly.Context("DELETE_FILE_TO_ARCHIVE")); } var fres = new FtpClientResponseFile { Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus, FileName = item.Name, ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError) }; var properties = new Dictionary <string, string> { { "payload", JsonConvert.SerializeObject(fres) } }; //_telemetryLogger.TrackEvent("", EventConstant.FtpCustomEventName, properties); response.FileData.Add(fres); } } catch (Exception) { throw; } } ); //_telemetryLogger.TrackMetric("", EventConstant.FtpCustomEventName, 1); await Stop(); } catch (FtpNotConnectException e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; } catch (FtpSecurityNotAvailableException e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; } catch (Exception e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; } return(response); }
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack, Func <string, string, string, MemoryStream, bool> copyFileFunc) { var response = new FtpClientResponse { ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>() }; try { await CreateFtpClientAsync(config, securePasswordCallBack); Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File; if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex)) { var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase); predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name); } var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy) .Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}")); if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath)) { var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath); if (!folderExisted) { _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER")); } } Parallel.ForEach( listOfFiles, new ParallelOptions { MaxDegreeOfParallelism = config.Parellelism }, item => { try { var fileName = item.FullName.Remove(0, 1); using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(fileName), new Polly.Context($"READ_FILE_{item.FullName}"))) { bool isArchived = false; MemoryStream memoryStream = new MemoryStream(); istream.CopyTo(memoryStream); memoryStream.Position = 0; //_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information); var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName, fileName, config.Retry.StorageRetryPolicy, memoryStream); if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath)) { isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE")); } var fres = new FtpClientResponseFile { Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus, FileName = item.Name, ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError) }; response.FileData.Add(fres); } } catch (Exception) { throw; } } ); await Stop(); } catch (FtpNotConnectException e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; } catch (FtpSecurityNotAvailableException e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; } catch (Exception e) { response.ErrorMessage = response.ErrorMessage + e.Message; response.Status = FtpConstants.FailureStatus; } return(response); }