コード例 #1
0
        public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > myFunc, Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
        {
            var response = new FtpClientResponse {
                ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
            };

            try
            {
                var connectionInfo = await SetUpSftpAsync(config, myFunc);

                _sftpClient = new SftpClient(connectionInfo);
                using (_sftpClient)
                {
                    _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpConnectPolicy).Execute(() => _sftpClient.Connect());

                    var listOfFiles = _sftpClient.ListDirectory(config.Ftp.Path).Where(f => !f.IsDirectory)
                                      .Select(f => f)
                                      .ToList();
                    if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath))
                    {
                        var folderExisted = _sftpClient.Exists(config.Ftp.ArchivedPath);
                        if (!folderExisted)
                        {
                            _sftpClient.CreateDirectory(config.Ftp.ArchivedPath);
                        }
                    }
                    var resolvedStorageAccountKey =
                        await myFunc.Invoke(config.AzureBlobStorage.StorageKeyVault);


                    foreach (var item in listOfFiles)
                    {
                        bool copied     = false;
                        bool isArchived = false;

                        try
                        {
                            var fileName = item.Name;
                            using (var istream = _sftpClient.OpenRead(item.FullName))
                            {
                                MemoryStream memoryStream = new MemoryStream();
                                istream.CopyTo(memoryStream);
                                memoryStream.Position = 0;
                                copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
                                                             fileName,
                                                             config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);

                                if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath))
                                {
                                    var fileNameSplit = item.Name.Split('.');
                                    var newFileName   =
                                        fileNameSplit[0] + "_" + DateTime.UtcNow.ToString("MMddyyyy_HHmmss") + "." +
                                        fileNameSplit[1];
                                    var path = config.Ftp.ArchivedPath + "/" + newFileName;

                                    item.MoveTo(path);
                                    isArchived = true;
                                }

                                var fres = new FtpClientResponseFile
                                {
                                    Status       = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
                                    FileName     = fileName,
                                    ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
                                };
                                response.FileData.Add(fres);
                            }
                        }
                        catch (Exception ex)
                        {
                            _logger.LogError(ex, $"{nameof(ActorSftpClient)} failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
                        }
                    }
                }
                await Stop();
            }

            catch (Exception e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
                _logger.LogError(e, $"{nameof(ActorSftpClient)} failed to start creating the client of {config.Ftp.Host} : {e.Message}");
            }
            return(response);
        }
コード例 #2
0
        public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
                                                        Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
        {
            var response = new FtpClientResponse {
                ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
            };

            try
            {
                await CreateFtpClientAsync(config, securePasswordCallBack);

                Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File;
                if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex))
                {
                    var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase);
                    predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name);
                }

                var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
                                  .Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));

                if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
                {
                    var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
                    if (!folderExisted)
                    {
                        _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
                    }
                }

                var resolvedStorageAccountKey =
                    await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);

                foreach (var item in listOfFiles)
                {
                    bool copied     = false;
                    bool isArchived = false;
                    try
                    {
                        var fileName = item.Name;
                        using (var istream = await _ftpClient.OpenReadAsync(item.FullName))
                        {
                            MemoryStream memoryStream = new MemoryStream();
                            istream.CopyTo(memoryStream);
                            memoryStream.Position = 0;
                            copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
                                                         fileName,
                                                         config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
                        }
                        if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
                        {
                            isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
                        }
                        else if (config.Ftp.ArchivedPath == config.Ftp.Path)
                        {
                            //delete mode
                            isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
                            {
                                _ftpClient.DeleteFile(item.FullName);
                                return(true);
                            }, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
                        }

                        var fres = new FtpClientResponseFile
                        {
                            Status       = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
                            FileName     = item.Name,
                            ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
                        };
                        response.FileData.Add(fres);
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, $"ActorFtpClient failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
                    }
                }
                await Stop();
            }
            catch (Exception e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
                _logger.LogError(e, $"ActorFtpClient failed to start creating the client of {config.Ftp.Host} : {e.Message}");
            }
            return(response);
        }
コード例 #3
0
        public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
                                                        Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
        {
            var response = new FtpClientResponse {
                ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
            };

            try
            {
                if (_ftpClient == null)
                {
                    await CreateFtpClientAsync(config, securePasswordCallBack);
                }
                else
                {
                    if (!_ftpClient.IsConnected)
                    {
                        await CreateFtpClientAsync(config, securePasswordCallBack);
                    }
                }

                var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(f => f.Type == FtpFileSystemObjectType.File), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));

                //config.Ftp.ArchivedPath != config.Ftp.Path is delete mode
                if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
                {
                    var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
                    if (!folderExisted)
                    {
                        _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
                    }
                }

                var resolvedStorageAccountKey =
                    await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);

                Parallel.ForEach(
                    listOfFiles,
                    new ParallelOptions {
                    MaxDegreeOfParallelism = config.Parellelism
                },
                    item =>
                {
                    try
                    {
                        var fileName = item.Name;
                        using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(item.FullName), new Polly.Context($"READ_FILE_{item.FullName}")))
                        {
                            bool isArchived           = false;
                            MemoryStream memoryStream = new MemoryStream();
                            istream.CopyTo(memoryStream);
                            memoryStream.Position = 0;
                            //_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information);
                            var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
                                                             fileName,
                                                             config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);

                            if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
                            {
                                isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
                            }
                            else if (config.Ftp.ArchivedPath == config.Ftp.Path)
                            {
                                //delete mode
                                isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
                                {
                                    _ftpClient.DeleteFile(item.FullName);
                                    return(true);
                                }, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
                            }

                            var fres = new FtpClientResponseFile
                            {
                                Status       = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
                                FileName     = item.Name,
                                ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
                            };

                            var properties = new Dictionary <string, string> {
                                { "payload", JsonConvert.SerializeObject(fres) }
                            };
                            //_telemetryLogger.TrackEvent("", EventConstant.FtpCustomEventName, properties);
                            response.FileData.Add(fres);
                        }
                    }
                    catch (Exception)
                    {
                        throw;
                    }
                }
                    );

                //_telemetryLogger.TrackMetric("", EventConstant.FtpCustomEventName, 1);
                await Stop();
            }
            catch (FtpNotConnectException e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
            }
            catch (FtpSecurityNotAvailableException e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
            }
            catch (Exception e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
            }
            return(response);
        }
コード例 #4
0
        public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack, Func <string, string, string, MemoryStream, bool> copyFileFunc)
        {
            var response = new FtpClientResponse {
                ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
            };

            try
            {
                await CreateFtpClientAsync(config, securePasswordCallBack);

                Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File;
                if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex))
                {
                    var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase);
                    predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name);
                }

                var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
                                  .Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));

                if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath))
                {
                    var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
                    if (!folderExisted)
                    {
                        _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
                    }
                }

                Parallel.ForEach(
                    listOfFiles,
                    new ParallelOptions {
                    MaxDegreeOfParallelism = config.Parellelism
                },
                    item =>
                {
                    try
                    {
                        var fileName = item.FullName.Remove(0, 1);
                        using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(fileName), new Polly.Context($"READ_FILE_{item.FullName}")))
                        {
                            bool isArchived           = false;
                            MemoryStream memoryStream = new MemoryStream();
                            istream.CopyTo(memoryStream);
                            memoryStream.Position = 0;
                            //_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information);
                            var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
                                                             fileName,
                                                             config.Retry.StorageRetryPolicy, memoryStream);

                            if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath))
                            {
                                isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
                            }

                            var fres = new FtpClientResponseFile
                            {
                                Status       = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
                                FileName     = item.Name,
                                ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
                            };
                            response.FileData.Add(fres);
                        }
                    }
                    catch (Exception)
                    {
                        throw;
                    }
                }

                    );
                await Stop();
            }
            catch (FtpNotConnectException e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
            }
            catch (FtpSecurityNotAvailableException e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
            }
            catch (Exception e)
            {
                response.ErrorMessage = response.ErrorMessage + e.Message;
                response.Status       = FtpConstants.FailureStatus;
            }
            return(response);
        }