public string GetLocalDestinationPath(string remotePath, string localPath)
{
string tempFileDirPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, localPath);
logger.AddInformationLog(nameof(tempFileDirPath) + ": " + tempFileDirPath);
DirectoryInfo tempFileDir = new DirectoryInfo(tempFileDirPath);
logger.AddInformationLog(nameof(tempFileDir) + " object is created.");
if (!tempFileDir.Exists)
{
logger.AddInformationLog(nameof(tempFileDir) + " not exists.");
tempFileDir.Create();
logger.AddInformationLog(nameof(tempFileDir) + " created.");
}
string fileName = remotePath.Substring(remotePath.LastIndexOf(@"/") + 1);
string localDestinationPath = Path.Combine(tempFileDir.FullName, fileName);
return(BuildUniqueFilePath(localDestinationPath));
}
private string DownloadFTPFile(string remoteFtpPath, string localPath, string username, string password)
{
string localDestinationPath = string.Empty;
try
{
localDestinationPath = iOHelper.GetLocalDestinationPath(remoteFtpPath, localPath);
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(remoteFtpPath);
request.Method = WebRequestMethods.Ftp.DownloadFile;
request.Credentials = new NetworkCredential(username, password);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
var bufferSizeConfigValue = ConfigurationManager.AppSettings["BufferSize"];
logger.AddInformationLog($"BufferSize config value: {bufferSizeConfigValue}");
var bufferSize = !string.IsNullOrWhiteSpace(bufferSizeConfigValue) ? Convert.ToInt32(bufferSizeConfigValue) : 2048;
using (FileStream writer = new FileStream(localDestinationPath, FileMode.Create))
{
byte[] buffer = new byte[bufferSize];
int readCount = responseStream.Read(buffer, 0, bufferSize);
while (readCount > 0)
{
writer.Write(buffer, 0, readCount);
readCount = responseStream.Read(buffer, 0, bufferSize);
}
}
logger.AddInformationLog($"Download file {remoteFtpPath} completed");
response.Close();
return(localDestinationPath);
}
catch (Exception ex)
{
var fileInfo = new FileInfo(localDestinationPath);
if (fileInfo.Exists)
{
File.Delete(fileInfo.FullName);
}
logger.AddInformationLog($"Download file {remoteFtpPath} didn't complete");
throw;
}
}
public void Process(string sources)
{
//TODO : Move this logic to separate windows service and make the service listen on Message Queue like Rabbitmq
logger.AddInformationLog($"sources value: {sources}");
//Data validations
if (string.IsNullOrWhiteSpace(sources))
{
throw new ArgumentException($"The sources Is null or empty string.");
}
string delimiter = ConfigurationManager.AppSettings["Delimiter"];
logger.AddInformationLog($"Delimiter config value: {delimiter}");
string sourcesParsingDelimiter = string.IsNullOrWhiteSpace(delimiter) ? "," : delimiter;
logger.AddInformationLog($"Sources parsing delimiter: {sourcesParsingDelimiter}");
string localPath = ConfigurationManager.AppSettings["LocalDirectory"];
logger.AddInformationLog($"LocalDirectory config value: {localPath}");
List <string> soursesUrls = parser.Parse(sources, sourcesParsingDelimiter).ToList();
logger.AddInformationLog($"sourses Urls counts after parsing: {soursesUrls?.Count}");
ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>();
using (repoDownloadedFile)
{
Parallel.ForEach(soursesUrls, url =>
{
try
{
IDownloadStrategy downloadStrategy = downloadStrategyFactory.Build(url);
string physicalPath = downloadStrategy.Download(url, localPath);
string virtualPath = $"{localPath.Replace('\\', '/')}/{Path.GetFileName(physicalPath)}";
logger.AddInformationLog($"Local path is: {localPath} for URL: {url} ");
repoDownloadedFile.Add(new DomainModels.DownloadedFile()
{
FileRemotePath = url,
LocalPath = virtualPath,
ProcessingStatus = (int)DownloadStatutes.ReadyForProcessing,
});
repoDownloadedFile.SaveChanges();
logger.AddInformationLog($"Source :{url} saved to the database with status ready for processing");
}
catch (Exception ex)
{
exceptions.Enqueue(ex);
logger.AddErrorLog(ex);
}
});
if (exceptions.Any())
{
throw new AggregateException(exceptions);
}
}
}
private string DownloadSFTPFile(string host, int port, string username, string password, string sftpInboundPath, string filename)
{
try
{
using (SftpClient client = new SftpClient(host, port, username, password))
{
logger.AddInformationLog(nameof(client) + " is created.");
client.Connect();
logger.AddInformationLog(nameof(client) + " is connected.");
if (client.IsConnected)
{
logger.AddInformationLog("FTP Connection established.");
string tempFileDirPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Temp");
logger.AddInformationLog(nameof(tempFileDirPath) + ": " + tempFileDirPath);
DirectoryInfo tempFileDir = new DirectoryInfo(tempFileDirPath);
logger.AddInformationLog(nameof(tempFileDir) + " object is created.");
if (!tempFileDir.Exists)
{
logger.AddInformationLog(nameof(tempFileDir) + " not exists.");
tempFileDir.Create();
logger.AddInformationLog(nameof(tempFileDir) + " created.");
}
string tempFilePath = $"{tempFileDir.FullName}\\{filename}";
logger.AddInformationLog(nameof(tempFilePath) + ": " + tempFilePath);
// Delete the file if exists.
if (File.Exists(tempFilePath))
{
logger.AddInformationLog(nameof(tempFilePath) + " exists.");
File.Delete(tempFilePath);
logger.AddInformationLog(nameof(tempFilePath) + " deleted.");
}
using (FileStream file = File.OpenWrite(tempFilePath))
{
logger.AddInformationLog(nameof(file) + " opened.");
string bufferSize = ConfigurationManager.AppSettings["BufferSize"];//4096
client.BufferSize = string.IsNullOrWhiteSpace(bufferSize) ? 4096 : Convert.ToUInt32(bufferSize);
logger.AddInformationLog($"BufferSize config value: {client?.BufferSize}");
client.DownloadFile($"{sftpInboundPath}{filename}", file);
logger.AddInformationLog($"file download from sftp success at path : {tempFilePath}");
}
return(tempFilePath);
}
else
{
logger.AddErrorLog("Unable to establish FTP Connection.");
}
return(null);
}
}
catch (Exception ex)
{
logger.AddErrorLog("Error downloading the file via FTP.", ex);
throw;
}
}