private IEnumerable <FTPEntity> GetTarGzFilesDetail(String ftpUrl)
{
List <FTPEntity> result = new List <FTPEntity>();
try
{
var ftpRequester = WebRequest.Create(ftpUrl) as FtpWebRequest;
if (ftpRequester != null)
{
ftpRequester.Method = WebRequestMethods.Ftp.ListDirectoryDetails;
using (var resonse = ftpRequester.GetResponse())
{
var responseStream = resonse.GetResponseStream();
if (responseStream != null)
{
using (var streamReader = new StreamReader(responseStream))
{
var detailAggregatge = streamReader.ReadToEnd();
var detailLines = detailAggregatge.Split(new String[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
var filesDetailLines = detailLines.Where(x => x.StartsWith("-") && x.EndsWith(".tar.gz"));
foreach (var detailLine in filesDetailLines)
{
/*
* -rw-rw-r-- 1 919 501 240045532 Dec 04 07:35 Apis_capp_global_20151204-082045.tar.gz
* -rw-rw-r-- 1 919 501 211800045 Dec 03 14:39 Apis_cass_global_20151203-153959.tar.gz
*/
String[] majorParts = detailLine.Split(new String[] { " ", " " }, StringSplitOptions.RemoveEmptyEntries);
// reprets info like size, date and name
String mainDesription = majorParts.Last();
String[] majorSubParts = mainDesription.Split(new String[] { " " }, StringSplitOptions.RemoveEmptyEntries);
String fileName = majorSubParts[4]; // apis_20151214-211526.tar.gz
String unformatedDate = fileName.Split('_').Last().Split('-').First();
DateTime date = DateTime.ParseExact(unformatedDate, "yyyyMMdd", System.Globalization.CultureInfo.InvariantCulture);
String size = majorSubParts[0];
FTPEntity ftpEntity = new FTPEntity()
{
FileDate = date,
FileName = fileName,
FileSize = size,
FilePath = Path.Combine(ftpUrl, fileName)
};
if (ftpEntity.FileName.EndsWith(".tar.gz"))
{
result.Add(ftpEntity);
}
}
}
}
}
}
}
catch (Exception e)
{
this.Logger.Error(ftpUrl, e);
}
return(result);
}
private IEnumerable <(String filePath, DateTime date, Byte[] data)> DownloadFiles(FTPEntity ftpEntity)
{
try
{
var request = (FtpWebRequest)WebRequest.Create(ftpEntity.FilePath);
request.Timeout = (int)TimeSpan.FromMinutes(10).TotalMilliseconds;
request.ReadWriteTimeout = request.Timeout;
using (var webResponse = (FtpWebResponse)request.GetResponse())
using (var responseStream = webResponse.GetResponseStream())
{
var unziped = UnzipToMemory(originalZipStream: responseStream);
return(unziped.Select(x => (x.Key, ftpEntity.FileDate, x.Value)));
}
}
catch (Exception e)
{
this.Logger.Error(ftpEntity.FilePath, e);
return(Enumerable.Empty <(String, DateTime, byte[])>());
}
}
