private void StartConsumingResumeQueue()
{
_logger.Debug("Starting to consume resume queue.");
while (_resumeQueue.Count != 0 && !_isStopped)
{
FileStreamInformation fileStreamInformation = _resumeQueue.Dequeue();
try
{
SendDownloadRequest(fileStreamInformation);
_fileWriter.ResumeFileWrite(fileStreamInformation);
}
catch (StreamDisconnectedException ex)
{
_logger.Warn($"Received an StreamDisconnectedException while downloading file {fileStreamInformation.FullPath}. Trying to resume.", ex);
if (fileStreamInformation.RetryCount-- > 0)
{
_resumeQueue.Enqueue(fileStreamInformation);
}
}
catch (Exception ex)
{
if (File.Exists(fileStreamInformation.FullPath))
{
File.Delete(fileStreamInformation.FullPath);
}
_logger.Error($"Failed to download file with url: {fileStreamInformation.FullPath}", ex);
}
}
}
private void StartConsumingDownloadQueue(Queue <string> urlQueue)
{
_logger.Debug($"Starting to consume queue for {GetPrefix()} urls");
_isStopped = false;
while (urlQueue.Count != 0 && !_isStopped)
{
FileStreamInformation fileStreamInformation = new FileStreamInformation();
try
{
fileStreamInformation.Url = urlQueue.Dequeue();
SendDownloadRequest(fileStreamInformation);
fileStreamInformation.FullPath = LocalDownloadPath + fileStreamInformation.Url.RemoveIllegalFileChars();
_fileWriter.WriteFileStream(fileStreamInformation);
}
catch (StreamDisconnectedException ex)
{
_logger.Warn($"Received an StreamDisconnectedException while downloading file {fileStreamInformation.FullPath}. Trying to resume.", ex);
if (fileStreamInformation.ResumeEnabled)
{
fileStreamInformation.RetryCount = int.Parse(ResumeRetryCount);
_resumeQueue.Enqueue(fileStreamInformation);
}
}
catch (Exception ex)
{
if (File.Exists(fileStreamInformation.FullPath))
{
File.Delete(fileStreamInformation.FullPath);
}
_logger.Error($"Failed to download file with url: {fileStreamInformation.FullPath}", ex);
}
}
}
public void DownloadManager_TestGoogleDownload_Success()
{
var fileStreamInformation = new FileStreamInformation();
fileStreamInformation.Url = $@"http://www.google.com/";
_httpDownloader.SendDownloadRequest(fileStreamInformation);
Assert.IsNotNull(fileStreamInformation.FileStream);
}
public void WriteFileStream(FileStreamInformation fileStreamInformation)
{
if(fileStreamInformation == null || string.IsNullOrEmpty(fileStreamInformation.FullPath) || fileStreamInformation.FileStream == null)
{
var message = "Received invalid file stream information to write!";
_logger.Error(message);
throw new ArgumentException(message);
}
fileStreamInformation.FullPath = CreateLocalFileName(fileStreamInformation.FullPath);
WriteFile(fileStreamInformation);
}
public override void SendDownloadRequest(FileStreamInformation fileStreamInformation)
{
if (string.IsNullOrEmpty(fileStreamInformation.Url))
{
throw new ArgumentException($"Received an invalid url for ftp download. Url : {fileStreamInformation.Url}");
}
using (var webClient = new WebClient())
{
fileStreamInformation.FileStream = webClient.OpenRead(fileStreamInformation.Url);
fileStreamInformation.BytesRead = 0;
fileStreamInformation.ResumeEnabled = false;
}
}
/// <summary>
/// Erzeugt eine Beschreibung aus der DVB.NET Repräsentation.
/// </summary>
/// <param name="file">Die DVB.NET Repräsentation.</param>
/// <param name="scheduleIdentifier">Die zugehörige Aufzeichnung.</param>
/// <exception cref="ArgumentNullException">Es wurde keine Information angegeben.</exception>
public static FileInformation Create(FileStreamInformation file, Guid scheduleIdentifier)
{
// Validate
if (file == null)
{
throw new ArgumentNullException("file");
}
else
{
return new FileInformation {
VideoType = file.VideoType, Path = file.FilePath, ScheduleIdentifier = scheduleIdentifier.ToString("N").ToLower()
}
};
}
private void WriteFile(FileStreamInformation fileStreamInformation)
{
//create file stream
_logger.Debug("Creating local file stream.");
byte[] buffer = new byte[BufferBytes];
int len;
using (var fileStream = File.Create(fileStreamInformation.FullPath))
{
while ((len = fileStreamInformation.FileStream.Read(buffer, 0, buffer.Length)) > 0)
{
fileStream.Write(buffer, 0, len);
fileStreamInformation.BytesRead += len;
}
}
if(fileStreamInformation.Size != fileStreamInformation.BytesRead)
}
public override void SendDownloadRequest(FileStreamInformation fileStreamInformation)
{
if (string.IsNullOrEmpty(fileStreamInformation.Url))
{
throw new ArgumentException($"Received an invalid url for ftp download. Url : {fileStreamInformation.Url}");
}
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(fileStreamInformation.Url);
request.Method = WebRequestMethods.Ftp.DownloadFile;
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
fileStreamInformation.FileStream = response.GetResponseStream();
fileStreamInformation.BytesRead = 0;
fileStreamInformation.Size = response.ContentLength;
fileStreamInformation.ResumeEnabled = String.Compare(response.Headers["Accept-Ranges"], "bytes", true) == 0;;
}
/// <summary>
/// Parse the outputBuffer to a list of FileStreamInformation structures
/// </summary>
/// <param name="outputBuffer">The outputBuffer returned from server</param>
/// <returns>A list of FileStreamInformation structures</returns>
private List <FileStreamInformation> ParseFileStreamInformations(byte[] outputBuffer)
{
List <FileStreamInformation> fileStreamInformations = new List <FileStreamInformation>();
FileStreamInformation fileStreamInformation = new FileStreamInformation();
int offset = 0;
fileStreamInformation = TypeMarshal.ToStruct <FileStreamInformation>(outputBuffer);
fileStreamInformations.Add(fileStreamInformation);
while (fileStreamInformation.NextEntryOffset != 0)
{
offset += (int)(fileStreamInformation.NextEntryOffset);
int temp = offset;
fileStreamInformation = TypeMarshal.ToStruct <FileStreamInformation>(outputBuffer, ref temp);
fileStreamInformations.Add(fileStreamInformation);
}
return(fileStreamInformations);
}
public override long Seek(long offset, SeekOrigin origin)
{
switch (origin)
{
case SeekOrigin.Begin:
_position = 0;
break;
case SeekOrigin.Current:
break;
case SeekOrigin.End:
var status = _fileStore.GetFileInformation(out FileInformation result, _fileHandle, FileInformationClass.FileStreamInformation);
status.HandleStatus();
FileStreamInformation fileStreamInformation = (FileStreamInformation)result;
_position += fileStreamInformation.Entries[0].StreamSize;
return(_position);
}
_position += offset;
return(_position);
}
public void ResumeFileWrite(FileStreamInformation fileStreamInformation)
{
_logger.Debug($"Resuming file stream download for file:{fileStreamInformation.FullPath}");
WriteFile(fileStreamInformation);
}
public static NTStatus GetFileInformation(out QueryInformation result, INTFileStore fileStore, object handle, QueryInformationLevel informationLevel)
{
result = null;
FileInformation fileInfo;
switch (informationLevel)
{
case QueryInformationLevel.SMB_INFO_QUERY_ALL_EAS:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case QueryInformationLevel.SMB_INFO_IS_NAME_VALID:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case QueryInformationLevel.SMB_QUERY_FILE_BASIC_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileBasicInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileBasicInformation fileBasicInfo = (FileBasicInformation)fileInfo;
QueryFileBasicInfo information = new QueryFileBasicInfo();
information.CreationTime = fileBasicInfo.CreationTime;
information.LastAccessTime = fileBasicInfo.LastAccessTime;
information.LastWriteTime = fileBasicInfo.LastWriteTime;
information.LastChangeTime = fileBasicInfo.LastWriteTime;
information.ExtFileAttributes = (ExtendedFileAttributes)fileBasicInfo.FileAttributes;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_STANDARD_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileStandardInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileStandardInformation fileStandardInfo = (FileStandardInformation)fileInfo;
QueryFileStandardInfo information = new QueryFileStandardInfo();
information.AllocationSize = fileStandardInfo.AllocationSize;
information.EndOfFile = fileStandardInfo.EndOfFile;
information.DeletePending = fileStandardInfo.DeletePending;
information.Directory = fileStandardInfo.Directory;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_EA_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileEaInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileEaInformation fileEAInfo = (FileEaInformation)fileInfo;
QueryFileExtendedAttributeInfo information = new QueryFileExtendedAttributeInfo();
information.EASize = fileEAInfo.EaSize;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_NAME_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileNameInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileNameInformation fileNameInfo = (FileNameInformation)fileInfo;
QueryFileNameInfo information = new QueryFileNameInfo();
information.FileName = fileNameInfo.FileName;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_ALL_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileAllInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileAllInformation fileAllInfo = (FileAllInformation)fileInfo;
QueryFileAllInfo information = new QueryFileAllInfo();
information.CreationDateTime = fileAllInfo.BasicInformation.CreationTime;
information.LastAccessDateTime = fileAllInfo.BasicInformation.LastAccessTime;
information.LastWriteDateTime = fileAllInfo.BasicInformation.LastWriteTime;
information.LastChangeTime = fileAllInfo.BasicInformation.LastWriteTime;
information.ExtFileAttributes = (ExtendedFileAttributes)fileAllInfo.BasicInformation.FileAttributes;
information.AllocationSize = fileAllInfo.StandardInformation.AllocationSize;
information.EndOfFile = fileAllInfo.StandardInformation.EndOfFile;
information.DeletePending = fileAllInfo.StandardInformation.DeletePending;
information.Directory = fileAllInfo.StandardInformation.Directory;
information.EASize = fileAllInfo.EaInformation.EaSize;
information.FileName = fileAllInfo.NameInformation.FileName;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_ALT_NAME_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileAlternateNameInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileAlternateNameInformation fileAltNameInfo = (FileAlternateNameInformation)fileInfo;
QueryFileAltNameInfo information = new QueryFileAltNameInfo();
information.FileName = fileAltNameInfo.FileName;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_STREAM_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileStreamInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileStreamInformation fileStreamInfo = (FileStreamInformation)fileInfo;
QueryFileStreamInfo information = new QueryFileStreamInfo();
information.StreamSize = fileStreamInfo.StreamSize;
information.StreamAllocationSize = fileStreamInfo.StreamAllocationSize;
information.StreamName = fileStreamInfo.StreamName;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case QueryInformationLevel.SMB_QUERY_FILE_COMPRESSION_INFO:
{
NTStatus status = fileStore.GetFileInformation(out fileInfo, handle, FileInformationClass.FileCompressionInformation);
if (status != NTStatus.STATUS_SUCCESS)
{
return(status);
}
FileCompressionInformation fileCompressionInfo = (FileCompressionInformation)fileInfo;
QueryFileCompressionInfo information = new QueryFileCompressionInfo();
information.CompressedFileSize = fileCompressionInfo.CompressedFileSize;
information.CompressionFormat = fileCompressionInfo.CompressionFormat;
information.CompressionUnitShift = fileCompressionInfo.CompressionUnitShift;
information.ChunkShift = fileCompressionInfo.ChunkShift;
information.ClusterShift = fileCompressionInfo.ClusterShift;
information.Reserved = fileCompressionInfo.Reserved;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
default:
{
result = null;
return(NTStatus.STATUS_OS2_INVALID_LEVEL);
}
}
}
public static QueryInformation FromFileInformation(FileInformation fileInformation)
{
if (fileInformation is FileBasicInformation)
{
FileBasicInformation fileBasicInfo = (FileBasicInformation)fileInformation;
QueryFileBasicInfo result = new QueryFileBasicInfo();
result.CreationTime = fileBasicInfo.CreationTime;
result.LastAccessTime = fileBasicInfo.LastAccessTime;
result.LastWriteTime = fileBasicInfo.LastWriteTime;
result.LastChangeTime = fileBasicInfo.ChangeTime;
result.ExtFileAttributes = (ExtendedFileAttributes)fileBasicInfo.FileAttributes;
return(result);
}
else if (fileInformation is FileStandardInformation)
{
FileStandardInformation fileStandardInfo = (FileStandardInformation)fileInformation;
QueryFileStandardInfo result = new QueryFileStandardInfo();
result.AllocationSize = fileStandardInfo.AllocationSize;
result.EndOfFile = fileStandardInfo.EndOfFile;
result.DeletePending = fileStandardInfo.DeletePending;
result.Directory = fileStandardInfo.Directory;
return(result);
}
else if (fileInformation is FileEaInformation)
{
FileEaInformation fileEAInfo = (FileEaInformation)fileInformation;
QueryFileEaInfo result = new QueryFileEaInfo();
result.EaSize = fileEAInfo.EaSize;
return(result);
}
else if (fileInformation is FileNameInformation)
{
FileNameInformation fileNameInfo = (FileNameInformation)fileInformation;
QueryFileNameInfo result = new QueryFileNameInfo();
result.FileName = fileNameInfo.FileName;
return(result);
}
else if (fileInformation is FileAllInformation)
{
FileAllInformation fileAllInfo = (FileAllInformation)fileInformation;
QueryFileAllInfo result = new QueryFileAllInfo();
result.CreationTime = fileAllInfo.BasicInformation.CreationTime;
result.LastAccessTime = fileAllInfo.BasicInformation.LastAccessTime;
result.LastWriteTime = fileAllInfo.BasicInformation.LastWriteTime;
result.LastChangeTime = fileAllInfo.BasicInformation.ChangeTime;
result.ExtFileAttributes = (ExtendedFileAttributes)fileAllInfo.BasicInformation.FileAttributes;
result.AllocationSize = fileAllInfo.StandardInformation.AllocationSize;
result.EndOfFile = fileAllInfo.StandardInformation.EndOfFile;
result.DeletePending = fileAllInfo.StandardInformation.DeletePending;
result.Directory = fileAllInfo.StandardInformation.Directory;
result.EaSize = fileAllInfo.EaInformation.EaSize;
result.FileName = fileAllInfo.NameInformation.FileName;
return(result);
}
else if (fileInformation is FileAlternateNameInformation)
{
FileAlternateNameInformation fileAltNameInfo = (FileAlternateNameInformation)fileInformation;
QueryFileAltNameInfo result = new QueryFileAltNameInfo();
result.FileName = fileAltNameInfo.FileName;
return(result);
}
else if (fileInformation is FileStreamInformation)
{
FileStreamInformation fileStreamInfo = (FileStreamInformation)fileInformation;
QueryFileStreamInfo result = new QueryFileStreamInfo();
result.Entries.AddRange(fileStreamInfo.Entries);
return(result);
}
else if (fileInformation is FileCompressionInformation)
{
FileCompressionInformation fileCompressionInfo = (FileCompressionInformation)fileInformation;
QueryFileCompressionInfo result = new QueryFileCompressionInfo();
result.CompressedFileSize = fileCompressionInfo.CompressedFileSize;
result.CompressionFormat = fileCompressionInfo.CompressionFormat;
result.CompressionUnitShift = fileCompressionInfo.CompressionUnitShift;
result.ChunkShift = fileCompressionInfo.ChunkShift;
result.ClusterShift = fileCompressionInfo.ClusterShift;
result.Reserved = fileCompressionInfo.Reserved;
return(result);
}
else
{
throw new NotImplementedException();
}
}
public NTStatus GetFileInformation(out FileInformation result, object handle, FileInformationClass informationClass)
{
FileHandle fileHandle = (FileHandle)handle;
string path = fileHandle.Path;
FileSystemEntry entry;
try
{
entry = m_fileSystem.GetEntry(path);
}
catch (Exception ex)
{
if (ex is IOException || ex is UnauthorizedAccessException)
{
NTStatus status = ToNTStatus(ex);
Log(Severity.Verbose, "GetFileInformation on '{0}' failed. {1}", path, status);
result = null;
return(status);
}
else
{
throw;
}
}
switch (informationClass)
{
case FileInformationClass.FileBasicInformation:
{
FileBasicInformation information = new FileBasicInformation();
information.CreationTime = entry.CreationTime;
information.LastAccessTime = entry.LastAccessTime;
information.LastWriteTime = entry.LastWriteTime;
information.ChangeTime = entry.LastWriteTime;
information.FileAttributes = GetFileAttributes(entry);
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FileStandardInformation:
{
FileStandardInformation information = new FileStandardInformation();
information.AllocationSize = (long)GetAllocationSize(entry.Size);
information.EndOfFile = (long)entry.Size;
information.Directory = entry.IsDirectory;
information.DeletePending = fileHandle.DeleteOnClose;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FileInternalInformation:
{
FileInternalInformation information = new FileInternalInformation();
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FileEaInformation:
{
FileEaInformation information = new FileEaInformation();
information.EaSize = 0;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FileAccessInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileNameInformation:
{
FileNameInformation information = new FileNameInformation();
information.FileName = entry.Name;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FilePositionInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileFullEaInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileModeInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileAlignmentInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileAllInformation:
{
FileAllInformation information = new FileAllInformation();
information.BasicInformation.CreationTime = entry.CreationTime;
information.BasicInformation.LastAccessTime = entry.LastAccessTime;
information.BasicInformation.LastWriteTime = entry.LastWriteTime;
information.BasicInformation.ChangeTime = entry.LastWriteTime;
information.BasicInformation.FileAttributes = GetFileAttributes(entry);
information.StandardInformation.AllocationSize = (long)GetAllocationSize(entry.Size);
information.StandardInformation.EndOfFile = (long)entry.Size;
information.StandardInformation.Directory = entry.IsDirectory;
information.StandardInformation.DeletePending = fileHandle.DeleteOnClose;
information.NameInformation.FileName = entry.Name;
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FileAlternateNameInformation:
{
// If there is no alternate name Windows will return STATUS_OBJECT_NAME_NOT_FOUND
result = null;
return(NTStatus.STATUS_OBJECT_NAME_NOT_FOUND);
}
case FileInformationClass.FileStreamInformation:
{
// This information class is used to enumerate the data streams of a file or a directory.
// A buffer of FileStreamInformation data elements is returned by the server.
FileStreamInformation information = new FileStreamInformation();
List <KeyValuePair <string, ulong> > dataStreams = m_fileSystem.ListDataStreams(fileHandle.Path);
foreach (KeyValuePair <string, ulong> dataStream in dataStreams)
{
FileStreamEntry streamEntry = new FileStreamEntry();
ulong streamSize = dataStream.Value;
streamEntry.StreamSize = (long)streamSize;
streamEntry.StreamAllocationSize = (long)GetAllocationSize(streamSize);
streamEntry.StreamName = dataStream.Key;
information.Entries.Add(streamEntry);
}
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FilePipeInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FilePipeLocalInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FilePipeRemoteInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileCompressionInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
case FileInformationClass.FileNetworkOpenInformation:
{
FileNetworkOpenInformation information = new FileNetworkOpenInformation();
information.CreationTime = entry.CreationTime;
information.LastAccessTime = entry.LastAccessTime;
information.LastWriteTime = entry.LastWriteTime;
information.ChangeTime = entry.LastWriteTime;
information.AllocationSize = (long)GetAllocationSize(entry.Size);
information.EndOfFile = (long)entry.Size;
information.FileAttributes = GetFileAttributes(entry);
result = information;
return(NTStatus.STATUS_SUCCESS);
}
case FileInformationClass.FileAttributeTagInformation:
{
result = null;
return(NTStatus.STATUS_NOT_IMPLEMENTED);
}
default:
result = null;
return(NTStatus.STATUS_INVALID_INFO_CLASS);
}
}
public abstract void SendResumeRequest(FileStreamInformation fileStreamInformation);
public abstract void SendDownloadRequest(FileStreamInformation fileStreamInformation);
