private async Task CompressSmallFile(FileHeader header)
{
header.NumberOfChunks = 1;
await using var readStream = ArchiveStream.OpenRead(header.FullPath);
var chunkCompressor = GetChunkCompressor();
var chunk = await readStream.ReadBytesAsync(header.FileSize);
var compressedChunk = await chunkCompressor(chunk, header.CompressionType);
var chunkHeader = new ChunkHeader()
{
Size = compressedChunk.Length,
SerialNumber = 0
};
header.Chunks.Add(chunkHeader);
using (await _asyncLock.LockAsync())
{
header.Position = _outputStream.Position + header.SizeOf;
await _outputStream.WriteFileHeaderAsync(header);
await _outputStream.WriteAsync(compressedChunk);
}
_archiveProgress.Report(header.RelativePath, chunk.Length, chunkHeader.SerialNumber, header.NumberOfChunks);
}
public async Task ExtractAsync(string inputPath, string outputPath)
{
await using var stream = ArchiveStream.OpenRead(inputPath);
var directoryHeaders = (await stream.ReadAllDirectoriesAsync(outputPath)).ToList();
var fileHeaders = (await stream.ReadAllFileHeadersAsync(outputPath)).ToList();
directoryHeaders.AsParallel().ForAll(x =>
{
if (!string.IsNullOrEmpty(x.FullPath) && !Directory.Exists(x.FullPath))
{
Directory.CreateDirectory(x.FullPath);
}
});
try
{
await Decompressor.DecompressAsync(stream, fileHeaders);
}
catch (Exception ex)
{
await DeleteFiles(fileHeaders);
await DeleteDirectories(directoryHeaders);
throw new ArchiveException(ex.Message);
}
}
private async Task DecompressFileAsync(FileHeader fileHeader)
{
await using var outputStream = ArchiveStream.Create(fileHeader.FullPath);
var chunkDecompressor = GetChunkDecompressor(fileHeader.IsEncrypted);
foreach (var chunk in fileHeader.Chunks)
{
byte[] compressChunk;
using (await _asyncLock.LockAsync())
{
_inputStream.Position = fileHeader.Position;
compressChunk = await _inputStream.ReadBytesAsync(chunk.Size);
}
fileHeader.Position += chunk.Size;
try
{
await chunkDecompressor(compressChunk, outputStream, fileHeader.CompressionType);
}
catch
{
if (fileHeader.IsEncrypted)
{
throw new ArchiveException(ExceptionResource.InvalidEncryptKey);
}
throw new ArchiveException(ExceptionResource.ThisFileIsCorrupted);
}
_archiveProgress.Report(fileHeader.RelativePath, chunk.Size, chunk.SerialNumber, fileHeader.NumberOfChunks);
}
}
public async Task CreateAsync(string inputPath, string outputPath)
{
await using var outputStream = ArchiveStream.Create(outputPath);
var metadata = new Metadata.Metadata(inputPath);
var directoryHeaders = await metadata.GenerateDirectoryHeadersAsync();
var fileHeaders = await metadata.GenerateFileHeadersAsync();
var archiveInfo = new ArchiveInfo()
{
Header = ArchiveResource.ArchiveHeader,
IsEncrypted = Compressor.Settings.IsEncryptEnable,
NumberOfDirectories = directoryHeaders.Count,
NumberOfFiles = fileHeaders.Count,
};
archiveInfo.DirectoriesBlockPosition = archiveInfo.SizeOf;
outputStream.Seek(archiveInfo.SizeOf, SeekOrigin.Current);
await outputStream.WriteDirectoriesAsync(directoryHeaders);
archiveInfo.FilesBlockPosition = outputStream.Position;
await outputStream.WriteArchiveInfoAsync(archiveInfo);
outputStream.Position = archiveInfo.FilesBlockPosition;
await Compressor.CompressAsync(fileHeaders, outputStream);
}
private void ReadCentralDirectory()
{
try
{
// assume ReadEndOfCentralDirectory has been called and has populated _centralDirectoryStart
ArchiveStream.Seek(_centralDirectoryStart, SeekOrigin.Begin);
long numberOfEntries = 0;
//read the central directory
while (ZipCentralDirectoryFileHeader.TryReadBlock(ArchiveReader, out var currentHeader))
{
AddEntry(new ZipArchiveEntry(this, currentHeader));
numberOfEntries++;
}
if (numberOfEntries != _expectedNumberOfEntries)
{
throw new InvalidDataException(SR.NumEntriesWrong);
}
}
catch (EndOfStreamException ex)
{
throw new InvalidDataException(SR.Format(SR.CentralDirectoryInvalid, ex));
}
}
public async Task <List <FileHeader> > GetFilesAsync(string path)
{
await using var stream = ArchiveStream.OpenRead(path);
var fileHeaders = await stream.ReadAllFileHeadersAsync(path);
return(fileHeaders.ToList());
}
public async Task <bool> IsEncryptedAsync(string path)
{
await using var stream = ArchiveStream.OpenRead(path);
var archiveInfo = await stream.ReadArchiveInfoAsync();
return(archiveInfo.IsEncrypted);
}
public async Task <ArchiveInfo> GetArchiveInfoAsync(string path)
{
await using var stream = ArchiveStream.OpenRead(path);
var archiveInfo = await stream.ReadArchiveInfoAsync();
return(archiveInfo);
}
public async Task <List <DirectoryHeader> > GetDirectoriesAsync(string path)
{
await using var stream = ArchiveStream.OpenRead(path);
var directoryHeaders = await stream.ReadAllDirectoriesAsync(path);
return(directoryHeaders.ToList());
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
Archive?.Dispose();
ArchiveStream?.Dispose();
}
disposedValue = true;
}
}
public async Task <byte[]> ReadFile(FileRecord record)
{
var isCompressed = (Header.ArchiveFlags & BSA105ArchiveFlags.CompressedArchive) == BSA105ArchiveFlags.CompressedArchive;
if ((isCompressed && !record.CompressionBit) || (!isCompressed && record.CompressionBit))
{
return(await ReadCompressedFile(record));
}
ArchiveStream.Seek(record.DataOffset, SeekOrigin.Begin);
var fileBytes = new byte[record.Size];
await ArchiveStream.ReadAsync(fileBytes, 0, fileBytes.Length);
return(fileBytes);
}
public async Task <bool> IsArchiveAsync(string path)
{
await using var stream = ArchiveStream.OpenRead(path);
try
{
await stream.ReadArchiveInfoAsync();
return(true);
}
catch
{
return(false);
}
}
public async Task <byte[]> ReadCompressedFile(FileRecord record)
{
ArchiveStream.Seek(record.DataOffset, SeekOrigin.Begin);
var reader = new BinaryReader(ArchiveStream);
var realSize = reader.ReadUInt32();
var source = new byte[record.Size - 4];
await ArchiveStream.ReadAsync(source, 0, source.Length);
using (var memory = new MemoryStream(source))
{
var output = LZ4Stream.Decode(memory);
var targetBytes = new byte[realSize];
await output.ReadAsync(targetBytes, 0, targetBytes.Length);
return(targetBytes);
}
}
private void CloseStreams()
{
if (!_leaveOpen)
{
ArchiveStream.Dispose();
_backingStream?.Dispose();
ArchiveReader?.Dispose();
}
else
{
// if _backingStream isn't null, that means we assigned the original stream they passed
// us to _backingStream (which they requested we leave open), and _archiveStream was
// the temporary copy that we needed
if (_backingStream != null)
{
ArchiveStream.Dispose();
}
}
}
/// <summary>
/// Starts recorded video stream for the requested cameraId
/// </summary>
/// <param name="cameraId"></param>
/// /// <param name="compression">mjpeg or h264 - if null provided will default to transcoded MJPEG</param>
/// <param name="startTime">Start time in UTC</param>
/// <param name="endTime">End time in UTC - optional</param>
/// <returns>The HTTP MJPEG or RTSP H256 Url need to acquire the stream from the Transcoder - or null upon failure</returns>
public async Task <ArchiveStream> StartArchive(Guid cameraId, string compression, DateTime startTime, DateTime endTime = default(DateTime))
{
ArchiveStream archiveStreamDetails = null;
try
{
if (SessionId != Guid.Empty)
{
var startTimeStr = startTime == default(DateTime) ? "instantreplay" : startTime.ToString(CultureInfo.InvariantCulture);
var endTimeStr = endTime == default(DateTime) ? string.Empty : string.Format("&EndTime={0}", endTime);
var compressionParam = string.IsNullOrWhiteSpace(compression) ? string.Empty : string.Format("&Compression={0}", compression);
// compressionParam is optional (defaults to MJPEG) and endTimeStr is also optional
var response = await _httpClient.GetAsync(string.Format("/StartArchive?CameraGuid={0}&SessionId={1}{2}&StartTime={3}{4}", cameraId, SessionId, compressionParam, startTimeStr, endTimeStr));
response.EnsureSuccessStatusCode(); // Throw if not a success code.
var stream = await response.Content.ReadAsStreamAsync();
var serializer = new XmlSerializer(typeof(WebResponse));
var webResponse = serializer.Deserialize(stream) as WebResponse;
if (webResponse != null)
{
archiveStreamDetails = webResponse.Body.ArchiveStream;
if (webResponse.Header.Error == ErrorType.None)
{
Utils.Trace("UnitedVmsProvider StartArchive Url: " + webResponse.Body.ArchiveStream.Url);
Utils.Trace("UnitedVmsProvider StartArchive StreamStatus: " + webResponse.Body.ArchiveStream.StreamStatus);
}
else
{
Utils.Trace("UnitedVmsProvider StartArchive Error Response: " + webResponse.Header.Error);
}
}
}
}
catch (Exception ex)
{
Utils.Trace("UnitedVmsProvider StartArchive Error", ex);
}
return(archiveStreamDetails);
}
private static void UnpackOnFirstRun()
{
Debug.WriteLine("unpack archive - " + Globals.MainZzzPath);
Directory.CreateDirectory(WorkspacePath);
foreach (var f in WorkspaceFiles)
{
var destPath = Path.Combine(WorkspacePath, f);
Directory.CreateDirectory(Path.GetDirectoryName(destPath));
if (!File.Exists(destPath))
{
using (var source = new ArchiveStream(Globals.MainZzzPath + @";data\" + f))
using (var dest = new FileStream(destPath, FileMode.Create, FileAccess.Write, FileShare.None))
{
source.CopyTo(dest);
}
}
}
}
private async Task CompressBigFileAsync(FileHeader header)
{
await using var readStream = ArchiveStream.OpenRead(header.FullPath, Settings.ThreadsCount);
var chunkCompressor = GetChunkCompressor();
header.NumberOfChunks = readStream.GetChunksCount();
_outputStream.Seek(header.SizeOf, SeekOrigin.Current);
header.Position = _outputStream.Position;
var chunkSerialNumber = 0;
await foreach (var chunks in readStream.ReadFileInChunksAsync(Settings.ChunkSize))
{
var tasks = chunks.Select(x => chunkCompressor(x, header.CompressionType));
var chunksSizes = chunks.Select(x => x.Length).ToArray();
var compressedChunks = await Task.WhenAll(tasks);
for (var i = 0; i < compressedChunks.Length; i++)
{
var chunkHeader = new ChunkHeader()
{
Size = compressedChunks[i].Length,
SerialNumber = chunkSerialNumber++,
};
header.Chunks.Add(chunkHeader);
await _outputStream.WriteAsync(compressedChunks[i]);
_archiveProgress.Report(header.RelativePath, chunksSizes[i], chunkHeader.SerialNumber,
header.NumberOfChunks);
}
var currentPosition = _outputStream.Position;
_outputStream.Position = header.Position - header.SizeOf;
await _outputStream.WriteFileHeaderAsync(header);
_outputStream.Position = currentPosition;
}
}
// This function reads all the EOCD stuff it needs to find the offset to the start of the central directory
// This offset gets put in _centralDirectoryStart and the number of this disk gets put in _numberOfThisDisk
// Also does some verification that this isn't a split/spanned archive
// Also checks that offset to CD isn't out of bounds
private void ReadEndOfCentralDirectory()
{
try
{
// this seeks to the start of the end of central directory record
ArchiveStream.Seek(-ZipEndOfCentralDirectoryBlock.SizeOfBlockWithoutSignature, SeekOrigin.End);
if (!ZipHelper.SeekBackwardsToSignature(ArchiveStream, ZipEndOfCentralDirectoryBlock.SignatureConstant))
{
throw new InvalidDataException(SR.EOCDNotFound);
}
long eocdStart = ArchiveStream.Position;
// read the EOCD
ZipEndOfCentralDirectoryBlock eocd;
bool eocdProper = ZipEndOfCentralDirectoryBlock.TryReadBlock(ArchiveReader, out eocd);
Debug.Assert(eocdProper); // we just found this using the signature finder, so it should be okay
if (eocd.NumberOfThisDisk != eocd.NumberOfTheDiskWithTheStartOfTheCentralDirectory)
{
throw new InvalidDataException(SR.SplitSpanned);
}
NumberOfThisDisk = eocd.NumberOfThisDisk;
_centralDirectoryStart = eocd.OffsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber;
if (eocd.NumberOfEntriesInTheCentralDirectory != eocd.NumberOfEntriesInTheCentralDirectoryOnThisDisk)
{
throw new InvalidDataException(SR.SplitSpanned);
}
_expectedNumberOfEntries = eocd.NumberOfEntriesInTheCentralDirectory;
// only bother looking for zip64 EOCD stuff if we suspect it is needed because some value is FFFFFFFFF
// because these are the only two values we need, we only worry about these
// if we don't find the zip64 EOCD, we just give up and try to use the original values
if (eocd.NumberOfThisDisk == ZipHelper.Mask16Bit ||
eocd.OffsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber == ZipHelper.Mask32Bit ||
eocd.NumberOfEntriesInTheCentralDirectory == ZipHelper.Mask16Bit)
{
// we need to look for zip 64 EOCD stuff
// seek to the zip 64 EOCD locator
ArchiveStream.Seek(eocdStart - Zip64EndOfCentralDirectoryLocator.SizeOfBlockWithoutSignature, SeekOrigin.Begin);
// if we don't find it, assume it doesn't exist and use data from normal eocd
if (ZipHelper.SeekBackwardsToSignature(ArchiveStream, Zip64EndOfCentralDirectoryLocator.SignatureConstant))
{
// use locator to get to Zip64EOCD
Zip64EndOfCentralDirectoryLocator locator;
bool zip64EOCDLocatorProper = Zip64EndOfCentralDirectoryLocator.TryReadBlock(ArchiveReader, out locator);
Debug.Assert(zip64EOCDLocatorProper); // we just found this using the signature finder, so it should be okay
if (locator.OffsetOfZip64EOCD > long.MaxValue)
{
throw new InvalidDataException(SR.FieldTooBigOffsetToZip64EOCD);
}
long zip64EOCDOffset = (long)locator.OffsetOfZip64EOCD;
ArchiveStream.Seek(zip64EOCDOffset, SeekOrigin.Begin);
// read Zip64EOCD
Zip64EndOfCentralDirectoryRecord record;
if (!Zip64EndOfCentralDirectoryRecord.TryReadBlock(ArchiveReader, out record))
{
throw new InvalidDataException(SR.Zip64EOCDNotWhereExpected);
}
NumberOfThisDisk = record.NumberOfThisDisk;
if (record.NumberOfEntriesTotal > long.MaxValue)
{
throw new InvalidDataException(SR.FieldTooBigNumEntries);
}
if (record.OffsetOfCentralDirectory > long.MaxValue)
{
throw new InvalidDataException(SR.FieldTooBigOffsetToCD);
}
if (record.NumberOfEntriesTotal != record.NumberOfEntriesOnThisDisk)
{
throw new InvalidDataException(SR.SplitSpanned);
}
_expectedNumberOfEntries = (long)record.NumberOfEntriesTotal;
_centralDirectoryStart = (long)record.OffsetOfCentralDirectory;
}
}
if (_centralDirectoryStart > ArchiveStream.Length)
{
throw new InvalidDataException(SR.FieldTooBigOffsetToCD);
}
}
catch (EndOfStreamException ex)
{
throw new InvalidDataException(SR.CDCorrupt, ex);
}
catch (IOException ex)
{
throw new InvalidDataException(SR.CDCorrupt, ex);
}
}
