private void TryFileCompression(ref FileCopyInfo fileCopyInfo, out bool fileCompressed)
{
fileCompressed = false;
if (fileCopyInfo.SourceFileName.EndsWith(".zip"))
{
// already compressed skip it.
return;
}
string tempArchivePath = string.Empty;
string actualFilePath = fileCopyInfo.SourceFullPath;
string archiveEntryName = Path.GetFileNameWithoutExtension(fileCopyInfo.SourceFileName);
try
{
FileCompressor.Compress(fileCopyInfo.SourceFullPath, archiveEntryName, out tempArchivePath);
Utility.PerformIOWithRetries(
() => { FabricFile.Delete(actualFilePath); });
}
catch (Exception e)
{
// Consume the exception and upload the uncompressed file.
this.TraceSource.WriteExceptionAsWarning(
this.LogSourceId,
e,
"Failed to compress crash dump for upload");
return;
}
fileCopyInfo.SourceFileName = archiveEntryName + ".zip";
fileCopyInfo.SourceFullPath = tempArchivePath;
fileCompressed = true;
}
public void TestMethodMoveArchive()
{
string srcFullFileName = Path.Combine(SourceFolderTest, SourceFileName);
string arcFullFileName = FileCompressor.GetArchiveFileName(srcFullFileName);
string arcFileName = Path.GetFileName(arcFullFileName);
string destFullFileName = Path.Combine(TargetFolderTest, arcFileName);
string[] sourceFiles;
string[] sourceZipFiles;
string[] targetFiles;
sourceFiles = Directory.GetFiles(SourceFolderTest, SourceFileName);
sourceZipFiles = Directory.GetFiles(SourceFolderTest, arcFileName);
targetFiles = Directory.GetFiles(TargetFolderTest, arcFileName);
Assert.AreEqual(1, sourceFiles.Length, "Source file must be present");
Assert.AreEqual(0, sourceZipFiles.Length, "Source compressed file must be absent");
Assert.AreEqual(0, targetFiles.Length, "Target compreesd file must be absent");
FileCompressor.Compress(srcFullFileName);
sourceZipFiles = Directory.GetFiles(SourceFolderTest, arcFileName);
Assert.AreEqual(1, sourceZipFiles.Length, "Source compressed file must be present");
FileMover.Move(arcFullFileName, destFullFileName);
sourceZipFiles = Directory.GetFiles(SourceFolderTest, arcFileName);
targetFiles = Directory.GetFiles(TargetFolderTest, arcFileName);
Assert.AreEqual(0, sourceZipFiles.Length, "Source compressed file must be absent");
Assert.AreEqual(1, targetFiles.Length, "Target compreesd file must be present");
}
/// <param name="args">Command-line args</param>
/// <param name="processorCount">Number of processors on the current machine.</param>
/// <param name="memory">Size of RAM on the current machine.</param>
public static void Run(string[] args, int processorCount, ulong memory)
{
var compressor = new FileCompressor(processorCount, GetUsedMemory(memory));
try
{
var command = CommandParser.Parse(args);
switch (command)
{
case CompressCommand compressCommand:
compressor.Compress(compressCommand.InputFilePath, compressCommand.OutputFilePath);
break;
case DecompressCommand decompressCommand:
compressor.Decompress(decompressCommand.InputFilePath, decompressCommand.OutputFilePath);
break;
default:
throw new InvalidOperationException(Messages.CommandNotSupported);
}
System.Console.WriteLine(Messages.SuccessOperation);
}
catch (Exception ex)
{
System.Console.WriteLine(ex.Message);
}
System.Console.ReadKey();
}
public void TestMethodCreate()
{
FileCompressor.Compress(@"E:\Alex\a932.jpg");
bool zipExist = File.Exists(@"E:\Alex\a932.jpg");
Assert.AreEqual(true, zipExist);
}
internal static void CopyFile(string source, string destination, bool overwrite, bool compressDestination)
{
string tempArchivePath = string.Empty;
try
{
if (compressDestination)
{
string archiveEntryName = Path.GetFileNameWithoutExtension(destination);
FileCompressor.Compress(source, archiveEntryName, out tempArchivePath);
source = tempArchivePath;
}
CopyFileParameters copyParam = new CopyFileParameters()
{
Source = source,
Destination = destination,
Overwrite = true
};
Utility.PerformIOWithRetries(
CopyFileWorker,
(object)copyParam);
}
finally
{
if (false == string.IsNullOrEmpty(tempArchivePath))
{
Utility.PerformIOWithRetries(
() => { FabricFile.Delete(tempArchivePath); });
}
}
}
public XElement WriteObject(IPackageFileContent value)
{
FileCompressor fileCompressor = new FileCompressor();
return(new XElement(ELEMENT_FILE,
new XAttribute(ATTR_FILE_PATH, value.FilePath),
Convert.ToBase64String(fileCompressor.Compress(value.FileContent))));
}
// Archiwizuje wskazany po ścieżce plik
public void ArchiveFile(string path)
{
if (!IsOpen)
{
throw new Exception("Archiwum nieotwarte");
}
if (!ArchiveKey.IsDerived)
{
throw new Exception("Klucz niewyprowadzony");
}
if (!File.Exists(path))
{
throw new Exception("Plik pod wskazaną lokalizacją nie istnieje");
}
FileData file = FileData.Create(path);
string plaintextName = file.NameStr;
if (!FileNameUnique(plaintextName))
{
throw new FileNamingException(plaintextName);
}
if (IsCompressed)
{
FileCompressor.CompressFile(file);
}
FileEncryptor.EncryptFile(file, ArchiveKey);
SQLiteCommand command = new SQLiteCommand(Connection);
command.CommandText = "INSERT INTO FileData(name_l, name, content_l, content, iv) VALUES" +
"(" +
"'" + Convert.ToString(file.NameLength) + "', " +
"@Name, " +
"'" + Convert.ToString(file.ContentLength) + "', " +
"@Content, " +
"@IV" +
")";
command.Parameters.Add(new SQLiteParameter("Name", file.Name));
command.Parameters.Add(new SQLiteParameter("Content", file.Content));
command.Parameters.Add(new SQLiteParameter("IV", file.IV));
command.ExecuteNonQuery();
command.CommandText = "SELECT last_insert_rowid();";
long lastRowId = (long)command.ExecuteScalar();
int fileId = (int)lastRowId;
// Przechowanie pary id_pliku-nazwa_pliku
IdFileNamePairs.Add(fileId, plaintextName);
}
// Wypakowuje z archiwum wskazany po identyfikatorze plik do wskazanego po ścieżce katalogu
public void Extract(int id, string destPath)
{
if (!IsOpen)
{
throw new Exception("Archiwum nieotwarte");
}
if (!ArchiveKey.IsDerived)
{
throw new Exception("Klucz niewyprowadzony");
}
SQLiteCommand command = new SQLiteCommand(Connection);
command.CommandText = "SELECT name_l, length(name), name, content_l, length(content), content, iv FROM FileData WHERE id='" + Convert.ToString(id) + "';";
SQLiteDataReader reader = command.ExecuteReader();
FileData file = null;
while (reader.Read())
{
int nameLength = reader.GetInt32(0);
int nameBlobLength = reader.GetInt32(1);
byte[] fileName = new byte[nameBlobLength];
reader.GetBytes(2, 0, fileName, 0, nameBlobLength);
int contentLength = reader.GetInt32(3);
int contentBlobLength = reader.GetInt32(4);
byte[] content = new byte[contentBlobLength];
reader.GetBytes(5, 0, content, 0, contentBlobLength);
byte[] iv = new byte[EncryptionKey.BlockLength];
reader.GetBytes(6, 0, iv, 0, EncryptionKey.BlockLength);
file = FileData.Create(nameLength, fileName, contentLength, content, iv);
}
if (file != null)
{
FileEncryptor.DecryptFile(file, ArchiveKey);
if (IsCompressed)
{
FileCompressor.DecompressFile(file);
}
File.WriteAllBytes(Path.Combine(destPath, file.NameStr), file.Content);
}
else
{
throw new Exception(string.Format("Nie udało się odczytać danych pliku o id={0}", id));
}
}
//Compresses a file and returns the new path
protected async Task <string> CompressFileAsync(string path)
{
try
{
return(await Task.Run(() => FileCompressor.Compress(new FileInfo(Path.GetFullPath(path))).FullName));
}
catch (Exception ex)
{
throw new Exception(ex.Message, ex);
}
}
public void TrailingSlashDoesNotChangeResults()
{
var success = FileCompressor.Compress(k_SourceDirectory, "artifacts1.sbpGz");
Assert.IsTrue(success);
success = FileCompressor.Compress(k_SourceDirectory + "/", "artifacts2.sbpGz");
Assert.IsTrue(success);
FileAssert.AreEqual("artifacts1.sbpGz", "artifacts2.sbpGz");
File.Delete("artifacts1.sbpGz");
File.Delete("artifacts2.sbpGz");
}
public void ShouldCompress()
{
// Arrange
var fileService = new Mock <IFileService>();
fileService.Setup(m => m.GetSize(It.IsAny <string>())).Returns(12);
var file = new byte[12] {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
};
var reader = new InmemoryReader(file);
var compressor = new Mocks.PassThroughCompressor();
var output = new byte[44];
var writer = new InmemoryWriter(output);
var fileCompressor = new FileCompressor(
new FileCompressorOptions {
Size = 10
},
new CompressionInfoGenerator(
new CompressorInfoGeenratorOptions {
Path = @"C:\file.txt", Size = 10
},
fileService.Object),
reader,
compressor,
writer);
// Act
fileCompressor.Execute();
// Assert
Assert.Equal(
new byte[16 + 10 + 16 + 2]
{
/* compressedSize */ 10, 0, 0, 0, /* source.offset */ 0, 0, 0, 0, 0, 0, 0, 0, /* source.size */ 10, 0, 0, 0,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
/* compressedSize */ 2, 0, 0, 0, /* source.offset */ 10, 0, 0, 0, 0, 0, 0, 0, /* source.size */ 2, 0, 0, 0,
11, 12
}, output);
}
public void CompressAndDecompressCanHandleSubdirectories()
{
var targetDirectory = k_SourceDirectory + "2";
var success = FileCompressor.Compress(k_SourceDirectory, "artifacts.sbpGz");
Assert.IsTrue(success);
success = FileCompressor.Decompress("artifacts.sbpGz", targetDirectory);
Assert.IsTrue(success);
for (int i = 0; i < k_SourceFiles.Length; i++)
{
var sourcePath = NormalizePath(k_SourceDirectory + k_SourceFiles[i]);
var targetPath = NormalizePath(targetDirectory + k_SourceFiles[i]);
FileAssert.Exists(targetPath);
FileAssert.AreEqual(sourcePath, targetPath);
}
File.Delete("artifacts.sbpGz");
Directory.Delete(targetDirectory, true);
}
/// <summary>
/// Decompress the received file or folder
/// <para>Returns the path of the folder or file.</para>
/// </summary>
/// <param name="path"></param>
public string Decompress(string path)
{
FileInfo info = new FileInfo(path);
if (info.Extension == FileCompressor.Extension)
{
FileInfo decompressedFile = FileCompressor.Decompress(info);
File.Delete(info.FullName);
return(decompressedFile.Name);
}
if (info.Extension == FolderCompressor.Extension)
{
DirectoryInfo extractedFolder = new DirectoryInfo(info.FullName.Remove(info.FullName.Length - info.Extension.Length));
FolderCompressor.Extract(info.FullName, extractedFolder.FullName);
File.Delete(info.FullName);
return(extractedFolder.Name);
}
return(info.Name);
}
public void AddNewItem1(FileSystemEventArgs e, DataItems items, IDataSerializer dataSerializer, string targetFolder)
{
try
{
string sourceFilePath = e.FullPath;
//string archiveName = FileCompressor.GetArchiveFileName(e.FullPath);
string fileName = Path.GetFileName(sourceFilePath);
string destFilePath = targetFolder;
Task.Run(
() =>
{
if (FileMover.Move(sourceFilePath, destFilePath))
{
FileCompressor.Compress(destFilePath);
dataSerializer.Store(items);
}
}
);
}
catch (Exception ex)
{
_log.Error(ex);
}
}
public void RunHandlingOfWorkingItems(
Action <FileItem> addItemToListAction,
Action storeItems,
Action <Exception, FileItem> errorAction,
bool exitAfterFinishAllItems = true)
{
bool isRunning;
lock (_lockObj)
{
isRunning = IsRunning;
}
if (isRunning)
{
return;
}
lock (_lockObj)
{
IsRunning = true;
OnStateChanged(IsRunning);
_log.Debug($"MainWorker will be started for {_filesQueue.Count} files. Cansellation pending {CancellationPending}");
}
Task.Run(
() =>
{
_log.Debug($" MainWorker started");
Thread.CurrentThread.Name = "Main worker";
while (!CancellationPending)
{
if (_filesQueue.TryDequeue(out FileSystemEventArgs moveItem))
{
_log.Debug($" Do work for {moveItem.Name}");
FileItem fileItem = new FileItem {
Name = moveItem.Name, CopyDate = DateTime.Now
};
//_log.Debug($"New item detected {e.FullPath}. Zip and move it");
DateTime creationTime = GetFileCreationTime(moveItem.FullPath);
//item.TimeStamp = creationTime.ToShortDateString() + " " + creationTime.ToShortTimeString();
fileItem.TimeStamp = creationTime;
try
{
addItemToListAction?.Invoke(fileItem);
string sourceFilePath = moveItem.FullPath;
string archiveName = FileCompressor.GetArchiveFileName(sourceFilePath);
string fileName = Path.GetFileName(archiveName);
string destFilePath = Path.Combine(TargetFolder, fileName);
CompressFile(sourceFilePath);
MoveFile(archiveName, destFilePath);
fileItem.TargetFilePath = destFilePath;
DeleteFile(sourceFilePath);
storeItems?.Invoke();
//_dataSerializer.Store(_items);
}
catch (Exception ex)
{
_log.Error(ex);
if (errorAction != null)
{
errorAction(ex, fileItem);
}
//Not good as exception could come in theory but we need to store error state
storeItems?.Invoke();
}
}
else
{
if (exitAfterFinishAllItems && _filesQueue.Count == 0)
{
_log.Debug($" MainWorker - break by zero items");
break;
}
}
Thread.Sleep(50);
}
lock (_lockObj)
{
IsRunning = false;
OnStateChanged(IsRunning);
}
_log.Debug($"MainWorker stopped. CancellationPending={CancellationPending}");
});
}
public virtual void CompressFile(string sourceFilePath)
{
FileCompressor.Compress(sourceFilePath);
}
/////////*************************************DEPRECATED************************************************//////////////
/// <summary>
/// Creates an array of bytes. DEPRECATED.
/// <para>This method sets the location of where the file will be copied to.
/// It also gets all bytes in a file and writes it to fileData byte array</para>
/// </summary>
/// <param name="fileLocation"></param>
/// <param name="remoteSaveLocation"></param>
/// <param name="encryptFile"></param>
/// <param name="compressFile"></param>
/// <returns>Byte[]</returns>
protected byte[] CreateByteFile(string fileLocation, string remoteSaveLocation, bool encryptFile, bool compressFile)
{
try
{
FileInfo fileToSend = new FileInfo(fileLocation);
byte[] header = null;
if (compressFile)
{
fileToSend = FileCompressor.Compress(new FileInfo(fileLocation));
remoteSaveLocation += FileCompressor.Extension;
}
//Check if the file and header have to be encrypted.
if (encryptFile)
{
Encrypter.FileEncrypt(fileToSend.FullName);
//Delete compressed file
if (compressFile)
{
File.Delete(fileToSend.FullName);
}
fileToSend = new FileInfo(fileToSend.FullName + Encrypter.Extension);
remoteSaveLocation += Encrypter.Extension;
byte[] encryptedPrefix = Encoding.UTF8.GetBytes("ENCRYPTED_");
byte[] encryptedHeader = Encrypter.EncryptStringToBytes(remoteSaveLocation);
header = new byte[encryptedHeader.Length + encryptedPrefix.Length];
encryptedPrefix.CopyTo(header, 0);
encryptedHeader.CopyTo(header, 10);
}
else
{
header = Encoding.UTF8.GetBytes(remoteSaveLocation);
}
//Message
byte[] messageData = File.ReadAllBytes(fileToSend.FullName);
byte[] headerBytes = header;
byte[] headerLen = BitConverter.GetBytes(headerBytes.Length);
byte[] messageLength = BitConverter.GetBytes(messageData.Length);
//Delete encrypted file after it has been read.
if (encryptFile)
{
File.Delete(fileToSend.FullName);
}
var data = new byte[4 + 4 + headerBytes.Length + messageData.Length];
messageLength.CopyTo(data, 0);
headerLen.CopyTo(data, 4);
headerBytes.CopyTo(data, 8);
messageData.CopyTo(data, 8 + headerBytes.Length);
return(data);
}
catch (Exception ex)
{
throw new Exception(ex.ToString());
}
}
