private void processTask(BackupTask task)
{
Logger.Debug("StorageThread:processTask:BackupTask");
//generate list of files
LinkedList <FileInChunk> allFiles = new LinkedList <FileInChunk>();
recursivelyFillFileQueue(ref allFiles, task.Path, "");
int chunkID = 0;
while (true)
{ //while need more chunks
Logger.Debug2("StorageThread:processTask outer loop");
if (allFiles.Count() == 0)
{
break;
}
string oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
TarOutputStream tarOutputStream = newTarOutputStream(oFilename);
Chunk chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
long size = 0;
string relPath, fullPath;
while (true)
{ //while need more files in chunk
//Logger.Debug2("StorageThread:processTask inner loop");
if (allFiles.Count() == 0)
{
tarOutputStream.Close();
break;
}
FileInChunk fileInChunk = allFiles.First();
relPath = fileInChunk.path;
fullPath = task.Path + '\\' + relPath;
if (Directory.Exists(fullPath))
{
//Logger.Debug2("StorageThread:processTask inner loop if 1");
TarEntry entry2 = TarEntry.CreateTarEntry(relPath);
//entry2.Name = relPath;
entry2.TarHeader.TypeFlag = TarHeader.LF_DIR;
tarOutputStream.PutNextEntry(entry2);
chunk.AddLast(fileInChunk);
size += 512;
allFiles.RemoveFirst();
continue;
}
Stream inputStream = File.OpenRead(fullPath);
long s = inputStream.Length;
long toWrite = s;
//add file header size to chunk size
size += 512;
TarEntry entry = TarEntry.CreateTarEntry(relPath);
if (size + s < CHUNK_SIZE)
{
//Logger.Debug2("StorageThread:processTask inner loop if 2");
entry.Size = s;
tarOutputStream.PutNextEntry(entry);
byte[] buffer = new byte[32 * 1024];
int totalRead = 0;
while (true)
{
int numRead = inputStream.Read(buffer, 0, buffer.Length);
if (numRead <= 0)
{
break;
}
totalRead += numRead;
tarOutputStream.Write(buffer, 0, numRead);
}
tarOutputStream.CloseEntry();
chunk.AddLast(fileInChunk);
allFiles.RemoveFirst();
size += s;
//if size is not a multiple of 512, round it up to next multiple of 512
if (size % 512 != 0)
{
size += 512 - (size % 512);
}
}
else
{ //file is too big to fit in chunk in entirety
//Logger.Debug2("StorageThread:processTask inner loop else 2");
long totalFileRead = 0;
int partLength = CHUNK_SIZE - (int)size;
createSplitFileInChunks(ref allFiles, partLength, s);
fileInChunk = allFiles.First();
entry.Size = partLength;
tarOutputStream.PutNextEntry(entry);
byte[] buffer = new byte[32 * 1024];
while (true)
{ //while we have more chunks from this file to write
//Logger.Debug2("StorageThread:processTask inner-inner loop");
while (true)
{
int numRead = inputStream.Read(buffer, 0, Math.Min(buffer.Length, partLength));
if (numRead <= 0)
{
break;
}
partLength -= numRead;
size += numRead;
totalFileRead += numRead;
tarOutputStream.Write(buffer, 0, numRead);
}
tarOutputStream.CloseEntry();
partLength = CHUNK_SIZE - 512;
chunk.AddLast(fileInChunk);
allFiles.RemoveFirst();
if (allFiles.Count == 0)
{
break;
}
if (allFiles.First.Value.fileStart != 0)
{
//Logger.Debug2("StorageThread:processTask inner-inner loop if");
tarOutputStream.Close();
lock (_lock)
{
backupResults.Enqueue(chunk);
}
chunkID++;
oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
tarOutputStream = newTarOutputStream(oFilename);
chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
entry.Size = Math.Min(CHUNK_SIZE - 512, s - totalFileRead);
tarOutputStream.PutNextEntry(entry);
size = 512;
}
else
{
//Logger.Debug2("StorageThread:processTask inner-inner loop else");
break;
}
}
}
if (size >= CHUNK_SIZE)
{
//Logger.Debug2("StorageThread:processTask inner loop if 3");
tarOutputStream.Close();
break;
}
}
lock (_lock)
{
backupResults.Enqueue(chunk);
}
chunkID++;
}
}
private void processTask(BackupTask task)
{
Logger.Debug("StorageThread:processTask:BackupTask");
//generate list of files
LinkedList<FileInChunk> allFiles = new LinkedList<FileInChunk>();
recursivelyFillFileQueue(ref allFiles, task.Path, "");
int chunkID = 0;
while (true)
{ //while need more chunks
Logger.Debug2("StorageThread:processTask outer loop");
if (allFiles.Count() == 0) break;
string oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
TarOutputStream tarOutputStream = newTarOutputStream(oFilename);
Chunk chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
long size = 0;
string relPath, fullPath;
while (true)
{ //while need more files in chunk
//Logger.Debug2("StorageThread:processTask inner loop");
if (allFiles.Count() == 0)
{
tarOutputStream.Close();
break;
}
FileInChunk fileInChunk = allFiles.First();
relPath = fileInChunk.path;
fullPath = task.Path + '\\' + relPath;
if (Directory.Exists(fullPath))
{
//Logger.Debug2("StorageThread:processTask inner loop if 1");
TarEntry entry2 = TarEntry.CreateTarEntry(relPath);
//entry2.Name = relPath;
entry2.TarHeader.TypeFlag = TarHeader.LF_DIR;
tarOutputStream.PutNextEntry(entry2);
chunk.AddLast(fileInChunk);
size += 512;
allFiles.RemoveFirst();
continue;
}
Stream inputStream = File.OpenRead(fullPath);
long s = inputStream.Length;
long toWrite = s;
//add file header size to chunk size
size += 512;
TarEntry entry = TarEntry.CreateTarEntry(relPath);
if (size + s < CHUNK_SIZE)
{
//Logger.Debug2("StorageThread:processTask inner loop if 2");
entry.Size = s;
tarOutputStream.PutNextEntry(entry);
byte[] buffer = new byte[32 * 1024];
int totalRead = 0;
while (true)
{
int numRead = inputStream.Read(buffer, 0, buffer.Length);
if (numRead <= 0)
{
break;
}
totalRead += numRead;
tarOutputStream.Write(buffer, 0, numRead);
}
tarOutputStream.CloseEntry();
chunk.AddLast(fileInChunk);
allFiles.RemoveFirst();
size += s;
//if size is not a multiple of 512, round it up to next multiple of 512
if (size % 512 != 0)
{
size += 512 - (size % 512);
}
}
else
{ //file is too big to fit in chunk in entirety
//Logger.Debug2("StorageThread:processTask inner loop else 2");
long totalFileRead = 0;
int partLength = CHUNK_SIZE - (int)size;
createSplitFileInChunks(ref allFiles, partLength, s);
fileInChunk = allFiles.First();
entry.Size = partLength;
tarOutputStream.PutNextEntry(entry);
byte[] buffer = new byte[32 * 1024];
while (true)
{ //while we have more chunks from this file to write
//Logger.Debug2("StorageThread:processTask inner-inner loop");
while (true)
{
int numRead = inputStream.Read(buffer, 0, Math.Min(buffer.Length, partLength));
if (numRead <= 0)
{
break;
}
partLength -= numRead;
size += numRead;
totalFileRead += numRead;
tarOutputStream.Write(buffer, 0, numRead);
}
tarOutputStream.CloseEntry();
partLength = CHUNK_SIZE - 512;
chunk.AddLast(fileInChunk);
allFiles.RemoveFirst();
if (allFiles.Count == 0) break;
if (allFiles.First.Value.fileStart != 0)
{
//Logger.Debug2("StorageThread:processTask inner-inner loop if");
tarOutputStream.Close();
lock (_lock)
{
backupResults.Enqueue(chunk);
}
chunkID++;
oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
tarOutputStream = newTarOutputStream(oFilename);
chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
entry.Size = Math.Min(CHUNK_SIZE - 512, s - totalFileRead);
tarOutputStream.PutNextEntry(entry);
size = 512;
}
else
{
//Logger.Debug2("StorageThread:processTask inner-inner loop else");
break;
}
}
}
if (size >= CHUNK_SIZE)
{
//Logger.Debug2("StorageThread:processTask inner loop if 3");
tarOutputStream.Close();
break;
}
}
lock (_lock)
{
backupResults.Enqueue(chunk);
}
chunkID++;
}
}
/// <summary>
/// tests the basic Tar and GZip functionality.
/// </summary>
static void testTarGZip()
{
Console.WriteLine("starting testTarBzip2");
byte[] b = {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1};
Guid guid = new Guid(b);
string tempPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\temp\\scratch";
string tempPath2 = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\temp\\scratch 2";
StorageThread st = new StorageThread(tempPath, guid);
BackupTask task = new BackupTask(tempPath, tempPath2, 123, 0);
st.EnqueueStorageTask(task);
Console.WriteLine("queued task");
int x = 0;
while (st.IsWorking())
{
x++;
Console.WriteLine("waiting for thread to finish." + x);
Thread.Sleep(1000);
}
Console.WriteLine("thread finished.");
st.RequestStop();
Console.WriteLine("requested thread terminate.");
while (st.IsAlive())
{
x++;
Console.WriteLine("waiting for thread to die. " + x);
Thread.Sleep(1000);
}
Console.WriteLine("thread is dead.");
Console.WriteLine("number of chunks: " + st.NumChunks());
while (st.NumChunks() > 0)
{
Chunk c = st.DequeueChunk();
Console.WriteLine(c);
}
Console.WriteLine("press a key to continue");
Console.ReadKey();
}
public void StartBackup(string[] directories)
{
foreach (string dir in directories)
{
if (Directory.Exists(dir))
{
BackupTask bt = new BackupTask(dir, Node.GetTemporaryDirectory(), 123, 0);
//if (storageThread == null) Logger.Log("shit");
storageThread.EnqueueStorageTask(bt);
}
else
{
Logger.Warn("EchoBackupService:StartBackup Directory '" + dir + "' does not exist.");
}
}
}