Example #1
0
        private void processTask(BackupTask task)
        {
            Logger.Debug("StorageThread:processTask:BackupTask");
            //generate list of files
            LinkedList <FileInChunk> allFiles = new LinkedList <FileInChunk>();

            recursivelyFillFileQueue(ref allFiles, task.Path, "");
            int chunkID = 0;

            while (true)
            { //while need more chunks
                Logger.Debug2("StorageThread:processTask outer loop");
                if (allFiles.Count() == 0)
                {
                    break;
                }
                string          oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
                TarOutputStream tarOutputStream = newTarOutputStream(oFilename);
                Chunk           chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
                long            size = 0;
                string          relPath, fullPath;
                while (true)
                { //while need more files in chunk
                    //Logger.Debug2("StorageThread:processTask inner loop");
                    if (allFiles.Count() == 0)
                    {
                        tarOutputStream.Close();
                        break;
                    }
                    FileInChunk fileInChunk = allFiles.First();
                    relPath  = fileInChunk.path;
                    fullPath = task.Path + '\\' + relPath;
                    if (Directory.Exists(fullPath))
                    {
                        //Logger.Debug2("StorageThread:processTask inner loop if 1");
                        TarEntry entry2 = TarEntry.CreateTarEntry(relPath);
                        //entry2.Name = relPath;
                        entry2.TarHeader.TypeFlag = TarHeader.LF_DIR;
                        tarOutputStream.PutNextEntry(entry2);
                        chunk.AddLast(fileInChunk);
                        size += 512;
                        allFiles.RemoveFirst();
                        continue;
                    }
                    Stream inputStream = File.OpenRead(fullPath);
                    long   s           = inputStream.Length;
                    long   toWrite     = s;
                    //add file header size to chunk size
                    size += 512;
                    TarEntry entry = TarEntry.CreateTarEntry(relPath);
                    if (size + s < CHUNK_SIZE)
                    {
                        //Logger.Debug2("StorageThread:processTask inner loop if 2");
                        entry.Size = s;
                        tarOutputStream.PutNextEntry(entry);
                        byte[] buffer    = new byte[32 * 1024];
                        int    totalRead = 0;
                        while (true)
                        {
                            int numRead = inputStream.Read(buffer, 0, buffer.Length);
                            if (numRead <= 0)
                            {
                                break;
                            }
                            totalRead += numRead;
                            tarOutputStream.Write(buffer, 0, numRead);
                        }
                        tarOutputStream.CloseEntry();
                        chunk.AddLast(fileInChunk);
                        allFiles.RemoveFirst();
                        size += s;
                        //if size is not a multiple of 512, round it up to next multiple of 512
                        if (size % 512 != 0)
                        {
                            size += 512 - (size % 512);
                        }
                    }
                    else
                    { //file is too big to fit in chunk in entirety
                        //Logger.Debug2("StorageThread:processTask inner loop else 2");
                        long totalFileRead = 0;
                        int  partLength    = CHUNK_SIZE - (int)size;
                        createSplitFileInChunks(ref allFiles, partLength, s);
                        fileInChunk = allFiles.First();
                        entry.Size  = partLength;
                        tarOutputStream.PutNextEntry(entry);
                        byte[] buffer = new byte[32 * 1024];
                        while (true)
                        { //while we have more chunks from this file to write
                            //Logger.Debug2("StorageThread:processTask inner-inner loop");
                            while (true)
                            {
                                int numRead = inputStream.Read(buffer, 0, Math.Min(buffer.Length, partLength));
                                if (numRead <= 0)
                                {
                                    break;
                                }
                                partLength    -= numRead;
                                size          += numRead;
                                totalFileRead += numRead;
                                tarOutputStream.Write(buffer, 0, numRead);
                            }
                            tarOutputStream.CloseEntry();
                            partLength = CHUNK_SIZE - 512;
                            chunk.AddLast(fileInChunk);
                            allFiles.RemoveFirst();
                            if (allFiles.Count == 0)
                            {
                                break;
                            }
                            if (allFiles.First.Value.fileStart != 0)
                            {
                                //Logger.Debug2("StorageThread:processTask inner-inner loop if");
                                tarOutputStream.Close();
                                lock (_lock)
                                {
                                    backupResults.Enqueue(chunk);
                                }
                                chunkID++;
                                oFilename       = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
                                tarOutputStream = newTarOutputStream(oFilename);
                                chunk           = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
                                entry.Size      = Math.Min(CHUNK_SIZE - 512, s - totalFileRead);
                                tarOutputStream.PutNextEntry(entry);
                                size = 512;
                            }
                            else
                            {
                                //Logger.Debug2("StorageThread:processTask inner-inner loop else");
                                break;
                            }
                        }
                    }
                    if (size >= CHUNK_SIZE)
                    {
                        //Logger.Debug2("StorageThread:processTask inner loop if 3");
                        tarOutputStream.Close();
                        break;
                    }
                }
                lock (_lock)
                {
                    backupResults.Enqueue(chunk);
                }
                chunkID++;
            }
        }
Example #2
0
 private void processTask(BackupTask task)
 {
     Logger.Debug("StorageThread:processTask:BackupTask");
     //generate list of files
     LinkedList<FileInChunk> allFiles = new LinkedList<FileInChunk>();
     recursivelyFillFileQueue(ref allFiles, task.Path, "");
     int chunkID = 0;
     while (true)
     { //while need more chunks
         Logger.Debug2("StorageThread:processTask outer loop");
         if (allFiles.Count() == 0) break;
         string oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
         TarOutputStream tarOutputStream = newTarOutputStream(oFilename);
         Chunk chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
         long size = 0;
         string relPath, fullPath;
         while (true)
         { //while need more files in chunk
             //Logger.Debug2("StorageThread:processTask inner loop");
             if (allFiles.Count() == 0)
             {
                 tarOutputStream.Close();
                 break;
             }
             FileInChunk fileInChunk = allFiles.First();
             relPath = fileInChunk.path;
             fullPath = task.Path + '\\' + relPath;
             if (Directory.Exists(fullPath))
             {
                 //Logger.Debug2("StorageThread:processTask inner loop if 1");
                 TarEntry entry2 = TarEntry.CreateTarEntry(relPath);
                 //entry2.Name = relPath;
                 entry2.TarHeader.TypeFlag = TarHeader.LF_DIR;
                 tarOutputStream.PutNextEntry(entry2);
                 chunk.AddLast(fileInChunk);
                 size += 512;
                 allFiles.RemoveFirst();
                 continue;
             }
             Stream inputStream = File.OpenRead(fullPath);
             long s = inputStream.Length;
             long toWrite = s;
             //add file header size to chunk size
             size += 512;
             TarEntry entry = TarEntry.CreateTarEntry(relPath);
             if (size + s < CHUNK_SIZE)
             {
                 //Logger.Debug2("StorageThread:processTask inner loop if 2");
                 entry.Size = s;
                 tarOutputStream.PutNextEntry(entry);
                 byte[] buffer = new byte[32 * 1024];
                 int totalRead = 0;
                 while (true)
                 {
                     int numRead = inputStream.Read(buffer, 0, buffer.Length);
                     if (numRead <= 0)
                     {
                         break;
                     }
                     totalRead += numRead;
                     tarOutputStream.Write(buffer, 0, numRead);
                 }
                 tarOutputStream.CloseEntry();
                 chunk.AddLast(fileInChunk);
                 allFiles.RemoveFirst();
                 size += s;
                 //if size is not a multiple of 512, round it up to next multiple of 512
                 if (size % 512 != 0)
                 {
                     size += 512 - (size % 512);
                 }
             }
             else
             { //file is too big to fit in chunk in entirety
                 //Logger.Debug2("StorageThread:processTask inner loop else 2");
                 long totalFileRead = 0;
                 int partLength = CHUNK_SIZE - (int)size;
                 createSplitFileInChunks(ref allFiles, partLength, s);
                 fileInChunk = allFiles.First();
                 entry.Size = partLength;
                 tarOutputStream.PutNextEntry(entry);
                 byte[] buffer = new byte[32 * 1024];
                 while (true)
                 { //while we have more chunks from this file to write
                     //Logger.Debug2("StorageThread:processTask inner-inner loop");
                     while (true)
                     {
                         int numRead = inputStream.Read(buffer, 0, Math.Min(buffer.Length, partLength));
                         if (numRead <= 0)
                         {
                             break;
                         }
                         partLength -= numRead;
                         size += numRead;
                         totalFileRead += numRead;
                         tarOutputStream.Write(buffer, 0, numRead);
                     }
                     tarOutputStream.CloseEntry();
                     partLength = CHUNK_SIZE - 512;
                     chunk.AddLast(fileInChunk);
                     allFiles.RemoveFirst();
                     if (allFiles.Count == 0) break;
                     if (allFiles.First.Value.fileStart != 0)
                     {
                         //Logger.Debug2("StorageThread:processTask inner-inner loop if");
                         tarOutputStream.Close();
                         lock (_lock)
                         {
                             backupResults.Enqueue(chunk);
                         }
                         chunkID++;
                         oFilename = task.TempPath + '\\' + guid + '_' + task.BackupID + '_' + chunkID + ".tgz";
                         tarOutputStream = newTarOutputStream(oFilename);
                         chunk = new Chunk(task.BackupID, chunkID, task.Path, oFilename);
                         entry.Size = Math.Min(CHUNK_SIZE - 512, s - totalFileRead);
                         tarOutputStream.PutNextEntry(entry);
                         size = 512;
                     }
                     else
                     {
                         //Logger.Debug2("StorageThread:processTask inner-inner loop else");
                         break;
                     }
                 }
             }
             if (size >= CHUNK_SIZE)
             {
                 //Logger.Debug2("StorageThread:processTask inner loop if 3");
                 tarOutputStream.Close();
                 break;
             }
         }
         lock (_lock)
         {
             backupResults.Enqueue(chunk);
         }
         chunkID++;
     }
 }
Example #3
0
        /// <summary>
        /// tests the basic Tar and GZip functionality.
        /// </summary>
        static void testTarGZip()
        {
            Console.WriteLine("starting testTarBzip2");
            byte[] b = {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1};
            Guid guid = new Guid(b);
            string tempPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\temp\\scratch";
            string tempPath2 = Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\temp\\scratch 2";
            StorageThread st = new StorageThread(tempPath, guid);
            BackupTask task = new BackupTask(tempPath, tempPath2, 123, 0);
            st.EnqueueStorageTask(task);
            Console.WriteLine("queued task");
            int x = 0;
            while (st.IsWorking())
            {
                x++;
                Console.WriteLine("waiting for thread to finish." + x);
                Thread.Sleep(1000);
            }
            Console.WriteLine("thread finished.");
            st.RequestStop();
            Console.WriteLine("requested thread terminate.");
            while (st.IsAlive())
            {
                x++;
                Console.WriteLine("waiting for thread to die. " + x);
                Thread.Sleep(1000);
            }
            Console.WriteLine("thread is dead.");
            Console.WriteLine("number of chunks: " + st.NumChunks());
            while (st.NumChunks() > 0)
            {
                Chunk c = st.DequeueChunk();
                Console.WriteLine(c);
            }

            Console.WriteLine("press a key to continue");
            Console.ReadKey();
        }
Example #4
0
 public void StartBackup(string[] directories)
 {
     foreach (string dir in directories)
     {
         if (Directory.Exists(dir))
         {
             BackupTask bt = new BackupTask(dir, Node.GetTemporaryDirectory(), 123, 0);
             //if (storageThread == null) Logger.Log("shit");
             storageThread.EnqueueStorageTask(bt);
         }
         else
         {
             Logger.Warn("EchoBackupService:StartBackup Directory '" + dir + "' does not exist.");
         }
     }
 }