Example #1
0
        private void ArchiveDirectory_Click(object sender, System.EventArgs e)
        {
            if (textBoxAWSVaultName.Text == String.Empty || textBoxAWSDynamoTableName.Text == String.Empty)
            {
                throw new Exception("textBoxAWSVaultName.Text ==String.Empty || textBoxAWSDynamoTableName.Text==String.Empty");
            }

            //AWS Requiers more than three charaters for a table name
            if (textBoxAWSDynamoTableName.Text.Length < 3)
            {
                throw new Exception("AWS Dynamo Requiers more than three charaters for a table name");
            }

            bool vaultExists = GalicierHelper.checkVault(textBoxAWSVaultName.Text);

            if (!vaultExists)
            {
                GalicierHelper.createVault(textBoxAWSVaultName.Text);
            }

            bool tableOK = DynamoDBHelper.DynamoTableCheck(textBoxAWSDynamoTableName.Text);

            if (!tableOK)
            {
                throw new Exception("AWS  Dynamo Table is not OK.  Check AWS credentials.");
            }

            folderBrowserDialog1.ShowDialog();
            String directory = folderBrowserDialog1.SelectedPath;

            DirectoryInfo di = new System.IO.DirectoryInfo(directory);

            string[] files = Directory.GetFiles(directory, "*.*", SearchOption.AllDirectories);

            DateTime dt = new DateTime();

            dt = DateTime.Now;
            String dtString = dt.ToString();

            dtString = dtString.Replace("/", "_").Replace(":", "").Replace(" ", "");

            String fileList = "fileList" + dtString + ".csv";

            StreamWriter sr = new System.IO.StreamWriter(fileList);

            for (int i = 0; i < files.Length; i++)
            {
                FileInfo fio = new System.IO.FileInfo(files[i]);
                sr.WriteLine("," + fio.Directory + "," + fio.Name);
            }
            sr.Flush();
            sr.Close();

            BulkTransfer(fileList);
        }
Example #2
0
        private void Copy_Click(object sender, EventArgs e)
        {
            if (textBoxAWSVaultName.Text == String.Empty || textBoxAWSDynamoTableName.Text == String.Empty)
            {
                throw new Exception("textBoxAWSVaultName.Text ==String.Empty || textBoxAWSDynamoTableName.Text==String.Empty");
            }

            //AWS Requiers more than three charaters for a table name
            if (textBoxAWSDynamoTableName.Text.Length < 3)
            {
                throw new Exception("AWS Dynamo Requiers more than three charaters for a table name");
            }

            bool vaultExists = GalicierHelper.checkVault(textBoxAWSVaultName.Text);

            if (!vaultExists)
            {
                GalicierHelper.createVault(textBoxAWSVaultName.Text);
            }

            DynamoDBHelper.DynamoTableCheck(textBoxAWSDynamoTableName.Text);

            BulkTransfer(fileLogFileDialog.FileName);
        }
Example #3
0
        private void BulkTransfer(String fileList)
        {
            FileInfo fioi = new FileInfo(fileList);

            String fileLogName = fioi.Name.Replace(".csv", "");

            DateTime dt = new DateTime();

            dt = DateTime.Now;
            String dtString = dt.ToString();

            dtString = dtString.Replace("/", "_").Replace(":", "").Replace(" ", "");

            String transactionLogName = "GlacierTransationLog_" + fileLogName + "_" + dtString + ".csv";

            glacierTransactionLog = new StreamWriter(transactionLogName);
            log.Info("Making ArchiveLog :" + transactionLogName);

            glacierTransactionLog.WriteLine("AWSVaultName" + "," + "ArchiveName" + "," + "ArchiveID" + "," + "SHA256 Tree Checksum Received" + "," + "SHA256 Tree Checksum Sent");

            String tarFileIdLogName = "Tar_FileID_" + fileLogName + "_" + dtString + ".csv";

            TarFileIdLog = new StreamWriter(tarFileIdLogName);
            log.Info("Making ArchiveLog :" + tarFileIdLogName);

            StreamWriter ResubmitJobFileIds = new StreamWriter(fileLogName + "AWSGlacierCopyResubmit" + dtString + ".csv");

            ArrayList serverName      = new ArrayList();
            ArrayList directoryName   = new ArrayList();
            ArrayList archiveFileList = new ArrayList();

            ArrayList fileLogLines = new ArrayList();
            String    headers      = String.Empty;

            try
            {
                StreamReader sr = new StreamReader(fileList);

                headers = sr.ReadLine();

                ResubmitJobFileIds.WriteLine(headers);

                String line = String.Empty;

                while ((line = sr.ReadLine()) != null)
                {
                    fileLogLines.Add(line);

                    string[] fields = line.Split(new char[] { ',' });

                    serverName.Add(fields[0]);
                    directoryName.Add(fields[1]);
                    archiveFileList.Add(fields[2]);
                }
            }
            catch (Exception ex)
            {
                log.Error(ex.ToString());
            }

            try
            {
                int partNumber = 0;

                createTARAndTxLog(partNumber);

                //We compress, tar and transfer in chunkc of size maxTARGBThreshold
                for (int i = 0; i < archiveFileList.Count; i++)
                {
                    bool lastFile = i == archiveFileList.Count - 1?true:false;

                    String sourceDirectory = String.Empty;
                    if (serverName[i] != String.Empty)
                    {
                        sourceDirectory = "\\\\" + serverName[i] + "\\" + directoryName[i] + "\\";
                    }
                    else
                    {
                        sourceDirectory = directoryName[i] + "\\";
                    }

                    System.IO.Directory.SetCurrentDirectory(System.AppDomain.CurrentDomain.BaseDirectory);
                    string sinkDirectory = Directory.GetCurrentDirectory();

                    long bytesTarred = 0;

                    FileInfo fiobt = new System.IO.FileInfo(archiveName);
                    bytesTarred = fiobt.Length;

                    try
                    {
                        ArrayList awsDynamoFields = new ArrayList();
                        ArrayList awsDynamoVals   = new ArrayList();

                        archiveFile(sinkDirectory, sourceDirectory, (string)archiveFileList[i], awsDynamoFields, awsDynamoVals);

                        awsDynamoFields.Add("ArchiveName");
                        awsDynamoVals.Add(archiveName);

                        string[] fieldsA         = (string[])awsDynamoFields.ToArray(typeof(string));
                        string[] valsA           = (string[])awsDynamoVals.ToArray(typeof(string));
                        String   dynamoTableName = textBoxAWSDynamoTableName.Text;
                        DynamoDBHelper.MakeDynamoEntry(dynamoTableName, fieldsA, valsA);

                        TarFileIdLog.WriteLine(archiveName + "," + fileLogLines[i]);
                        TarFileIdLog.Flush();
                    }
                    catch (Exception ex)
                    {
                        ResubmitJobFileIds.WriteLine(fileLogLines[i]);
                        ResubmitJobFileIds.Flush();
                        log.Error(ex.ToString());
                    }


                    if (bytesTarred > (Math.Pow(10, 9) * maxTARGBThreshold) || lastFile)
                    {
                        glacierTransactionLog.Flush();

                        tarLog.Flush();

                        tarLog.Close();

                        TarEntry entry = TarEntry.CreateEntryFromFile(tarLogName);
                        archive.WriteEntry(entry, true);

                        archive.Close();

                        //Copy TarFiles and Logs To Scratch Place
                        DirectoryInfo di = new DirectoryInfo(sinkDirectory);

                        FileInfo[] zFiles = di.GetFiles("*.z", SearchOption.TopDirectoryOnly);

                        for (int j = 0; j < zFiles.Length; j++)
                        {
                            FileInfo fio = zFiles[j];
                            fio.Delete();
                        }

                        //Write Out Remaining Log in case of problem
                        StreamWriter remainingFileLog = new StreamWriter(fileLogName + "_Recover_" + "_" + dtString + ".csv");
                        remainingFileLog.WriteLine(headers);

                        for (int k = i; k < serverName.Count; k++)
                        {
                            remainingFileLog.WriteLine(fileLogLines[k]);
                        }
                        remainingFileLog.Flush();
                        remainingFileLog.Close();

                        //Transfer Tar Here
                        log.Info("Calculating SHA256TreeHash on Tar");
                        FileStream inputFile = File.Open(archiveName, FileMode.Open, FileAccess.Read);
                        byte[]     treeHash  = ComputeSHA256TreeHash(inputFile);
                        String     Checksum  = BitConverter.ToString(treeHash).Replace("-", "").ToLower();
                        log.Info("Sending to Glacier:  " + archiveName + "With Checksum " + Checksum);

                        ArchiveUploadMultipartParallel aruo = new ArchiveUploadMultipartParallel(awsVaultName);

                        AWSArchiveResult ar = aruo.UploadFile(archiveName, "", archiveName);

                        glacierTransactionLog.WriteLine(awsVaultName + "," + archiveName + "," + ar.ArchiveID + "," + ar.Checksum + "," + Checksum);

                        glacierTransactionLog.Flush();

                        //Make Dynamo Entry for Archive
                        ArrayList awsDynamoFields = new ArrayList();
                        ArrayList awsDynamoVals   = new ArrayList();

                        awsDynamoFields.Add("ArchiveName");
                        awsDynamoVals.Add(archiveName);

                        awsDynamoFields.Add("ArchiveId");
                        awsDynamoVals.Add(ar.ArchiveID);

                        awsDynamoFields.Add("ArchiveCheckSum");
                        awsDynamoVals.Add(ar.Checksum);

                        string[] fieldsA         = (string[])awsDynamoFields.ToArray(typeof(string));
                        string[] valsA           = (string[])awsDynamoVals.ToArray(typeof(string));
                        String   dynamoTableName = textBoxAWSDynamoTableName.Text;
                        DynamoDBHelper.MakeDynamoEntry(dynamoTableName, fieldsA, valsA);

                        if (!lastFile)
                        {
                            partNumber += 1;

                            createTARAndTxLog(partNumber);
                        }
                    }
                }
                ////Close and send the last TarBall
                //{
                //    glacierTransactionLog.Flush();

                //    tarLog.Flush();

                //    tarLog.Close();

                //    TarEntry entry = TarEntry.CreateEntryFromFile(tarLogName);
                //    archive.WriteEntry(entry, true);
                //    archive.Close();

                //    //Transfer Tar Here
                //    log.Info("Calculating SHA256TreeHash on Tar");
                //    FileStream inputFile = File.Open(archiveName, FileMode.Open, FileAccess.Read);
                //    byte[] treeHash = ComputeSHA256TreeHash(inputFile);
                //    String Checksum = BitConverter.ToString(treeHash).Replace("-", "").ToLower();
                //    log.Info("Sending to Glacier:  " + archiveName + "With Checksum " + Checksum);

                //    ArchiveUploadMultipartParallel aruo = new ArchiveUploadMultipartParallel(awsVaultName);

                //    AWSArchiveResult ar = aruo.UploadFile(archiveName, "", archiveName);

                //    glacierTransactionLog.WriteLine(awsVaultName + "," + archiveName + "," + ar.ArchiveID + "," + ar.Checksum + "," + Checksum);
                //    glacierTransactionLog.Flush();

                //    MessageBox.Show("AWS Glacier Transation Complete ");
                //    //Application.Exit();
                //}
            }
            catch (Exception ex)
            {
                log.Error(ex.ToString());
            }
        }