Exemplo n.º 1
0
 private static ISync CreateSyncForAccount(MetaDataService.AccountInfo account, string containerName, Logger log, SynchronizeDirection synchronizeDirection = SynchronizeDirection.Upload)
 {
     // Create Synchronizer
     if (account.location == "None")
     {
         return(null);
     }
     else
     {
         LocationInfo Li           = new LocationInfo(account.accountName, account.accountKey, SyncFactory.GetSynchronizerType(account.location));
         ISync        synchronizer = SyncFactory.Instance.CreateSynchronizer(Li, containerName, log, synchronizeDirection);
         return(synchronizer);
     }
 }
Exemplo n.º 2
0
 protected void CreateSync(SynchronizeDirection dir)
 {
     // Create Synchronizer
     if (account.location == "None")
     {
         synchronizer = null;
     }
     else if (streamtype == StreamFactory.StreamSecurityType.Secure)
     {
         LocationInfo Li = new LocationInfo(account.accountName, account.accountKey, SyncFactory.GetSynchronizerType(account.location));
         synchronizer = SyncFactory.Instance.CreateSynchronizer(Li, streamid.ToString().Replace('/', '-').ToLower() + "-" + seq_num, logger, dir, streamcompressiontype, this.StreamChunkSizeForUpload, this.StreamThreadPoolSize, EncryptionType.AES, acl_md.encKey, acl_md.IV);
         synchronizer.SetLocalSource(targetDir);
         synchronizer.SetIndexFileName(IndexFileName);
         synchronizer.SetDataFileName(DataLogFileName);
     }
     else
     {
         LocationInfo Li = new LocationInfo(account.accountName, account.accountKey, SyncFactory.GetSynchronizerType(account.location));
         synchronizer = SyncFactory.Instance.CreateSynchronizer(Li, streamid.ToString().Replace('/', '-').ToLower() + "-" + seq_num, logger, dir, streamcompressiontype, this.StreamChunkSizeForUpload, this.StreamThreadPoolSize);
         synchronizer.SetLocalSource(targetDir);
         synchronizer.SetIndexFileName(IndexFileName);
         synchronizer.SetDataFileName(DataLogFileName);
     }
 }
Exemplo n.º 3
0
        public byte[] UploadFileAsChunks(string filePath)
        {
            string           blobName;
            List <ChunkInfo> chunkList_cloud = new List <ChunkInfo>();; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
            List <ChunkInfo> chunkList_local;                           // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)

            try
            {
                // First check if chunkMD exists on the server or not
                if (logger != null)
                {
                    logger.Log("Start Synchronizer Check Blob Exists");
                }
                blobName = Path.GetFileName(filePath);
                CloudBlockBlob chunkMDblockBlob = GetBlockBlobReference(ChunkMetadataBlobPrefix + blobName);
                bool           blobExists       = BlockBlobExists(chunkMDblockBlob);
                if (logger != null)
                {
                    logger.Log("End Synchronizer Check Blob Exists");
                }

                if (blobExists)
                {
                    if (logger != null)
                    {
                        logger.Log("Start Synchronizer Fill Remote ChunkList");
                    }
                    // Fill chunkList
                    FileMD fileMD = JsonConvert.DeserializeObject <FileMD>(chunkMDblockBlob.DownloadText());
                    StaticChunkSize = fileMD.StaticChunkSize;
                    // Get chunklist at cloud in memory locally
                    chunkList_cloud = fileMD.ChunkList;
                    if (logger != null)
                    {
                        logger.Log("End Synchronizer Fill Remote ChunkList");
                    }
                    chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType);
                    chunkEncryptionType  = SyncFactory.GetEncryptionType(fileMD.encryptionType);
                }

                if (logger != null)
                {
                    logger.Log("Start Synchronizer Fill Local ChunkList");
                }
                StaticChunk staticChunker = new StaticChunk(StaticChunkSize);
                // Store local chunkList in memory
                //long start = DateTime.Now.Ticks;
                chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface
                //long end = DateTime.Now.Ticks;
                // Console.WriteLine("time taken : " + (double)((double)(end - start) / (double)10000000));

                if (logger != null)
                {
                    logger.Log("End Synchronizer Fill Local ChunkList");
                }
                // structuredLog("I", "Number of chunks locally: " + chunkList_local.Count);

                // Figure out the changes
                if (logger != null)
                {
                    logger.Log("Start Synchronizer ChunkList Compare");
                }
                List <ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud);
                // structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count);
                // structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count);
                if (logger != null)
                {
                    logger.Log("End Synchronizer ChunkList Compare");
                }

                if (logger != null)
                {
                    logger.Log("Start Synchronizer Upload Multiple Chunks");
                }
                UploadChunkList(ref chunkList_toUpload, filePath, blobName);
                if (logger != null)
                {
                    logger.Log("End Synchronizer Upload Multiple Chunks");
                }

                // structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count);

                if (logger != null)
                {
                    logger.Log("Start Synchronizer Commit BlockList");
                }
                // Commit the ordered blocklist
                if (chunkList_toUpload.Count > 0)// write new metadata, and putblocklist() if we uploaded some chunks
                {
                    CloudBlockBlob blockBlob         = GetBlockBlobReference(blobName);
                    List <int>     blockIDCommitList = GetBlockIDList(ref chunkList_local, ref chunkList_cloud, ref chunkList_toUpload);
                    long           startt            = DateTime.Now.Ticks;
                    blockBlob.PutBlockList(EncodeBlockList(blockIDCommitList), GetBlobRequestOptions());
                    long endt = DateTime.Now.Ticks;
                    if (logger != null)
                    {
                        logger.Log("PUTBLOCK LIST : " + (double)((double)(endt - startt) / (double)10000000));
                    }
                }
                if (logger != null)
                {
                    logger.Log("End Synchronizer Commit BlockList");
                }

                // Upload the local chunklist to the cloud
                if (logger != null)
                {
                    logger.Log("Start Synchronizer ChunkList Upload");
                }
                string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter());

                if (chunkList_toUpload.Count > 0 || chunkList_local.Count == 0) //upload new chunk list only if we uploaded some new chunks, or if this is a new stream, with no data in stream.dat
                {
                    chunkMDblockBlob.UploadText(json);
                }

                SHA1   sha1 = new SHA1CryptoServiceProvider();
                byte[] ret  = sha1.ComputeHash(Encoding.ASCII.GetBytes(json));
                if (logger != null)
                {
                    logger.Log("End Synchronizer ChunkList Upload");
                }
                return(ret);
            }
            catch (Exception e)
            {
                structuredLog("E", " . UploadFileAsChunks: " + e);
                return(null);
            }
        }
Exemplo n.º 4
0
        public byte[] UploadFileAsChunks(string filePath)
        {
            string           s3objectName;
            List <ChunkInfo> chunkList_cloud = new List <ChunkInfo>();; // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)
            List <ChunkInfo> chunkList_local;                           // list of chunk indexed by chunk-index (e.g. 0, 1, 2,....)

            try
            {
                if (logger != null)
                {
                    logger.Log("Start Synchronizer Check Blob Exists");
                }
                s3objectName = Path.GetFileName(filePath);
                bool s3ObjectExists = S3ObjectExists(ChunkMetadataObjectPrefix + s3objectName);
                if (logger != null)
                {
                    logger.Log("End Synchronizer Check Blob Exists");
                }

                if (s3ObjectExists)
                {
                    if (logger != null)
                    {
                        logger.Log("Start Synchronizer Fill Remote ChunkList");
                    }
                    GetObjectRequest request = new GetObjectRequest();
                    request.WithBucketName(bucketName);
                    request.WithKey(ChunkMetadataObjectPrefix + s3objectName);
                    GetObjectResponse response = amazonS3Client.GetObject(request);
                    StreamReader      reader   = new StreamReader(response.ResponseStream);

                    string chunkMD_JSON = reader.ReadToEnd();

                    FileMD fileMD = JsonConvert.DeserializeObject <FileMD>(chunkMD_JSON);
                    StaticChunkSize = fileMD.StaticChunkSize;
                    chunkList_cloud = fileMD.ChunkList;
                    if (logger != null)
                    {
                        logger.Log("End Synchronizer Fill Remote ChunkList");
                    }
                    chunkCompressionType = SyncFactory.GetCompressionType(fileMD.compressionType);
                    chunkEncryptionType  = SyncFactory.GetEncryptionType(fileMD.encryptionType);
                }

                if (logger != null)
                {
                    logger.Log("Start Synchronizer Fill Local ChunkList");
                }
                StaticChunk staticChunker = new StaticChunk(StaticChunkSize);
                chunkList_local = staticChunker.GetCurrentChunkList(filePath); // if doing other class that implements the IChunk interface
                // structuredLog("I", "Number of chunks locally: " + chunkList_local.Count);
                if (logger != null)
                {
                    logger.Log("End Synchronizer Fill Local ChunkList");
                }

                if (logger != null)
                {
                    logger.Log("Start Synchronizer ChunkList Compare");
                }
                List <ChunkInfo> chunkList_toUpload = staticChunker.GetUploadChunkList(chunkList_local, chunkList_cloud);
                // structuredLog("I", "Number of chunks on cloud blob: " + chunkList_cloud.Count);
                // structuredLog("I", "Number of chunks to be uploaded: " + chunkList_toUpload.Count);
                if (logger != null)
                {
                    logger.Log("End Synchronizer ChunkList Compare");
                }

                if (logger != null)
                {
                    logger.Log("Start Synchronizer Upload Multiple Chunks");
                }
                UploadChunkList(ref chunkList_toUpload, filePath, s3objectName);
                if (logger != null)
                {
                    logger.Log("End Synchronizer Upload Multiple Chunks");
                }

                // structuredLog("I", "Number of chunks uploaded: " + chunkList_toUpload.Count);

                if (logger != null)
                {
                    logger.Log("Start Synchronizer ChunkList Upload");
                }
                string json = JsonConvert.SerializeObject(new FileMD(StaticChunkSize, chunkList_local, SyncFactory.GetCompressionTypeAsString(this.chunkCompressionType), SyncFactory.GetEncryptionTypeAsString(this.chunkEncryptionType)), new KeyValuePairConverter());

                if (chunkList_toUpload.Count > 0) //upload new chunk list only if we uploaded some new chunks
                {
                    UploadStringToS3Object(ChunkMetadataObjectPrefix + s3objectName, json);
                }

                SHA1   sha1 = new SHA1CryptoServiceProvider();
                byte[] ret  = sha1.ComputeHash(Encoding.ASCII.GetBytes(json));
                if (logger != null)
                {
                    logger.Log("End Synchronizer ChunkList Upload");
                }
                return(ret);
            }
            catch (Exception e)
            {
                structuredLog("E", " . UploadFileAsChunks: " + e);
                return(null);
            }
        }