Exemple #1
0
        public static void Main(string[] args)
        {
            log4net.Config.XmlConfigurator.Configure();
            log.Info("Initializing and connecting to AWS...");

            s3 = AWSClientFactory.CreateAmazonS3Client(RegionEndpoint.USWest1);
            indexer = new FileIndexer("Files");
            indexer.Index();

            s3indexer = new S3Indexer(Settings.Default.BucketName, Settings.Default.FolderName, "S3Tmp", s3);
            s3indexer.Index();

            log.Info("Comparing local index and remote index.");

            var filesToUpload = (from filePair in indexer.FileIndex where !s3indexer.HashedFiles.ContainsKey(filePair.Key) || !s3indexer.HashedFiles[filePair.Key].SequenceEqual(filePair.Value) select filePair.Key).ToList();
            var filesToDelete = (from filePair in s3indexer.HashedFiles where !indexer.FileIndex.ContainsKey(filePair.Key) select filePair.Key).ToList();

            foreach(var fileDelete in filesToDelete)
            {
                log.Debug("Deleting file "+fileDelete);
                s3.DeleteObject(new DeleteObjectRequest()
                                    {
                                        BucketName = Settings.Default.BucketName,
                                        Key = Settings.Default.FolderName + "/" + fileDelete
                                    });
            }

            foreach(var fileUpload in filesToUpload)
            {
                log.Debug("Uploading file "+fileUpload);
                s3.PutObject(new PutObjectRequest()
                                 {
                                     BucketName = Settings.Default.BucketName,
                                     Key = Settings.Default.FolderName + "/" + fileUpload,
                                     AutoCloseStream = true,
                                     InputStream = new FileStream("Files/" + fileUpload, FileMode.Open)
                                 });
            }

            log.Info("Re-indexing files...");

            using (MemoryStream stream = new MemoryStream())
            {
                Serializer.Serialize(stream, indexer.FileIndex);
                stream.Position = 0;
                s3.PutObject(new PutObjectRequest()
                {
                    BucketName = Settings.Default.BucketName,
                    Key = Settings.Default.FolderName + "/" + "index.mhash",
                    InputStream = stream
                });
            }

            log.Info("Done!");

            Console.Read();
        }
Exemple #2
0
        /// <summary>
        /// Go!
        /// </summary>
        public void Index()
        {
            HashedFiles.Clear();
            //Find the hash object
            GetObjectResponse s3hash = null;
            log.Debug("Checking for index.mhash...");
            try
            {
                s3hash = s3.GetObject(new GetObjectRequest() {BucketName = bucketName, Key = folderName+"/"+"index.mhash"});
            }catch(Exception ex)
            {
            }
            if(s3hash != null)
            {
                log.Debug("Hash file found, deserializing!");
                Dictionary<string, byte[]> downloadedIndex;
                using(var stream = s3hash.ResponseStream)
                {
                    downloadedIndex = Serializer.Deserialize<Dictionary<string, byte[]>>(stream);
                }
                HashedFiles = downloadedIndex;
                log.Debug("Index deserialized, files: "+HashedFiles.Count);
            }else
            {
                log.Debug("Downloading all files to build a hash.");
                var files =
                    s3.ListObjects(new ListObjectsRequest()
                                       {BucketName = bucketName, Delimiter = "/", Prefix = folderName + "/"});

                foreach(var obj in files.S3Objects)
                {
                    if (obj.Key.Contains("index.mhash")) continue;
                    var folderDir = downloadPath + "/" + Path.GetDirectoryName(obj.Key);
                    Directory.CreateDirectory(folderDir);
                    using (GetObjectResponse response = s3.GetObject(new GetObjectRequest() { BucketName = bucketName, Key = obj.Key }))
                    {
                        using(Stream stream = response.ResponseStream)
                        {
                            using(FileStream file = new FileStream(downloadPath+"/"+obj.Key, FileMode.Create))
                            {
                                log.Debug("Downloading file "+obj.Key);
                                stream.CopyTo(file);
                            }
                        }
                    }
                }
                log.Debug("All files downloaded, hashing files...");
                var fileIndexer = new FileIndexer(downloadPath + "/" + folderName + "/");
                fileIndexer.Index();
                var index = fileIndexer.FileIndex;
                HashedFiles = index;
                log.Debug("All files hashed, serializing and uploading hash file...");
                using(MemoryStream stream = new MemoryStream())
                {
                    Serializer.Serialize(stream, HashedFiles);
                    stream.Position = 0;
                    s3.PutObject(new PutObjectRequest()
                                     {
                                         BucketName = bucketName,
                                         Key = folderName + "/" + "index.mhash",
                                         InputStream = stream
                                     });
                }
                log.Debug("Finished uploading new hash file!");
            }
        }