public static void Main(string[] args)
{
log4net.Config.XmlConfigurator.Configure();
log.Info("Initializing and connecting to AWS...");
s3 = AWSClientFactory.CreateAmazonS3Client(RegionEndpoint.USWest1);
indexer = new FileIndexer("Files");
indexer.Index();
s3indexer = new S3Indexer(Settings.Default.BucketName, Settings.Default.FolderName, "S3Tmp", s3);
s3indexer.Index();
log.Info("Comparing local index and remote index.");
var filesToUpload = (from filePair in indexer.FileIndex where !s3indexer.HashedFiles.ContainsKey(filePair.Key) || !s3indexer.HashedFiles[filePair.Key].SequenceEqual(filePair.Value) select filePair.Key).ToList();
var filesToDelete = (from filePair in s3indexer.HashedFiles where !indexer.FileIndex.ContainsKey(filePair.Key) select filePair.Key).ToList();
foreach(var fileDelete in filesToDelete)
{
log.Debug("Deleting file "+fileDelete);
s3.DeleteObject(new DeleteObjectRequest()
{
BucketName = Settings.Default.BucketName,
Key = Settings.Default.FolderName + "/" + fileDelete
});
}
foreach(var fileUpload in filesToUpload)
{
log.Debug("Uploading file "+fileUpload);
s3.PutObject(new PutObjectRequest()
{
BucketName = Settings.Default.BucketName,
Key = Settings.Default.FolderName + "/" + fileUpload,
AutoCloseStream = true,
InputStream = new FileStream("Files/" + fileUpload, FileMode.Open)
});
}
log.Info("Re-indexing files...");
using (MemoryStream stream = new MemoryStream())
{
Serializer.Serialize(stream, indexer.FileIndex);
stream.Position = 0;
s3.PutObject(new PutObjectRequest()
{
BucketName = Settings.Default.BucketName,
Key = Settings.Default.FolderName + "/" + "index.mhash",
InputStream = stream
});
}
log.Info("Done!");
Console.Read();
}
/// <summary>
/// Go!
/// </summary>
public void Index()
{
HashedFiles.Clear();
//Find the hash object
GetObjectResponse s3hash = null;
log.Debug("Checking for index.mhash...");
try
{
s3hash = s3.GetObject(new GetObjectRequest() {BucketName = bucketName, Key = folderName+"/"+"index.mhash"});
}catch(Exception ex)
{
}
if(s3hash != null)
{
log.Debug("Hash file found, deserializing!");
Dictionary<string, byte[]> downloadedIndex;
using(var stream = s3hash.ResponseStream)
{
downloadedIndex = Serializer.Deserialize<Dictionary<string, byte[]>>(stream);
}
HashedFiles = downloadedIndex;
log.Debug("Index deserialized, files: "+HashedFiles.Count);
}else
{
log.Debug("Downloading all files to build a hash.");
var files =
s3.ListObjects(new ListObjectsRequest()
{BucketName = bucketName, Delimiter = "/", Prefix = folderName + "/"});
foreach(var obj in files.S3Objects)
{
if (obj.Key.Contains("index.mhash")) continue;
var folderDir = downloadPath + "/" + Path.GetDirectoryName(obj.Key);
Directory.CreateDirectory(folderDir);
using (GetObjectResponse response = s3.GetObject(new GetObjectRequest() { BucketName = bucketName, Key = obj.Key }))
{
using(Stream stream = response.ResponseStream)
{
using(FileStream file = new FileStream(downloadPath+"/"+obj.Key, FileMode.Create))
{
log.Debug("Downloading file "+obj.Key);
stream.CopyTo(file);
}
}
}
}
log.Debug("All files downloaded, hashing files...");
var fileIndexer = new FileIndexer(downloadPath + "/" + folderName + "/");
fileIndexer.Index();
var index = fileIndexer.FileIndex;
HashedFiles = index;
log.Debug("All files hashed, serializing and uploading hash file...");
using(MemoryStream stream = new MemoryStream())
{
Serializer.Serialize(stream, HashedFiles);
stream.Position = 0;
s3.PutObject(new PutObjectRequest()
{
BucketName = bucketName,
Key = folderName + "/" + "index.mhash",
InputStream = stream
});
}
log.Debug("Finished uploading new hash file!");
}
}