public void FileIsArchived_GivenPopulatedTable_SaysYes()
{
var archiveModel = new ArchivedPictureModel
{
Archive = new ArchiveModel
{
ArchiveId = Guid.NewGuid().ToString(),
Status = HttpStatusCode.OK,
Location = "here",
PostedTimestamp = DateTime.MinValue,
Metadata = "idk some metadata"
},
Picture = new PictureModel
{
AbsolutePath = "/dev/null/pictures/picture.jpg",
CatalogRelativePath = "pictures/picture.jpg",
FileName = "picture.jpg",
FileId = Guid.NewGuid(),
ImageId = Guid.NewGuid(),
LibraryCount = 3
}
};
_sut.MarkFileAsArchived(archiveModel, "vault", "region", "catalog", "catalogUniqueId");
var result = _sut.FileIsArchived(archiveModel.Picture.FileId);
Assert.True(result);
}
public void MarkFileAsArchived(ArchivedPictureModel model, string vaultName, string vaultRegion, string catalogFilename, string catalogUniqueId)
{
if (_testMode)
{
return;
}
var vaultId = GetOrCreateVaultId(vaultName, vaultRegion);
var catalogId = GetOrCreateCatalogId(catalogFilename, catalogUniqueId);
using (var insert = _db.CreateCommand())
{
insert.CommandText = InsertPicture;
insert.Parameters.Add(new SqliteParameter("$catalogId", catalogId));
insert.Parameters.Add(new SqliteParameter("$vaultId", vaultId));
insert.Parameters.Add(new SqliteParameter("$fileAbsolutePath", model.Picture.AbsolutePath));
insert.Parameters.Add(new SqliteParameter("$fileCatalogPath", model.Picture.CatalogRelativePath));
insert.Parameters.Add(new SqliteParameter("$fileName", model.Picture.FileName));
insert.Parameters.Add(new SqliteParameter("$fileId", model.Picture.FileId));
insert.Parameters.Add(new SqliteParameter("$imageId", model.Picture.ImageId));
insert.Parameters.Add(new SqliteParameter("$glacierArchiveId", model.Archive.ArchiveId));
insert.Parameters.Add(new SqliteParameter("$glacierHttpStatusCode", (int)model.Archive.Status));
insert.Parameters.Add(new SqliteParameter("$glacierLocation", model.Archive.Location));
insert.Parameters.Add(new SqliteParameter("$glacierMetadata", model.Archive.Metadata));
insert.Parameters.Add(new SqliteParameter("$glacierTimestamp", model.Archive.PostedTimestamp));
insert.ExecuteNonQuery();
}
}
public async Task <AvalancheRunResult> Run()
{
var result = new AvalancheRunResult();
var catalogId = _lightroom.GetCatalogId();
var allPictures = _lightroom.GetAllPictures();
var filteredPictures = allPictures
.GroupBy(a => a.FileId)
.Select(a => a.First())
.Where(a => a.LibraryCount > 0 &&
!_avalanche.FileIsArchived(a.FileId))
.ToList();
_logger.LogInformation("Backing up {0} images", filteredPictures.Count);
await _glacier.AssertVaultExists(_parameters.Glacier.VaultName);
// todo: parallelize this if it improves uploads (not sure if it will)
var index = 0;
foreach (var f in filteredPictures)
{
var currentPath = Path.Combine(f.AbsolutePath, f.FileName);
_logger.LogInformation("Archiving {0} of {1}: {2}", ++index, filteredPictures.Count, currentPath);
// Retry for transient transport failures
ArchivedPictureModel archive = null;
for (var i = 0; i < RetryCount; ++i)
{
try
{
archive = await _glacier.SaveImage(f, _parameters.Glacier.VaultName);
break;
}
catch (Exception ex)
{
_logger.LogError("Error persisting file", ex);
continue;
}
}
if (archive == null)
{
_logger.LogError("Failed 3 times to persist {0}, giving up", currentPath);
result.Failures.Add(f);
continue;
}
_avalanche.MarkFileAsArchived(archive, _parameters.Glacier.VaultName, _parameters.Glacier.Region, _parameters.Avalanche.CatalogFilePath, catalogId.ToString());
result.Successes.Add(f);
}
_logger.LogInformation("Done");
return(result);
}
public void MarkFileAsArchived(ArchivedPictureModel model, string vaultName, string vaultRegion, string catalogFilename, string catalogUniqueId)
{
var vaultId = GetOrCreateVaultId(vaultName, vaultRegion);
var catalogId = GetOrCreateCatalogId(catalogFilename, catalogUniqueId);
using (var connection = OpenNewConnection())
{
var insert = connection.CreateCommand();
insert.CommandText = @"
INSERT INTO Pictures
(
CatalogId,
VaultId,
FileAbsolutePath,
FileCatalogPath,
FileName,
FileId,
ImageId,
GlacierArchiveId,
GlacierHttpStatusCode,
GlacierLocation,
GlacierMetadata,
GlacierTimestamp
)
SELECT
$catalogId,
$vaultId,
$fileAbsolutePath,
$fileCatalogPath,
$fileName,
$fileId,
$imageId,
$glacierArchiveId,
$glacierHttpStatusCode,
$glacierLocation,
$glacierMetadata,
$glacierTimestamp
";
insert.Parameters.Add(new SQLiteParameter("$catalogId", catalogId));
insert.Parameters.Add(new SQLiteParameter("$vaultid", vaultId));
insert.Parameters.Add(new SQLiteParameter("$fileAbsolutePath", model.Picture.AbsolutePath));
insert.Parameters.Add(new SQLiteParameter("$fileCatalogPath", model.Picture.CatalogRelativePath));
insert.Parameters.Add(new SQLiteParameter("$fileName", model.Picture.FileName));
insert.Parameters.Add(new SQLiteParameter("$fileId", model.Picture.FileId));
insert.Parameters.Add(new SQLiteParameter("$imageId", model.Picture.ImageId));
insert.Parameters.Add(new SQLiteParameter("$glacierArchiveId", model.Archive.ArchiveId));
insert.Parameters.Add(new SQLiteParameter("$glacierHttpStatusCode", (int)model.Archive.Status));
insert.Parameters.Add(new SQLiteParameter("$glacierLocation", model.Archive.Location));
insert.Parameters.Add(new SQLiteParameter("$glacierMetadata", model.Archive.Metadata));
insert.Parameters.Add(new SQLiteParameter("$glacierTimestamp", model.Archive.PostedTimestamp));
insert.ExecuteNonQuery();
}
}
private async Task ArchivePicture(PictureModel picture, RunState state)
{
var currentPath = Path.Combine(picture.AbsolutePath, picture.FileName);
_logger.LogInformation("Archiving {0} of {1}: {2}", ++state.Index, state.FilteredPictureCount, currentPath);
// Retry for transient transport failures
ArchivedPictureModel archive = null;
for (var i = 0; i < RetryCount; ++i)
{
try
{
using (var glacier = _glacierFactory.CreateWrapper())
{
archive = await glacier.Item.SaveImage(picture, _parameters.Glacier.VaultName);
}
break;
}
catch (Exception ex)
{
_logger.LogError("Error persisting file", ex);
continue;
}
}
if (archive == null)
{
_logger.LogError("Failed 3 times to persist {0}, giving up", currentPath);
state.Result.Failures.Add(picture);
return;
}
using (var avalanche = _avalancheFactory.CreateWrapper())
{
avalanche.Item.MarkFileAsArchived(archive, _parameters.Glacier.VaultName,
_parameters.Glacier.Region, _parameters.Avalanche.CatalogFilePath,
state.CatalogId.ToString());
}
state.Result.Successes.Add(picture);
}
public static void Main(string[] args)
{
var parameters = ExecutionParameters.GetParametersFromArgs(args);
if (parameters == null)
{
_log.Fatal("No config file could be found in the default location (my documents) and none was specified in the parameters.");
Environment.Exit(0);
}
var errors = parameters.GetValidationErrors();
if (errors.Any())
{
_log.Fatal("Configuration/parameter errors occurredt:");
foreach (var e in errors)
{
_log.Fatal(e);
}
Environment.Exit(0);
}
using (var insomniac = new Insomniac())
{
var lightroomRepo = new LightroomRepository(parameters.Avalanche.CatalongFilePath);
var avalancheRepo = new AvalancheRepository(parameters.Avalanche.AvalancheFilePath);
var gateway = new GlacierGateway(parameters.Glacier);
var catalogId = lightroomRepo.GetUniqueId();
var allPictures = lightroomRepo.GetAllPictures();
var filteredPictures = allPictures.Where(a => a.LibraryCount > 0 && !avalancheRepo.FileIsArchived(a.FileId)).ToList();
_log.InfoFormat("Backing up {0} images", filteredPictures.Count);
var index = 0;
foreach (var f in filteredPictures)
{
_log.InfoFormat("Archiving {0}/{1}: {2}", ++index, filteredPictures.Count, Path.Combine(f.AbsolutePath, f.FileName));
// Try three times
ArchivedPictureModel archive = null;
for (var i = 0; i < 3; ++i)
{
try
{
archive = gateway.SaveImage(f, parameters.Glacier.VaultName);
}
catch (Exception ex)
{
_log.ErrorFormat("Error!!! {0}", ex);
continue;
}
break;
}
if (archive == null)
{
continue;
}
avalancheRepo.MarkFileAsArchived(archive, parameters.Glacier.VaultName, parameters.Glacier.Region, parameters.Avalanche.CatalongFilePath, catalogId.ToString());
}
_log.Info("Done");
}
}
