private BlobJob GetBlobJob(BlobItem blob)
{
var itemCount = Interlocked.Increment(ref TotalItems);
if (itemCount % 5000 == 0)
{
// set progress JobChar for next console update
AddJobChar('.');
CheckPrintConsole();
}
if (blob == null)
{
Interlocked.Increment(ref IgnoredItems);
return(null);
}
Interlocked.Add(ref TotalSize, blob.Size);
var localFileName = blob.GetLocalFileName();
var bJob = new BlobJob(this, blob, Path.Combine(_localPath, localFileName));
if (localFileName == null)
{
throw new NullReferenceException();
}
lock (ExpectedLocalFilesLock)
ExpectedLocalFiles.Add(localFileName);
bJob.FileInfo = _sqlLite.GetFileInfo(blob, bJob.LocalFilePath);
bJob.AddDownloaded = AddDownloaded;
return(bJob);
}
internal void UpdateFromAzure(BlobItem blob)
{
LocalName = blob.GetLocalFileName();
RemPath = blob.Uri.AbsolutePath;
LastModifiedTime = blob.LastModifiedUtc.UtcDateTime;
Size = blob.Size;
MD5 = blob.MD5;
}
public ILocalFileInfo GetFileInfo(BlobItem blob, string localFilename = null)
{
ILocalFileInfo lfi = localFilename == null ? null : new LocalFileInfoDisk(localFilename);
var fi = GetFileInfo(lfi, blob.GetLocalFileName());
if (fi != null)
{
return(fi);
}
// create new instance
((LocalFileInfoDisk)lfi)?.GetMd5();
fi = new FileInfo(this, lfi, blob);
// make sure we insert item when there is nothing found
ExecuteNonQuery("INSERT INTO " + SQL_TABLENAME +
" (LocalName, RemPath, LastModifiedTime, Size, AzureMD5, LastDownloadedTime, DeleteDetectedTime)" +
" VALUES (@1, @2, @3, @4, @5, @6, @7)",
fi.LocalName, fi.RemPath, fi.LastModifiedTime, fi.Size, fi.MD5, fi.LastDownloadedTime, fi.DeleteDetectedTime);
return(fi);
}
public BlobJob(Backup bakParent, BlobItem blob, string localFilePath)
{
Bak = bakParent;
Blob = blob;
LocalFilePath = localFilePath;
}
public Backup PrepareJob(string accountName, string accountKey)
{
var localContainerPath = Path.Combine(_localPath, _containerName);
Directory.CreateDirectory(localContainerPath);
var localDir = new DirectoryInfo(localContainerPath);
bool?downloadOk = null;
try
{
_sqlLite.BeginTransaction();
BlobItem.BlobEnumerator(_containerName, accountName, accountKey).Select(GetBlobJob).Where(j => j != null).ForAll(bJob =>
{
var blob = bJob.Blob;
var file = bJob.FileInfo;
try
{
if (!file.Exists || bJob.ForceDownloadMissing())
{
bJob.NeedsJob = JobType.New;
bJob.SqlFileInfo.UpdateFromAzure(blob);
if (bJob.HandleWellKnownBlob())
{
Interlocked.Increment(ref IgnoredItems);
}
else
{
BlobJobQueue.AddDone(bJob);
}
Interlocked.Increment(ref NewItems);
Interlocked.Add(ref NewItemsSize, blob.Size);
}
else if (!file.IsSame(blob))
{
/*//
* if (file.Size != blob.Size ||
* file.MD5 != blob.MD5 ||
* file.LastModifiedTimeUtc == blob.LastModifiedTimeUtc) // debug stuff
* Console.WriteLine($"\n** Seen {file.DiffString(blob)} for {bJob.LocalFilePath}");
* //*/
bJob.NeedsJob = JobType.Modified;
BlobJobQueue.AddDone(bJob);
Interlocked.Increment(ref ModifiedItems);
Interlocked.Add(ref ModifiedItemsSize, blob.Size);
}
else
{
Interlocked.Increment(ref UpToDateItems);
}
}
catch (Exception ex)
{
downloadOk = false;
Interlocked.Increment(ref ExceptionCount);
Console.WriteLine($"INSIDE LOOP EXCEPTION while scanning {_containerName}. Item: {blob.Uri} Scanned Items: #{TotalItems}. Ex message:" + ex.Message);
}
});
if (downloadOk == null)
{
downloadOk = true;
}
}
catch (Exception ex)
{
downloadOk = false;
Interlocked.Increment(ref ExceptionCount);
Console.WriteLine($"OUTER EXCEPTION ({_containerName}) #{TotalItems}: " + ex.Message);
}
finally
{
_sqlLite.EndTransaction();
CheckPrintConsole(true);
Console.WriteLine(" Fetch done");
}
var nowUtc = DateTime.UtcNow;
var delTask = Task.Run(() =>
{
if (!downloadOk.HasValue || !downloadOk.Value)
{
Console.WriteLine(" Due to exception, no delete check will be done");
return;
}
Console.WriteLine(" Starting delete files known in local sql but not in azure");
_sqlLite.GetAllFileInfos().AsParallel().
Where(fi => !ExpectedLocalFiles.Contains(fi.LocalName)).
ForAll(fileInfo =>
{
AddJobChar('d');
fileInfo.DeleteDetectedTime = nowUtc;
fileInfo.UpdateDb();
string fileName = Path.Combine(_localPath, fileInfo.LocalName);
var fi = new FileInfo(fileName);
var newName = fileName + FLAG_DELETED + nowUtc.ToString(FLAG_DATEFORMAT) + FLAG_END;
if (fi.Exists)
{
fi.MoveTo(newName);
}
else if (Directory.Exists(Path.GetDirectoryName(newName)))
{
File.Create(newName + ".empty").Close(); // creates dummy file to mark as deleted
}
Interlocked.Increment(ref DeletedItems);
});
CheckPrintConsole(true);
Console.WriteLine(" Delete files known in local sql but not in azure done");
Console.WriteLine(" Starting delete existing local files not in azure");
// scan for deleted files by checking if we have a file locally that we did not find remotely
// load list of local files
// this might take a minute or 2 if many files, since we wait for first yielded item before continuing
// done after sql Loop, since that should "remove" them already
EnumerateFilesParallel(localDir).
Where(DoLocalFileDelete).
ForAll(fi =>
{
AddJobChar('D');
fi.MoveTo(fi.FullName + FLAG_DELETED + nowUtc.ToString(FLAG_DATEFORMAT) + FLAG_END);
Interlocked.Increment(ref DeletedItems);
});
CheckPrintConsole(true);
Console.WriteLine(" Delete existing local files not in azure done");
});
AddTasks(delTask);
BlobJobQueue.RunnerDone();
return(this);
}
internal FileInfo(FileInfoSqlite sqlite, ILocalFileInfo fi, BlobItem blob)
: this(sqlite, blob)
{
SrcFileInfo = fi;
UpdateFromFileInfo(fi);
}
private FileInfo(FileInfoSqlite sqlite, BlobItem blob)
: this(sqlite)
{
SrcFileInfo = blob;
UpdateFromAzure(blob);
}