protected override bool Export()
{
var backupfile = ExportPath;
var dtbinary = Tools.DtBinary;
Log.Information("Exporting: {Backupfile}", backupfile);
Stopwatch watch = Stopwatch.StartNew();
string appfolder = PathHelper.GetParentFolder(PathHelper.GetParentFolder(dtbinary));
string logfile = GetLogFileName(appfolder, Name);
string args = $"/ErrorLog:{logfile} /ErrorDetails:All /s:DocumentDB /s.ConnectionString:{ConnectionString} /s.Collection:{Collection} /t:JsonFile /t.File:{backupfile} /t.Prettify";
int result = RunCommand(dtbinary, args);
watch.Stop();
Statistics.ExportDocumentDBTime += watch.Elapsed;
long elapsedms = (long)watch.Elapsed.TotalMilliseconds;
if (new FileInfo(logfile).Length > 0)
{
Log.Information("Reading logfile: {Logfile}", logfile);
string[] rows = File.ReadAllLines(logfile);
Log.ForContext("LogfileContent", LogHelper.TruncateLogFileContent(rows)).Information("dt results");
}
Log.Information("Deleting logfile: {Logfile}", logfile);
File.Delete(logfile);
if (result == 0 && File.Exists(backupfile) && new FileInfo(backupfile).Length > 0)
{
long size = new FileInfo(backupfile).Length;
long sizemb = size / 1024 / 1024;
Statistics.UncompressedSize += size;
Log
.ForContext("ElapsedMS", elapsedms)
.ForContext("Backupfile", backupfile)
.ForContext("Size", size)
.ForContext("SizeMB", sizemb)
.Information("Export success");
return(true);
}
Log
.ForContext("Binary", dtbinary)
.ForContext("Commandargs", LogHelper.Mask(args, new[] { ConnectionString, Collection }))
.ForContext("Result", result)
.ForContext("ElapsedMS", elapsedms)
.ForContext("Backupfile", backupfile)
.Warning("Export fail");
if (File.Exists(backupfile) && new FileInfo(backupfile).Length == 0)
{
Log.Information("Deleting empty file: {Backupfile}", backupfile);
File.Delete(backupfile);
}
return(false);
}
protected override bool Export()
{
var backupfolder = ExportPath;
var azcopybinary = Tools.AzcopyBinary;
Log.Information("Exporting: {Backupfolder}", backupfolder);
Stopwatch watch = Stopwatch.StartNew();
string azcopyFolder = backupfolder.Replace("azurestorage_", "junkfolder_");
if (Directory.Exists(azcopyFolder))
{
Log.Information("Deleting useless folder: {AzcopyFolder}", azcopyFolder);
Directory.Delete(azcopyFolder, true);
}
if (Directory.Exists(backupfolder))
{
Log.Information("Deleting folder: {Backupfolder}", backupfolder);
RobustDelete(backupfolder);
}
Log.Information("Creating folder: {Backupfolder}", backupfolder);
Directory.CreateDirectory(backupfolder);
string appfolder = PathHelper.GetParentFolder(PathHelper.GetParentFolder(azcopybinary));
string logfile = GetLogFileName(appfolder, Name);
string subdirs = Regex.IsMatch(Url, "^https://[a-z0-9]+\\.blob\\.core\\.windows\\.net") ? " /S" : string.Empty;
string args = $"/Source:{Url} /Dest:{backupfolder} /SourceKey:{Key} /V:{logfile} /Z:{azcopyFolder}{subdirs}";
int result = RunCommand(azcopybinary, args);
watch.Stop();
Statistics.ExportAzureStorageTime += watch.Elapsed;
long elapsedms = (long)watch.Elapsed.TotalMilliseconds;
if (new FileInfo(logfile).Length > 0)
{
Log.Information("Reading logfile: {Logfile}", logfile);
string[] rows = File.ReadAllLines(logfile)
.Where(l => !l.Contains("][VERBOSE] Downloaded entities: ") && !l.Contains("][VERBOSE] Start transfer: ") && !l.Contains("][VERBOSE] Finished transfer: "))
.ToArray();
Log.ForContext("LogfileContent", LogHelper.TruncateLogFileContent(rows)).Information("azcopy results");
}
Log.Information("Deleting logfile: {Logfile}", logfile);
File.Delete(logfile);
if (result == 0 && ContainsFiles(backupfolder))
{
long size = Directory.GetFiles(backupfolder, "*", SearchOption.AllDirectories).Sum(f => new FileInfo(f).Length);
long sizemb = size / 1024 / 1024;
Statistics.UncompressedSize += size;
Log
.ForContext("ElapsedMS", elapsedms)
.ForContext("Backupfolder", backupfolder)
.ForContext("Size", size)
.ForContext("SizeMB", sizemb)
.Information("Export success");
return(true);
}
Log
.ForContext("Binary", azcopybinary)
.ForContext("Commandargs", LogHelper.Mask(args, new[] { Url, Key }))
.ForContext("Result", result)
.ForContext("ElapsedMS", elapsedms)
.ForContext("Backupfolder", backupfolder)
.Warning("Export fail");
if (IsEmpty(backupfolder))
{
Log.Information("Deleting empty folder: {Backupfolder}", backupfolder);
RobustDelete(backupfolder);
}
return(false);
}
static void SyncBackups(string zipfolder, string targetServer, string targetAccount, string targetCertfile)
{
string rsyncbinary = Tools.RsyncBinary;
string binfolder = PathHelper.GetParentFolder(rsyncbinary);
string rsyncfolder = PathHelper.GetParentFolder(binfolder);
string appfolder = PathHelper.GetParentFolder(rsyncfolder);
string synccert = Path.Combine(rsyncfolder, "synccert", $"{targetCertfile}");
string logfile = GetLogFileName(appfolder, "SyncBackups");
string source = "/cygdrive/" + char.ToLower(zipfolder[0]) + zipfolder.Substring(2).Replace("\\", "/");
string target = $"{targetAccount}@{targetServer}:.";
string?oldfolder = null;
try
{
oldfolder = Directory.GetCurrentDirectory();
Directory.SetCurrentDirectory(binfolder);
for (int tries = 1; tries <= 5; tries++)
{
using (new ContextLogger(new Dictionary <string, object>()
{
["Tries"] = tries
}))
{
string[] files = Directory.GetFiles(zipfolder);
Log
.ForContext("FileCount", files.Length)
.Information("Syncing backup files: {Source} -> {Target}", source, LogHelper.Mask(target, new[] { targetServer, targetAccount }));
Stopwatch watch = Stopwatch.StartNew();
string args = $"--checksum --remove-source-files -a -l -e './ssh -o StrictHostKeyChecking=no -i {synccert}' {source} {target} --log-file {logfile}";
int result = RunCommand(Path.GetFileName(rsyncbinary), args);
watch.Stop();
Statistics.SyncTime += watch.Elapsed;
long elapsedms = (long)watch.Elapsed.TotalMilliseconds;
if (new FileInfo(logfile).Length > 0)
{
Log.Information("Reading logfile: {Logfile}", logfile);
string[] rows = File.ReadAllLines(logfile).Where(l => !l.Contains(".d..t...... ") && !l.Contains("<f..t...... ")).ToArray();
Log.ForContext("LogfileContent", LogHelper.TruncateLogFileContent(rows)).Information("rsync results");
}
Log.Information("Deleting logfile: {Logfile}", logfile);
File.Delete(logfile);
if (result == 0)
{
Log
.ForContext("ElapsedMS", elapsedms)
.ForContext("Zipfolder", zipfolder)
.Information("Sync success");
return;
}
else
{
Log
.ForContext("Binary", rsyncbinary)
.ForContext("Commandargs", LogHelper.Mask(args, new[] { targetServer, targetAccount }))
.ForContext("Result", result)
.ForContext("ElapsedMS", elapsedms)
.ForContext("Zipfolder", zipfolder)
.Warning("Sync fail");
}
}
}
}
finally
{
if (oldfolder != null)
{
Directory.SetCurrentDirectory(oldfolder);
}
}
Statistics.SuccessCount = 0;
}
static void DoExceptionalStuff(bool backupSqlServer, bool backupDocumentDB, bool backupMongoDB, bool backupAzureStorage)
{
dynamic settings = LoadAppSettings();
string eventHubConnectionString = settings.EventHubConnectionString;
string serilogTeamName = settings.SerilogTeamName ?? "Unknown team";
string serilogDepartment = settings.SerilogDepartment ?? "Unknown department";
string defaultTargetServer = settings.TargetServer;
string defaultTargetAccount = settings.TargetAccount;
string defaultTargetCertfile = settings.TargetCertfile;
string preBackupAction = settings.PreBackupAction;
string preBackupActionArgs = settings.PreBackupActionArgs;
string postBackupAction = settings.PostBackupAction;
string postBackupActionArgs = settings.PostBackupActionArgs;
string preSyncAction = settings.PreSyncAction;
string preSyncActionArgs = settings.PreSyncActionArgs;
string postSyncAction = settings.PostSyncAction;
string postSyncActionArgs = settings.PostSyncActionArgs;
if (defaultTargetServer == null)
{
string errorMessage = "Missing TargetServer configuration in appsettings.json";
Log.Error(errorMessage);
throw new Exception(errorMessage);
}
if (defaultTargetAccount == null)
{
string errorMessage = "Missing TargetAccount configuration in appsettings.json";
Log.Error(errorMessage);
throw new Exception(errorMessage);
}
if (defaultTargetCertfile == null)
{
string errorMessage = "Missing TargetCertfile configuration in appsettings.json";
Log.Error(errorMessage);
throw new Exception(errorMessage);
}
ConfigureLogging(eventHubConnectionString, serilogTeamName, serilogDepartment);
string appfolder = PathHelper.GetParentFolder(Directory.GetCurrentDirectory());
Stopwatch totalwatch = Stopwatch.StartNew();
string[] jsonfiles = Directory.GetFiles(appfolder, "backupjobs*.json");
string exportFolder = Path.Combine(appfolder, "export");
Log.Information("Creating folder: {ExportFolder}", exportFolder);
Directory.CreateDirectory(exportFolder);
string date = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
var backupjobs = BackupJob.LoadBackupJobs(jsonfiles, defaultTargetServer, defaultTargetAccount, defaultTargetCertfile, exportFolder, date);
BackupJob.ExcludeBackupJobs(backupjobs, backupSqlServer, backupDocumentDB, backupMongoDB, backupAzureStorage);
BackupJob.LogBackupJobs(backupjobs);
Tools.Prepare(appfolder);
string sendfolder = Path.Combine(appfolder, "backups");
if (preBackupAction != null)
{
BackupJob.RunCommand(preBackupAction, preBackupActionArgs);
}
BackupJob.ExportBackups(backupjobs, exportFolder);
if (postBackupAction != null)
{
BackupJob.RunCommand(postBackupAction, postBackupActionArgs);
}
if (preSyncAction != null)
{
BackupJob.RunCommand(preSyncAction, preSyncActionArgs);
}
BackupJob.SendBackups(backupjobs, sendfolder);
totalwatch.Stop();
Statistics.TotalTime = totalwatch.Elapsed;
Log
.ForContext("UncompressedSize", Statistics.UncompressedSize)
.ForContext("UncompressedSizeMB", Statistics.UncompressedSize / 1024 / 1024)
.ForContext("CompressedSize", Statistics.CompressedSize)
.ForContext("CompressedSizeMB", Statistics.CompressedSize / 1024 / 1024)
.ForContext("ExportSqlServerTimeMS", (long)Statistics.ExportSqlServerTime.TotalMilliseconds)
.ForContext("ExportDocumentDBTimeMS", (long)Statistics.ExportDocumentDBTime.TotalMilliseconds)
.ForContext("ExportMongoDBTimeMS", (long)Statistics.ExportDocumentDBTime.TotalMilliseconds)
.ForContext("ExportAzureStorageTimeMS", (long)Statistics.ExportAzureStorageTime.TotalMilliseconds)
.ForContext("ZipTimeMS", (long)Statistics.ZipTime.TotalMilliseconds)
.ForContext("SyncTimeMS", (long)Statistics.SyncTime.TotalMilliseconds)
.ForContext("TotalTimeMS", (long)Statistics.TotalTime.TotalMilliseconds)
.ForContext("TotalBackupJobs", backupjobs.Count)
.ForContext("BackupSuccessCount", Statistics.SuccessCount)
.ForContext("BackupFailCount", backupjobs.Count - Statistics.SuccessCount)
.Information("Backup finished");
if (postSyncAction != null)
{
BackupJob.RunCommand(postSyncAction, postSyncActionArgs);
}
}