public void AddBackupRun(BackupRun br)
{
this.Logger.LogInformation("Adding BackupRun with {0} file refs", br.BackupFileRefs == null ? 0 : br.BackupFileRefs.Count);
this.dbContext.Add <BackupRun>(br);
this.dbContext.SaveChangesAsync();
}
public void UpdateBackupRun(BackupRun br)
{
this.Logger.LogInformation("Updating BackupRun set with BackupRunID: {0}", br.BackupRunID);
this.dbContext.Update <BackupRun>(br);
this.dbContext.SaveChangesAsync();
}
public void CompleteFileArchive(BackupRunFileRef backupRef, BackupRun br)
{
var cacheFileName = GetCacheEntryForFile(backupRef.FullFileName, br);
if (File.Exists(cacheFileName))
{
Logger.LogDebug("Deleting cache file after archive: {0}", cacheFileName);
File.Delete(cacheFileName);
}
}
public Stream GetCacheStreamForItem(BackupRunFileRef item, BackupRun br)
{
var cacheFile = new FileInfo(GetCacheEntryForFile(item.FullFileName, br));
if (File.Exists(cacheFile.FullName) == false)
{
cacheFile.Create();
}
return(cacheFile.OpenRead());
}
public bool ArchiveFile(BackupRun backupRun, BackupRunFileRef fileRef, Stream cacheFileStream)
{
if (false == isInitialized)
{
this.Initialize();
}
Logger.LogDebug("Called FileSystemBackup.ArchiveFile");
//TODO Move direcory check
//Don't copy directories
//if (Directory.Exists(cacheFile.FullName))
//{
// Logger.LogInformation(String.Format("Skipping archive for directory reference: {0}", fileRef.FullFileName));
// return true;
//}
FileInfo archiveFile = new FileInfo(String.Format("{0}{1}{2}{3}{4}",
this.baseBackupDir,
Path.DirectorySeparatorChar,
backupRun.BackupRunID,
Path.DirectorySeparatorChar,
fileRef.FullFileName.Substring(3)));
if (archiveFile.Exists)
{
Logger.LogInformation(String.Format("Deleting existing archive entry for file: {0}", fileRef.FullFileName));
archiveFile.Delete();
}
if (archiveFile.Directory.Exists == false)
{
Logger.LogInformation(String.Format("Creating parent directory for archive entry: {0}", archiveFile.Directory.FullName));
archiveFile.Directory.Create();
}
byte[] buffer = new byte[1024];
int offset = 0;
int count;
while ((count = cacheFileStream.Read(buffer, offset, buffer.Length)) > 0)
{
offset += count;
}
Logger.LogInformation(String.Format("Completed archive copy for entry: {0} with result: {1}", fileRef.FullFileName, archiveFile.Exists));
return(true);
}
protected override void ProcessRecord()
{
if (ParameterSetName == ParameterSetNames.GetList)
{
IEnumerable <BackupRun> backups = GetAllBackupRuns();
WriteObject(backups, true);
}
else
{
BackupRunsResource.GetRequest request = Service.BackupRuns.Get(Project, Instance, Id);
BackupRun result = request.Execute();
WriteObject(result);
}
}
public void PopulateFilesForBackupRun(BackupRun br)
{
List <FileInfo> directoryFiles = new List <FileInfo>();
Logger.LogInformation("Starting directory scan for backup run");
foreach (var dirRef in br.BackupDirectories)
{
var rootDir = dirRef.DirectoryFullFileName;
if (String.IsNullOrWhiteSpace(rootDir))
{
throw new Exception("No root directory provided");
}
else if (!Directory.Exists(rootDir))
{
throw new Exception(String.Format("Root directory {0} doesn't exist", rootDir));
}
PopulateFileList(rootDir, directoryFiles);
}
foreach (FileInfo info in directoryFiles)
{
//Make sure file wasn't added in an ealier scan for this run
if (br.BackupFileRefs.Where <BackupRunFileRef>(r => r.FullFileName.ToLower().Equals(info.FullName.ToLower())).ToList().Count == 0)
{
Logger.LogInformation(String.Format("Adding file ref {0} to backup run ID {1}", info.FullName, br.BackupRunID));
br.BackupFileRefs.Add(new BackupRunFileRef {
FullFileName = info.FullName
});
}
}
string backupCacheFullDir = String.Format("{0}{1}", this.BackupCacheDirectory, br.BackupRunID);
if (Directory.Exists(backupCacheFullDir) == false)
{
Logger.LogInformation(String.Format("Creating new backup cache directory at {0}", backupCacheFullDir));
Directory.CreateDirectory(backupCacheFullDir);
}
}
private BackupRun CreateBackupRun()
{
Logger.LogDebug("CreateBackupRun called");
var appConfigRoot = this.serviceProvider.GetService <IConfigurationRoot>();
var config = appConfigRoot.GetSection("BackupSettings");
if (config == null)
{
throw new Exception("No config section found for backup settings");
}
config = config.GetSection("BackupDirectories");
if (config == null)
{
throw new Exception("No config section found for backup directories");
}
var br = new BackupRun
{
BackupFileRefs = new List <BackupRunFileRef>(),
BackupDirectories = new List <BackupDirectoryRef>()
};
string[] dirList = this.GetBackupSettingsConfigValue("BackupDirectories").Split(',');
for (int i = 0; i < dirList.Length; i++)
{
br.BackupDirectories.Add(new BackupDirectoryRef {
DirectoryFullFileName = dirList[i]
});
}
this.ClientDBHandler.AddBackupRun(br);
Logger.LogInformation(String.Format("Created new backup run with ID: {0}", br.BackupRunID));
return(br);
}
/// <summary>
/// Creates a new backup run on demand. This method is applicable only to Second Generation instances.
/// Documentation https://developers.google.com/sqladmin/v1beta4/reference/backupRuns/insert
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Sqladmin service.</param>
/// <param name="project">Project ID of the project that contains the instance.</param>
/// <param name="instance">Cloud SQL instance ID. This does not include the project ID.</param>
/// <param name="body">A valid Sqladmin v1beta4 body.</param>
/// <returns>OperationResponse</returns>
public static Operation Insert(SqladminService service, string project, string instance, BackupRun body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
if (project == null)
{
throw new ArgumentNullException(project);
}
if (instance == null)
{
throw new ArgumentNullException(instance);
}
// Make the request.
return(service.BackupRuns.Insert(body, project, instance).Execute());
}
catch (Exception ex)
{
throw new Exception("Request BackupRuns.Insert failed.", ex);
}
}
private void Start()
{
this.Logger.LogInformation("Initializing BackupClient services...");
var appConfig = new ConfigurationBuilder()
.SetBasePath(System.IO.Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true).Build();
this.serviceProvider = new ServiceCollection()
.AddSingleton <ICloudBackupArchiveProvider, FileSystemBackupArchiveProvider>()
.AddSingleton <IClientDBHandler, InMemoryDBHandler>()
.AddSingleton <IConfigurationRoot>(provider => appConfig)
.AddSingleton <IClientFileCacheHandler, LocalClientFileCacheHandler>()
.AddLogging(builder => builder.AddConsole())
.BuildServiceProvider();
this.Logger.LogInformation("Starting backup run at {0}", DateTime.Now.ToString());
var appConfigRoot = this.serviceProvider.GetService <IConfigurationRoot>();
var config = appConfigRoot.GetSection("BackupSettings");
if (config == null)
{
throw new Exception("No config section found for backup settings");
}
this.stopRunTime = DateTime.Now.AddSeconds(this.TotalRunTimeSeconds);
try
{
BackupRun br = null;
try
{
br = GetOpenBackupRun();
if (br == null)
{
this.Logger.LogInformation("Creating new backup run");
br = CreateBackupRun();
}
this.Logger.LogInformation(String.Format("Processing open backup run with ID: {0}", br.BackupRunID));
ArchiveCurrentBackupRun(br);
}
catch (Exception ex)
{
if (br != null)
{
br.BackupRunCompleted = true;
br.BackupRunEnd = DateTime.Now;
br.FailedWithException = true;
br.ExceptionMessage = String.Format("{0}:{1}", ex.Message, ex.StackTrace);
try
{
this.ClientDBHandler.UpdateBackupRun(br);
}
catch (Exception dbEx)
{
this.Logger.LogError(dbEx, "Couldn't save backup run exception: {0}", dbEx.Message);
}
}
this.Logger.LogError(ex, "Failure in processing backup run: {0}", ex.Message);
}
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failure in starting DB: {0}", ex.Message);
}
finally
{
this.ClientDBHandler.Dispose();
}
}
private BackupRun InitializeBackupRun(BackupRun br)
{
Logger.LogDebug("BackupClient.InitializeNewBackupRun called");
this.ClientFileCacheHandler.PopulateFilesForBackupRun(br);
long currentBytes;
string cacheRef;
int fileCount;
br.BackupRunStart = DateTime.Now;
currentBytes = 0;
//File count tracked separately in case the last run is all directories
fileCount = 0;
this.Logger.LogInformation("Copying scanned files to backup cache");
foreach (var fileRef in br.BackupFileRefs)
{
if (currentBytes > this.MaxCacheMB * 1000000)
{
Logger.LogInformation("Halting cache file copy due to max copy bytes exceded");
break;
}
if (fileRef.CopiedToCache == true)
{
Logger.LogDebug(String.Format("Skipping file already copied to backup cache: {0}", fileRef.FullFileName));
continue;
}
cacheRef = GetCacheEntryForFile(fileRef.FullFileName, br);
//Verify file wasn't removed from source or already copied to the cache
if ((Directory.Exists(fileRef.FullFileName) || File.Exists(fileRef.FullFileName)) && File.Exists(cacheRef) == false)
{
if (Directory.Exists(fileRef.FullFileName))
{
Logger.LogInformation(String.Format("Creating backup cache directory: {0}", cacheRef));
Directory.CreateDirectory(cacheRef);
}
else
{
if (File.Exists(cacheRef))
{
Logger.LogInformation(String.Format("Deleting previous version of cache file: {0}", cacheRef));
File.Delete(cacheRef);
}
Logger.LogInformation(String.Format("Copying cache file from source: {0} to target: {1}", fileRef.FullFileName, cacheRef));
File.Copy(fileRef.FullFileName, cacheRef);
var cacheFile = new FileInfo(cacheRef);
currentBytes += cacheFile.Length;
Logger.LogDebug(String.Format("Cache copy current bytes count: {0}", currentBytes));
}
fileRef.CopiedToCache = true;
fileCount += 1;
Logger.LogDebug("Saving file ref changes");
this.ClientDBHandler.UpdateBackupFileRef(fileRef);
}
}
if (fileCount == 0)
{
Logger.LogInformation("Backup file count == 0, setting complete flag");
br.BackupRunCompleted = true;
br.BackupRunEnd = DateTime.Now;
this.ClientDBHandler.UpdateBackupRun(br);
}
return(br);
}
private void ArchiveCurrentBackupRun(BackupRun br)
{
bool haltTimeNotExceeded;
do
{
InitializeBackupRun(br);
foreach (var item in br.BackupFileRefs)
{
if (item.CopiedToCache == true && item.CopiedToArchive == false && br.FailedWithException == false)
{
try
{
var cacheFileStream = this.ClientFileCacheHandler.GetCacheStreamForItem(item, br);
bool success = this.serviceProvider.GetService <ICloudBackupArchiveProvider>().ArchiveFile(br, item, cacheFileStream);
cacheFileStream.Close();
if (success == false)
{
throw new Exception(String.Format("Filed to archive file ref: {0}", item.FullFileName));
}
else
{
Logger.LogInformation("Arcive copy successful for source file: {0}", item.FullFileName);
item.CopiedToArchive = true;
this.ClientDBHandler.UpdateBackupFileRef(item);
}
}
catch (Exception ex)
{
br.FailedWithException = true;
br.BackupRunEnd = DateTime.Now;
br.ExceptionMessage = ex.Message;
// dbContext.Update<BackupRun>(br);
this.ClientDBHandler.UpdateBackupRun(br);
throw new Exception(String.Format("Backup run with ID {0} failed with exception: {1}", br.BackupRunID, ex.Message));
}
finally
{
this.ClientDBHandler.UpdateBackupRun(br);
}
}
}
haltTimeNotExceeded = (DateTime.Now.CompareTo(this.stopRunTime) < 0);
Logger.LogInformation("Completed cache copy run with BackupRun.Completed = {0} and halt time {1} exceeded",
br.BackupRunCompleted,
(haltTimeNotExceeded ? "was not" : "was"));
} while (br.BackupRunCompleted == false && haltTimeNotExceeded == true);
foreach (var backupRef in br.BackupFileRefs)
{
this.ClientFileCacheHandler.CompleteFileArchive(backupRef, br);
}
}
private string GetCacheEntryForFile(string fullFileName, BackupRun br) => String.Format("{0}{1}{2}{3}", this.BackupCacheDirectory, br.BackupRunID, Path.DirectorySeparatorChar, fullFileName.Substring(fullFileName.IndexOf(":") + 1));
