public void TestDeleteObjectAsync()
{
var client = new S3Service();
String bucketName = Environment.GetEnvironmentVariable("SCREEN3_S3_BUCKET");
client.DeleteObject(bucketName, @"source/1997-2006.zip").Wait();
}
public IEnumerable <BackupDetails> RemoveOldBackups()
{
var files = GetAll().OrderBy(f => f.BackupDate).ToList();
if (_settings.NumberOfBackupsToKeep >= files.Count())
{
return(null);
}
var filesToDelete = files.Take(files.Count - _settings.NumberOfBackupsToKeep).ToList();
foreach (var fileToDelete in filesToDelete)
{
_service.DeleteObject(_settings.Bucket, GenerateFileName(fileToDelete));
}
return(filesToDelete);
}
public async Task ProcessSourceFile(S3Object fileInfo)
{
LambdaLogger.Log($"About to download file, {fileInfo.Key}\n");
String resultFileName = await this.DownloadFileAsync(fileInfo.BucketName, fileInfo.Key, this.Temp_Folder + "originSourceFiles/");
string dailyTargetFolder = this.Temp_Folder + $"originExtractedFiles/{fileInfo.Key}/";
LambdaLogger.Log($"Download file, {fileInfo.Key}, about to extract to {dailyTargetFolder}.\n");
List <String> dailyFileList = this.ExtractIntoDayData(resultFileName, dailyTargetFolder);
LambdaLogger.Log($"Extracted files done. items {dailyFileList.Count} \n");
foreach (string path in dailyFileList)
{
this.AddDailyFileIntoStockDict(path);
}
TickerBLL bll = new TickerBLL(this.S3_Bucket_Name, this.Temp_Folder);
// save tickers into S3
foreach (KeyValuePair <string, List <TickerEntity> > tickerGroup in this.stockDict)
{
await bll.SaveTickersToS3(tickerGroup.Key, tickerGroup.Value, true);
LambdaLogger.Log($"Save to S3 for {tickerGroup.Key} with items {tickerGroup.Value.Count} \n");
}
// move file to archive
S3Service service = new S3Service();
string archiveKey = fileInfo.Key.Replace("source", "archive");
await service.CopyObject(this.S3_Bucket_Name, fileInfo.Key, this.S3_Bucket_Name, archiveKey);
await service.DeleteObject(this.S3_Bucket_Name, fileInfo.Key);
// about to clean things
File.Delete(resultFileName);
FileHelper.ClearDirectory(this.Temp_Folder + $"originExtractedFiles/", true);
this.stockDict = new Dictionary <string, List <TickerEntity> >();
LambdaLogger.Log("Temp folder cleared");
}
public void DeleteObject(string bucketName, string key)
{
client.DeleteObject(bucketName, key);
}
public override bool DeleteFile()
{
S3Service.DeleteObject(bucketName, key);
return(true);
}
public override bool DeleteFolder()
{
S3Service.DeleteObject(bucketName, FolderKey);
return(true);
}
public Task Delete(string bucket, string key)
{
CheckBucket(bucket);
return(Task.Factory.StartNew(() => Service.DeleteObject(bucket, key)));
}
public Program()
{
var usingTrustedConnection = string.IsNullOrEmpty(Username) && string.IsNullOrEmpty(Password);
var sourceConnection = usingTrustedConnection
? new ServerConnection(ServerName)
{
LoginSecure = true
}
: new ServerConnection(ServerName, Username, Password);
var sqlServer = new Server(sourceConnection);
if (sqlServer != null)
{
var backup = new Backup();
var dbc = sqlServer.Databases;
if (dbc.Contains(DatabaseName))
{
backup.Action = BackupActionType.Database;
backup.Database = DatabaseName;
var dateFilename = DateTime.UtcNow.ToString("dd-MMM-yyyy");
var tempFilename = String.Format("{0}-{1}.bak", DatabaseName, dateFilename);
var tempBackupPath = String.Format("{0}{1}", TempFilePath, tempFilename);
//remove old backups from this local temp location
foreach (var file in Directory.GetFiles(TempFilePath))
{
if (file != tempBackupPath)
{
Console.WriteLine("Removing previous temp backup " + file);
File.Delete(file);
}
}
try
{
var backupDevice = new BackupDeviceItem(tempBackupPath, DeviceType.File);
backup.Devices.Add(backupDevice);
backup.Checksum = true;
backup.ContinueAfterError = false;
backup.LogTruncation = BackupTruncateLogType.Truncate;
//if file exists then do an incremental, otherwise do a full
if (File.Exists(tempBackupPath))
{
backup.Incremental = true;
}
else
{
backup.Incremental = false;
}
// Perform backup.
backup.SqlBackup(sqlServer);
//now move the backup to S3 - overwriting anything that is there with the same name
var s3 = new S3Service
{
AccessKeyID = AccessKeyID,
SecretAccessKey = SecretAccessKey
};
var bucket = Bucket;
s3.AddObject(tempBackupPath, bucket, tempFilename);
var metadataOnly = true;
foreach (var listEntry in s3.ListObjects(Bucket, ""))
{
var request = new GetObjectRequest(s3, Bucket, listEntry.Name, metadataOnly);
using (var response = request.GetResponse())
{
if (response.LastModified < DateTime.UtcNow.AddDays(DaysToKeepS3BackupFor * -1))
{
Console.WriteLine("Going to delete old archive " + listEntry.Name);
s3.DeleteObject(Bucket, listEntry.Name);
}
}
}
Console.Out.WriteLine("Backup to S3 is complete");
System.Threading.Thread.Sleep(10000);
}
catch (Exception ee)
{
Console.Out.WriteLine("Exception occurred - do not continue. Wait until next run to try again " + ee.ToString());
System.Threading.Thread.Sleep(10000);
}
}
}
}