private static async Task <List <int> > GetVMIDsToBackup(LNDynamic client, string source)
{
List <int> Result = new List <int>();
if (source.All(Char.IsDigit))
{
// Backing up a single vm_id
Result.Add(int.Parse(source));
}
else
{
// Backing up an entire region
var VMs = await client.VMListAsync();
foreach (var VM in VMs.OrderBy(x => x.vm_id))
{
if (VM.region == source)
{
Result.Add(VM.vm_id);
}
}
}
return(Result);
}
private static async Task <int> DoReplicate(LNDynamic client, int imageId, string destinationRegion)
{
Console.WriteLine($" - Replicating snapshot image to {destinationRegion}...");
int ReplicatedImageId = await client.ImageReplicateAndWaitAsync(imageId, destinationRegion, 60, 3, (s, e) =>
{
Console.WriteLine($" - New replication image {e.ImageId} queued for creation!");
}, (s, e) =>
{
Console.WriteLine($" - Image status is '{e.Status}', waiting {e.WaitSeconds} seconds for 'active'...");
}, (s, e) =>
{
Console.WriteLine($" - Retry {e.RetryNumber} of {e.MaxRetries}");
});
Console.WriteLine(" - Image status is 'active'!");
// Delete original image, leaving only replicated image
Console.WriteLine($" - Removing original snapshot...");
await client.ImageDeleteAsync(imageId);
Console.WriteLine($" - Original snapshot {imageId} deleted!");
return(ReplicatedImageId);
}
private static async Task DoDownload(LNDynamic client, int imageId, string filename)
{
Console.WriteLine($" - Downloading image {imageId} to {filename}...");
try
{
var LastProgressPercentage = -1.0;
await client.ImageRetrieveAsync(imageId, filename, (s, e) =>
{
// Only update every one hundredth of a percent (cuts cpu usage in half on my pc)
if (e.ProgressPercentage - LastProgressPercentage > 0.0001)
{
Console.Write($"\r - Downloaded {e.BytesReceived:n0} of {e.TotalBytesToReceive:n0} bytes ({e.ProgressPercentage:P})...");
LastProgressPercentage = e.ProgressPercentage;
}
});
Console.WriteLine();
Console.WriteLine(" - Image downloaded successfully!");
// TODO Validate checksum (here or in lndapi)?
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine(" - Image downloaded failed!");
Console.WriteLine(" - Message: " + ex.Message);
// TODO Is resume supported by the LND API? If so, retry instead of delete
// TODO If resume is not supported, then maybe queue up and prompt to re-download at end of program?
File.Delete(filename);
}
}
private static async Task DoCleanup(LNDynamic client, int newImageId, string newImagePrefix, string newImageFilename)
{
Console.WriteLine(" - Removing previous snapshot images from Luna Node...");
var ImagesToDelete = (await client.ImageListAsync()).Where(x => x.name.StartsWith($"{newImagePrefix} ") && x.image_id != newImageId);
if (ImagesToDelete.Any())
{
foreach (var ImageToDelete in ImagesToDelete)
{
await client.ImageDeleteAsync(ImageToDelete.image_id);
Console.WriteLine($" - Previous snapshot image {ImageToDelete.image_id} deleted!");
}
}
else
{
Console.WriteLine(" - No previous snapshot images found!");
}
Console.WriteLine(" - Removing previous snapshots from local filesystem...");
var FilesToDelete = Directory.GetFiles(Path.GetDirectoryName(newImageFilename), $"{newImagePrefix} *", SearchOption.TopDirectoryOnly).Where(x => Path.GetFileName(x) != Path.GetFileName(newImageFilename));
if (FilesToDelete.Any())
{
foreach (var FileToDelete in FilesToDelete)
{
File.Delete(FileToDelete);
Console.WriteLine($" - Previous snapshot image {FileToDelete} deleted!");
}
}
else
{
Console.WriteLine(" - No previous snapshot images found!");
}
}
private static async Task <int> DoSnapshot(LNDynamic client, int vmId, string newImageName)
{
Console.WriteLine($" - Creating snapshot image...");
int NewImageId = await client.VMSnapshotAndWaitAsync(vmId, newImageName, 60, 3, (s, e) =>
{
Console.WriteLine($" - New snapshot image {e.ImageId} queued for creation!");
}, (s, e) =>
{
Console.WriteLine($" - Image status is '{e.Status}', waiting {e.WaitSeconds} seconds for 'active'...");
}, (s, e) =>
{
Console.WriteLine($" - Retry {e.RetryNumber} of {e.MaxRetries}");
});
Console.WriteLine(" - Image status is 'active'!");
return(NewImageId);
}
static async Task MainAsync(string[] args)
{
Console.WriteLine(" lndcreditwatch starting up");
try {
using (LNDynamic client = new LNDynamic(Config.Default.APIID.GetPlainText(), Config.Default.APIKey.GetPlainText())) {
// Get current credit balance
double CurrentCreditBalance = await client.BillingCreditAsync();
// Handle the midnight hour, which will alert us to the usage from the previous day
string DailyUsageFilename = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "dailyusage.txt");
if (DateTime.Now.Hour == 0)
{
Console.WriteLine(" midnight hour, alerting previous day's usage");
if (File.Exists(DailyUsageFilename))
{
// Alert daily usage
SendEmail($"current Luna Node credit balance: ${CurrentCreditBalance}\r\nlndcreditwatch daily recap:\r\n\r\n{File.ReadAllText(DailyUsageFilename)}");
// And reset
File.Delete(DailyUsageFilename);
}
}
// Output some information
Console.WriteLine($" fixed cost per interval : ${Config.Default.FixedCostPerInterval}");
Console.WriteLine($" variable cost per interval : ${Config.Default.VariableCostPerInterval}");
Console.WriteLine($" total cost per interval : ${Config.Default.TotalCostPerInterval}");
Console.WriteLine($" current credit balance : ${CurrentCreditBalance}");
// Compare to previous credit balance
if (Config.Default.PreviousCreditBalance > double.MinValue)
{
Console.WriteLine($" previous credit balance : ${Config.Default.PreviousCreditBalance}");
double CostForThisInterval = Config.Default.PreviousCreditBalance - CurrentCreditBalance;
double CostPerDay = CostForThisInterval * 24.0; // TODOX Assumes hourly interval
double CostPerMonth = CostPerDay * 365.0 / 12.0;
Console.WriteLine($" cost for this interval : ${CostForThisInterval}");
Console.WriteLine($" cost per day : ${CostPerDay}");
Console.WriteLine($" cost per month : ${CostPerMonth}");
if (CostForThisInterval > Config.Default.TotalCostPerInterval)
{
// We spent too much during this last interval
File.AppendAllText(DailyUsageFilename, $"OVERSPEND\t${CostForThisInterval}\t{CostPerDay}\t{CostPerMonth}\r\n");
SendEmail($"lndcreditwatch noticed you overspent. At ${CostForThisInterval}/hr you'd be spending:\r\n${CostPerDay}/day\r\n${CostPerMonth}/month.\r\nCurrent balance: ${CurrentCreditBalance}");
}
else
{
// We were within our spending limit
File.AppendAllText(DailyUsageFilename, $"UNDERSPEND\t${CostForThisInterval}\t{CostPerDay}\t{CostPerMonth}\r\n");
if (Debugger.IsAttached)
{
SendEmail($"lndcreditwatch noticed you underspent. At ${CostForThisInterval}/hr you'd be spending:\r\n${CostPerDay}/day\r\n${CostPerMonth}/month.\r\nCurrent balance: ${CurrentCreditBalance}");
}
}
}
else
{
// No previous balance, so send an email alert saying watch has been setup
Console.WriteLine(" previous credit balance : N/A (looks like this is our first run)");
SendEmail("lndcreditwatch is now monitoring your account!");
}
// Store current credit as previous credit
Config.Default.PreviousCreditBalance = CurrentCreditBalance;
Config.Default.Save();
Console.WriteLine(" done");
}
} catch (Exception ex) {
SendEmail($"Exception running lndcreditwatch: {ex.ToString()}");
}
}
static async Task MainAsync(string[] args)
{
string Source = args[0];
string DestinationRegion = args[1];
string DestinationDirectory = args[2];
string APIID = args[3];
string APIKey = args[4];
// Ensure destination directory exists, or program fails right away if it can't be created
Directory.CreateDirectory(DestinationDirectory);
using (LNDynamic client = new LNDynamic(APIID, APIKey))
{
List <int> VMIDsToBackup = await GetVMIDsToBackup(client, Source);
if (VMIDsToBackup.Count == 0)
{
Console.WriteLine("Nothing to backup!");
}
else
{
Console.WriteLine($"Found {VMIDsToBackup.Count} VMs to backup");
foreach (int VMID in VMIDsToBackup)
{
try
{
Console.WriteLine($"- Backing up VMID {VMID}...");
var Details = await client.VMInfoAsync(VMID);
Console.WriteLine($" - name : {Details.extra.name}");
Console.WriteLine($" - region : {Details.extra.region}");
string NewImagePrefix = $"lndbackup {VMID}";
string NewImageName = $"{NewImagePrefix} {DateTime.Now.ToString("yyyy-MM-dd")} {Details.extra.name}";
string NewImageFilename = Path.Combine(DestinationDirectory, string.Join("_", NewImageName.Split(Path.GetInvalidFileNameChars())) + ".img");
int NewImageId = await DoSnapshot(client, VMID, NewImageName);
await DoDownload(client, NewImageId, NewImageFilename);
// TODO Compress after download using qemu-img? DoCompress();
if (Details.extra.region != DestinationRegion)
{
NewImageId = await DoReplicate(client, NewImageId, DestinationRegion);
}
await DoCleanup(client, NewImageId, NewImagePrefix, NewImageFilename);
}
catch (LNDException lndex)
{
Console.WriteLine($"lndapi error: {(Debugger.IsAttached ? lndex.ToString() : lndex.Message)}");
Console.WriteLine();
Console.WriteLine(" - Moving on to next VM to backup...");
Console.WriteLine();
Console.WriteLine();
}
catch (Exception ex)
{
Console.WriteLine($"fatal error: {(Debugger.IsAttached ? ex.ToString() : ex.Message)}");
Console.WriteLine();
Console.WriteLine(" - Moving on to next VM to backup...");
Console.WriteLine();
Console.WriteLine();
}
}
}
}
}
