Example #1
0
        public async Task Run(Guid serverId, string vmMoref = null, CancellationToken ctoken = default)
        {
            if (backup != null)
            {
                return;
            }

            this.ctoken = ctoken;

            Server = await metaDB.GetVMWareServer(serverId, true);

            backup = new DBBackup {
                Server = serverId, StartDate = DateTime.UtcNow, Status = Status.Running, Log = new List <string>()
            };
            backup.Id = await metaDB.AddBackup(backup);

            var vmBackup = new DBVMwareVM {
                Backup = backup.Id, Moref = vmMoref, StartDate = DateTime.UtcNow, Valid = false, Server = Server.Id
            };

            vmBackup.Id = await metaDB.AddVMwareVM(vmBackup);

            var snapRef = string.Empty;

            try
            {
                vim25Proxy = new Proxy(Server.Ip);
                await vim25Proxy.Login(Server.Username, Server.Password);

                if (vmMoref == null)
                {
                    int selected = -1;
                    var vmList   = (await vim25Proxy.GetVMs()).OrderBy(vm => vm.Name).ToArray();
                    do
                    {
                        var i = 1;
                        foreach (var vm in vmList)
                        {
                            Console.WriteLine("{0}\t{1}", i++, vm.Name);
                        }
                        Console.WriteLine("Choose a VM: ");
                        if (!int.TryParse(Console.ReadLine(), out selected) || selected < 1 || selected > vmList.Length)
                        {
                            selected = -1;
                        }
                    } while (selected < 1);
                    vmMoref = vmList[selected - 1].MoRef;
                }

                CheckCancelStatus();

                // Retrieve backup vm info, usefull for backup type
                var vmPowerState = await vim25Proxy.GetVMPowerState(vmMoref);

                var(changeTrackingEnabled, changeTrackingSupported) = await vim25Proxy.GetCBTState(vmMoref);

                // Check and enable CBT
                if (!changeTrackingEnabled && changeTrackingSupported)
                {
                    await vim25Proxy.ConfigureForCBT(vmMoref);
                }

                // Create backup snapshot
                snapRef = await vim25Proxy.CreateSnapshot(
                    vmMoref, "BerBackup", $"Backup started at {DateTime.Now}",
                    false, vmPowerState == VirtualMachinePowerState.poweredOn);

                // Retrieve full vm config from Snapshot
                var vmConfig = await vim25Proxy.GetVMConfigFromSnapshot(snapRef);

                vmBackup.Name = vim25Proxy.GetVMConfigParameter(vmConfig, "name") as string;

                vmBackup.Config = vim25Proxy.SerializeVMConfig(vmConfig);

                vmBackup.Id = await metaDB.AddVMwareVM(vmBackup);

                var previousVMBackup = changeTrackingEnabled ? await metaDB.GetLatestVM(serverId, vmMoref) : null;

                // Initialize VDDK stuff
                Environment.SetEnvironmentVariable("PATH", @"C:\Users\bedn\source\repos\backup\x64\Debug;C:\VMware-vix-disklib-6.5.2-6195444.x86_64\bin;%PATH%");
                var status = VixDiskLib.VixDiskLib_InitEx(6, 0, null, null, null, @"C:\VMware-vix-disklib-6.5.2-6195444.x86_64", null);

                var connectParams = new VixDiskLibConnectParams
                {
                    serverName  = Server.Ip,
                    thumbPrint  = Server.ThumbPrint,
                    credType    = VixDiskLibCredType.VIXDISKLIB_CRED_UID,
                    port        = 0,
                    nfcHostPort = 0,
                    vmxSpec     = string.Format("moref={0}", vmMoref)
                };

                connectParams.creds.uid.userName = Server.Username;
                connectParams.creds.uid.password = Server.Password;

                var vixConnHandle = IntPtr.Zero;
                var vixDiskHandle = IntPtr.Zero;

                status = VixDiskLib.VixDiskLib_ConnectEx(connectParams, (char)VixDiskLib.TRUE, snapRef, null, out vixConnHandle);

                var blocksReadBatch = 2048UL;
                var vddkReadBuffer  = new byte[blocksReadBatch * 512];

                foreach (var disk in vim25Proxy.GetDisksFromConfig(vmConfig))
                {
                    var diskInfo = vim25Proxy.GetDiskInfo(disk);

                    status = VixDiskLib.VixDiskLib_PrepareForAccess(connectParams, "BackupSoft");
                    status = VixDiskLib.VixDiskLib_Open(vixConnHandle, diskInfo.Path, (uint)VixDiskLib.VIXDISKLIB_FLAG_OPEN_READ_ONLY, out vixDiskHandle);

                    status = VixDiskLib.VixDiskLib_GetMetadataKeys(vixDiskHandle, null, 1, out uint metaLength);
                    var diskMeta = new byte[metaLength];
                    status = VixDiskLib.VixDiskLib_GetMetadataKeys(vixDiskHandle, diskMeta, metaLength, out metaLength);

                    var dbDisk = new DBVMDisk {
                        VM = vmBackup.Id, Path = diskInfo.Path, Key = diskInfo.Key, ChangeId = diskInfo.ChangeId, Metadata = diskMeta, Length = diskInfo.Capacity, Valid = false
                    };
                    dbDisk.Id = await metaDB.AddVMDisk(dbDisk);

                    Console.WriteLine(DateTime.Now);
                    Console.WriteLine($"Backup VM disk : {diskInfo.Path}");

                    var previousChangeId = "*";
                    var previousDisk     = previousVMBackup != null ? await metaDB.GetVMDisk(previousVMBackup.Id, diskInfo.Key) : null;

                    if (previousDisk != null)
                    {
                        previousChangeId = previousDisk.ChangeId;
                        await metaDB.CopyVMDiskBlocks(previousDisk.VM, dbDisk.Id);
                    }

                    long  bytestart  = 0;
                    long  bytelength = 0;
                    ulong iterations;
                    ulong startsector;
                    ulong lastSectors;

                    BlocksManager.dbBlocks = 0;
                    IList <byte[]> blocks;
                    var            totalBlocks = 0;

                    int alignoffset = 0;


                    foreach (var(start, length) in await vim25Proxy.GetDiskChangedAreas(vmMoref, snapRef, diskInfo.Key, bytestart, previousChangeId))
                    {
                        bytestart = start;

                        var _bytestart = await metaDB.GetPreviousVMDiskOffset(dbDisk.Id, bytestart);

                        if (_bytestart > 0)
                        {
                            bytestart = _bytestart;
                        }

                        var _byteend = await metaDB.GetNextVMDiskOffset(dbDisk.VM, start + length);

                        if (_byteend > 0)
                        {
                            bytelength = _byteend - bytestart;
                        }
                        else
                        {
                            bytelength = length;
                        }


                        iterations  = (Convert.ToUInt64(bytelength) / 512) / blocksReadBatch;
                        lastSectors = (Convert.ToUInt64(bytelength) / 512) % blocksReadBatch;

                        startsector = Convert.ToUInt64(bytestart) / 512;
                        alignoffset = Convert.ToInt32(bytestart % 512);
                        var startNotAligned = alignoffset > 0;

                        hasher.Initialize();

                        for (var pos = 0UL; pos < iterations; pos++)
                        {
                            status       = VixDiskLib.VixDiskLib_Read(vixDiskHandle, startsector, blocksReadBatch, vddkReadBuffer);
                            startsector += blocksReadBatch;
                            if (startNotAligned)
                            {
                                blocks          = hasher.NextBlock(vddkReadBuffer, alignoffset);
                                startNotAligned = false;
                            }
                            else
                            {
                                blocks = hasher.NextBlock(vddkReadBuffer, 0);
                            }
                            foreach (var block in blocks)
                            {
                                var blockGuid = await BlocksManager.AddBlockToDB(block);

                                await metaDB.AddVMDiskBlock(new DBVMDiskBlock { Block = blockGuid, VMDisk = dbDisk.Id, Offset = bytestart });

                                bytestart += block.Length;
                                totalBlocks++;
                            }
                        }

                        if (lastSectors > 0UL)
                        {
                            status       = VixDiskLib.VixDiskLib_Read(vixDiskHandle, startsector, lastSectors, vddkReadBuffer);
                            startsector += lastSectors;
                            if (startNotAligned)
                            {
                                blocks          = hasher.NextBlock(vddkReadBuffer, alignoffset, 512 * Convert.ToInt32(lastSectors));
                                startNotAligned = false;
                            }
                            else
                            {
                                blocks = hasher.NextBlock(vddkReadBuffer, 0, 512 * Convert.ToInt32(lastSectors));
                            }
                            foreach (var block in blocks)
                            {
                                var blockGuid = await BlocksManager.AddBlockToDB(block);

                                await metaDB.AddVMDiskBlock(new DBVMDiskBlock { Block = blockGuid, VMDisk = dbDisk.Id, Offset = bytestart });

                                bytestart += block.Length;
                                totalBlocks++;
                            }
                        }

                        if (hasher.HasRemainingBytes)
                        {
                            var diskSizeInSectors = Convert.ToUInt64(diskInfo.Capacity) / 512;
                            while (startsector < diskSizeInSectors)
                            {
                                var readSize = Math.Min(blocksReadBatch, diskSizeInSectors - startsector);
                                status       = VixDiskLib.VixDiskLib_Read(vixDiskHandle, startsector, readSize, vddkReadBuffer);
                                startsector += readSize;
                                if (startNotAligned)
                                {
                                    blocks          = hasher.NextBlock(vddkReadBuffer, alignoffset, Convert.ToInt32(readSize) * 512);
                                    startNotAligned = false;
                                }
                                else
                                {
                                    blocks = hasher.NextBlock(vddkReadBuffer, 0, Convert.ToInt32(readSize) * 512);
                                }
                                if (blocks.Count > 0)
                                {
                                    var block     = blocks[0];
                                    var blockGuid = await BlocksManager.AddBlockToDB(block);

                                    await metaDB.AddVMDiskBlock(new DBVMDiskBlock { Block = blockGuid, VMDisk = dbDisk.Id, Offset = bytestart });

                                    bytestart += block.Length;
                                    totalBlocks++;
                                    break;
                                }
                            }

                            if (startsector >= diskSizeInSectors && hasher.HasRemainingBytes)
                            {
                                var block     = hasher.RemainingBytes();
                                var blockGuid = await BlocksManager.AddBlockToDB(block);

                                await metaDB.AddVMDiskBlock(new DBVMDiskBlock { Block = blockGuid, VMDisk = dbDisk.Id, Offset = bytestart });

                                bytestart += block.Length;
                                totalBlocks++;
                            }
                        }

                        Console.WriteLine(DateTime.Now);

                        Console.WriteLine("Total blocks : " + totalBlocks);
                        Console.WriteLine("DB blocks : " + BlocksManager.dbBlocks);
                    }

                    Console.WriteLine(DateTime.Now);

                    Console.WriteLine("Total blocks : " + totalBlocks);
                    Console.WriteLine("DB blocks : " + BlocksManager.dbBlocks);

                    dbDisk.Valid = true;
                    await metaDB.AddVMDisk(dbDisk);

                    status = VixDiskLib.VixDiskLib_Close(vixDiskHandle);

                    dbDisk.Valid = true;
                    await metaDB.AddVMDisk(dbDisk);
                }


                status = VixDiskLib.VixDiskLib_Disconnect(vixConnHandle);
                VixDiskLib.VixDiskLib_Exit();

                vmBackup.EndDate = DateTime.UtcNow;
                vmBackup.Valid   = true;
                await metaDB.AddVMwareVM(vmBackup);
            }
            catch (Exception e)
            {
                Console.WriteLine($"{backup.Id} - Backup failed: {e.Message}");
                backup.AppendLog(e.Message);
                backup.Status = Status.Failed;
            }
            finally
            {
                if (vim25Proxy.IsConnected)
                {
                    if (snapRef != string.Empty)
                    {
                        await vim25Proxy.RemoveSnapshot(snapRef, false, true);
                    }
                    await vim25Proxy.Logout();
                }
            }
            vmBackup.EndDate = DateTime.UtcNow;
            await metaDB.AddVMwareVM(vmBackup);

            backup.EndDate = DateTime.UtcNow;
            await metaDB.AddBackup(backup);
        }
Example #2
0
        public async Task Run(Guid serverId, string[] items, CancellationToken ctoken = default)
        {
            if (backup != null)
            {
                return;
            }

            this.ctoken = ctoken;

            var server = await metaDB.GetWindowsServer(serverId, withcreds : true);

            agentProxy = new WindowsProxy(server.Ip, server.Username, server.Password)
            {
                BackupServiceCallback = this
            };

            backup = new DBBackup {
                Server = serverId, StartDate = DateTime.Now.ToUniversalTime(), Status = Status.Running, Log = new List <string>()
            };
            backup.Id = await metaDB.AddBackup(backup);

            backupStatus = Status.Successful;

            IObserver <CancellationTokenSource> observer = default;

            Observable.Interval(TimeSpan.FromSeconds(5)).TakeUntil(
                Observable.Create <CancellationTokenSource>(o =>
            {
                observer = o;
                return(() => { });
            }))
            .Subscribe(_ =>
            {
                var b = metaDB.GetBackup(backup.Id).GetAwaiter().GetResult();
                if (b.Status == Status.Cancelled)
                {
                    ctokenCancelBackup.Cancel();
                    observer.OnNext(ctokenCancelBackup);
                }
            });

            try
            {
                await agentProxy.Backup(items, backup.Id);

                foreach (var item in itemsQueue.GetConsumingEnumerable(ctoken))
                {
                    switch (item.Type)
                    {
                    case BackupItemType.File:
                    {
                        await BackupFile(item);
                    }
                    break;

                    case BackupItemType.Folder:
                    {
                        await BackupFolder(item);
                    }
                    break;
                    }
                }
            }
            catch (EndpointNotFoundException)
            {
                backup.AppendLog($"Cannot connect to server {server.Name} [{server.Ip}]");
                backupStatus = Status.Failed;
            }
            catch (OperationCanceledException)
            {
                backupStatus = Status.Cancelled;
            }
            catch (Exception e)
            {
                backup.AppendLog($"General Error: {e.Message}");
                backupStatus = Status.Failed;
            }
            finally
            {
                try
                {
                    await agentProxy.BackupComplete(backup.Id);
                }
                catch (Exception) { }

                backup.Status  = backupStatus;
                backup.EndDate = DateTime.Now.ToUniversalTime();
                await metaDB.AddBackup(backup);
            }

            Console.WriteLine($"{DateTime.Now} - Backup ended");
            Console.WriteLine($"{DateTime.Now} - Status: {backup.Status}");

            backup = null;
        }