示例#1
0
        private BlobJob GetBlobJob(BlobItem blob)
        {
            var itemCount = Interlocked.Increment(ref TotalItems);

            if (itemCount % 5000 == 0)
            {
                // set progress JobChar for next console update
                AddJobChar('.');
                CheckPrintConsole();
            }

            if (blob == null)
            {
                Interlocked.Increment(ref IgnoredItems);
                return(null);
            }

            Interlocked.Add(ref TotalSize, blob.Size);
            var localFileName = blob.GetLocalFileName();
            var bJob          = new BlobJob(this, blob, Path.Combine(_localPath, localFileName));

            if (localFileName == null)
            {
                throw new NullReferenceException();
            }
            lock (ExpectedLocalFilesLock)
                ExpectedLocalFiles.Add(localFileName);

            bJob.FileInfo      = _sqlLite.GetFileInfo(blob, bJob.LocalFilePath);
            bJob.AddDownloaded = AddDownloaded;

            return(bJob);
        }
示例#2
0
 internal void UpdateFromAzure(BlobItem blob)
 {
     LocalName        = blob.GetLocalFileName();
     RemPath          = blob.Uri.AbsolutePath;
     LastModifiedTime = blob.LastModifiedUtc.UtcDateTime;
     Size             = blob.Size;
     MD5 = blob.MD5;
 }
示例#3
0
        public ILocalFileInfo GetFileInfo(BlobItem blob, string localFilename = null)
        {
            ILocalFileInfo lfi = localFilename == null ? null : new LocalFileInfoDisk(localFilename);
            var            fi  = GetFileInfo(lfi, blob.GetLocalFileName());

            if (fi != null)
            {
                return(fi);
            }

            // create new instance
            ((LocalFileInfoDisk)lfi)?.GetMd5();
            fi = new FileInfo(this, lfi, blob);

            // make sure we insert item when there is nothing found
            ExecuteNonQuery("INSERT INTO " + SQL_TABLENAME +
                            " (LocalName, RemPath, LastModifiedTime, Size, AzureMD5, LastDownloadedTime, DeleteDetectedTime)" +
                            " VALUES (@1, @2, @3, @4, @5, @6, @7)",
                            fi.LocalName, fi.RemPath, fi.LastModifiedTime, fi.Size, fi.MD5, fi.LastDownloadedTime, fi.DeleteDetectedTime);

            return(fi);
        }
示例#4
0
 public BlobJob(Backup bakParent, BlobItem blob, string localFilePath)
 {
     Bak           = bakParent;
     Blob          = blob;
     LocalFilePath = localFilePath;
 }
示例#5
0
        public Backup PrepareJob(string accountName, string accountKey)
        {
            var localContainerPath = Path.Combine(_localPath, _containerName);

            Directory.CreateDirectory(localContainerPath);
            var localDir = new DirectoryInfo(localContainerPath);

            bool?downloadOk = null;

            try
            {
                _sqlLite.BeginTransaction();
                BlobItem.BlobEnumerator(_containerName, accountName, accountKey).Select(GetBlobJob).Where(j => j != null).ForAll(bJob =>
                {
                    var blob = bJob.Blob;
                    var file = bJob.FileInfo;
                    try
                    {
                        if (!file.Exists || bJob.ForceDownloadMissing())
                        {
                            bJob.NeedsJob = JobType.New;
                            bJob.SqlFileInfo.UpdateFromAzure(blob);

                            if (bJob.HandleWellKnownBlob())
                            {
                                Interlocked.Increment(ref IgnoredItems);
                            }
                            else
                            {
                                BlobJobQueue.AddDone(bJob);
                            }

                            Interlocked.Increment(ref NewItems);
                            Interlocked.Add(ref NewItemsSize, blob.Size);
                        }
                        else if (!file.IsSame(blob))
                        {
                            /*//
                             * if (file.Size != blob.Size ||
                             *  file.MD5 != blob.MD5 ||
                             *  file.LastModifiedTimeUtc == blob.LastModifiedTimeUtc) // debug stuff
                             *  Console.WriteLine($"\n** Seen {file.DiffString(blob)} for {bJob.LocalFilePath}");
                             * //*/
                            bJob.NeedsJob = JobType.Modified;
                            BlobJobQueue.AddDone(bJob);
                            Interlocked.Increment(ref ModifiedItems);
                            Interlocked.Add(ref ModifiedItemsSize, blob.Size);
                        }
                        else
                        {
                            Interlocked.Increment(ref UpToDateItems);
                        }
                    }
                    catch (Exception ex)
                    {
                        downloadOk = false;
                        Interlocked.Increment(ref ExceptionCount);
                        Console.WriteLine($"INSIDE LOOP EXCEPTION while scanning {_containerName}. Item: {blob.Uri} Scanned Items: #{TotalItems}. Ex message:" + ex.Message);
                    }
                });
                if (downloadOk == null)
                {
                    downloadOk = true;
                }
            }
            catch (Exception ex)
            {
                downloadOk = false;
                Interlocked.Increment(ref ExceptionCount);
                Console.WriteLine($"OUTER EXCEPTION ({_containerName}) #{TotalItems}: " + ex.Message);
            }
            finally
            {
                _sqlLite.EndTransaction();
                CheckPrintConsole(true);
                Console.WriteLine(" Fetch done");
            }

            var nowUtc  = DateTime.UtcNow;
            var delTask = Task.Run(() =>
            {
                if (!downloadOk.HasValue || !downloadOk.Value)
                {
                    Console.WriteLine(" Due to exception, no delete check will be done");
                    return;
                }
                Console.WriteLine(" Starting delete files known in local sql but not in azure");
                _sqlLite.GetAllFileInfos().AsParallel().
                Where(fi => !ExpectedLocalFiles.Contains(fi.LocalName)).
                ForAll(fileInfo =>
                {
                    AddJobChar('d');
                    fileInfo.DeleteDetectedTime = nowUtc;
                    fileInfo.UpdateDb();
                    string fileName = Path.Combine(_localPath, fileInfo.LocalName);
                    var fi          = new FileInfo(fileName);

                    var newName = fileName + FLAG_DELETED + nowUtc.ToString(FLAG_DATEFORMAT) + FLAG_END;
                    if (fi.Exists)
                    {
                        fi.MoveTo(newName);
                    }
                    else if (Directory.Exists(Path.GetDirectoryName(newName)))
                    {
                        File.Create(newName + ".empty").Close(); // creates dummy file to mark as deleted
                    }
                    Interlocked.Increment(ref DeletedItems);
                });
                CheckPrintConsole(true);
                Console.WriteLine(" Delete files known in local sql but not in azure done");
                Console.WriteLine(" Starting delete existing local files not in azure");

                // scan for deleted files by checking if we have a file locally that we did not find remotely
                // load list of local files
                // this might take a minute or 2 if many files, since we wait for first yielded item before continuing
                // done after sql Loop, since that should "remove" them already
                EnumerateFilesParallel(localDir).
                Where(DoLocalFileDelete).
                ForAll(fi =>
                {
                    AddJobChar('D');
                    fi.MoveTo(fi.FullName + FLAG_DELETED + nowUtc.ToString(FLAG_DATEFORMAT) + FLAG_END);
                    Interlocked.Increment(ref DeletedItems);
                });
                CheckPrintConsole(true);
                Console.WriteLine(" Delete existing local files not in azure done");
            });

            AddTasks(delTask);
            BlobJobQueue.RunnerDone();

            return(this);
        }
示例#6
0
 internal FileInfo(FileInfoSqlite sqlite, ILocalFileInfo fi, BlobItem blob)
     : this(sqlite, blob)
 {
     SrcFileInfo = fi;
     UpdateFromFileInfo(fi);
 }
示例#7
0
 private FileInfo(FileInfoSqlite sqlite, BlobItem blob)
     : this(sqlite)
 {
     SrcFileInfo = blob;
     UpdateFromAzure(blob);
 }