예제 #1
0
파일: B2Lib.cs 프로젝트: ijat/b2clone
        public async Task ScanFolderForUploads(string localPath, string b2Path)
        {
            try
            {
                if (B2Map.Count == 0)
                {
                    await GetB2List();
                }

                IDictionary <string, FileObject> dbFileObjects = B2Db.GetDbFileObjects();
                Log.Information("Scan started | Path = {0}, B2 Path = {1}", localPath, b2Path);

                int lastToUpload = UploadList.Count;
                int count        = 0;

                IEnumerable <string> dirEnumerable =
                    Directory.EnumerateFiles(localPath, "*.*", SearchOption.AllDirectories);

                Stopwatch stopWatch = Stopwatch.StartNew();

                await dirEnumerable.ForEachAsync(Environment.ProcessorCount, async filePath =>
                {
                    try
                    {
                        Interlocked.Increment(ref count);
                        ulong xhash = Utils.FilepathToXhash(filePath);

                        Log.Verbose("[Thread {1}] Path: {0}", filePath,
                                    Thread.CurrentThread.ManagedThreadId, xhash);

                        if (B2Map.TryGetValue(Utils.GetB2Filename(filePath, localPath, b2Path),
                                              out B2File existingB2File))
                        {
                            await HandleIfExistsInCloud(B2Db, filePath, existingB2File, xhash, localPath, b2Path);
                        }
                        else
                        {
                            await HandleNotExistsInCloud(B2Db, filePath, xhash, localPath, b2Path);
                        }
                    }
                    catch (IOException)
                    {
                        // ignore - file might be used by other processes
                    }
                    catch (Exception e)
                    {
                        Log.Error(e.ToString());
                    }
                });

                Log.Information(
                    "Scan finished | Total = {2}, To Upload = {3}, Execution time = {4} s",
                    localPath, b2Path, count, UploadList.Count - lastToUpload,
                    stopWatch.Elapsed.TotalSeconds);
            }
            catch (Exception e)
            {
                Log.Error(e.ToString());
            }
        }
예제 #2
0
파일: B2Lib.cs 프로젝트: ijat/b2clone
        private async Task HandleNotExistsInCloud(B2Db b2Db, string filePath, ulong xhash, string parentPath, string b2Path)
        {
            FileObject existingFileObject = b2Db.GetFileObject(filePath);

            // fo exists but not uploaded yet
            if (existingFileObject != null && existingFileObject.B2Files.Count == 0)
            {
                UploadList.Add(new UploadObject(existingFileObject, parentPath, b2Path));
            }
            // fo exists and b2file exits but not available on cloud, so clear b2file object
            else if (existingFileObject != null && existingFileObject.B2Files.Count >= 0)
            {
                existingFileObject.B2Files.Clear();
                existingFileObject.DateModified = DateTime.Now;
                await b2Db.UpdateFileObject(existingFileObject);

                UploadList.Add(new UploadObject(existingFileObject, parentPath, b2Path));
            }
            // no fo, no b2file
            else
            {
                FileObject tempFileObject = new FileObject(filePath, xhash);
                await b2Db.Add(tempFileObject);

                UploadList.Add(new UploadObject(tempFileObject, parentPath, b2Path));
            }
        }
예제 #3
0
파일: B2Lib.cs 프로젝트: ijat/b2clone
        private async Task HandleIfExistsInCloud(B2Db b2Db, string filePath, B2File existingB2File, ulong xhash,
                                                 string parentPath, string b2Path)
        {
            // try find local object in db
            FileObject existingFileObject = b2Db.GetFileObject(filePath);

            if (existingFileObject != null && existingFileObject.B2Files.ContainsKey(existingB2File.FileId))
            {
                // local object xhash diff from cloud, then mark this for upload
                if (existingFileObject.Xhash != xhash)
                {
                    existingFileObject.Xhash = xhash;
                    UploadList.Add(new UploadObject(existingFileObject, parentPath, b2Path));
                }
            }
            // if existing file object is available but b2file object is not available
            else if (existingFileObject != null && !existingFileObject.B2Files.ContainsKey(existingB2File.FileId))
            {
                // update b2file object and does not need to reupload
                if (existingB2File.ContentSHA1.Equals("none") ||
                    Utils.FilepathToSha1Hash(filePath).Equals(existingB2File.ContentSHA1.Replace("unverified:", ""), StringComparison.InvariantCultureIgnoreCase))
                {
                    existingFileObject.B2Files.Add(existingB2File.FileId, existingB2File);
                    existingFileObject.DateModified = DateTime.Now;
                    await b2Db.UpdateFileObject(existingFileObject);
                }
                else // mark this object to upload if diff sha
                {
                    existingFileObject.Xhash = xhash;
                    UploadList.Add(new UploadObject(existingFileObject, parentPath, b2Path));
                }
            }
            else
            {
                // local not available, so check it's sha1hash when same add to db or mark to upload
                FileObject tempFileObject = new FileObject(filePath, xhash);
                if (existingB2File.ContentSHA1.Equals("none") ||
                    Utils.FilepathToSha1Hash(filePath).Equals(existingB2File.ContentSHA1.Replace("unverified:", ""), StringComparison.InvariantCultureIgnoreCase))
                {
                    tempFileObject.B2Files.Add(existingB2File.FileId, existingB2File);
                    await b2Db.Add(tempFileObject);
                }
                else
                {
                    UploadList.Add(new UploadObject(tempFileObject, parentPath, b2Path));
                }
            }
        }
예제 #4
0
파일: B2Lib.cs 프로젝트: ijat/b2clone
 public B2Lib(string keyId, string appId, string bucketId)
 {
     try
     {
         B2Db      = new B2Db();
         B2Options = new B2Options()
         {
             BucketId       = bucketId,
             PersistBucket  = true,
             KeyId          = keyId,
             ApplicationKey = appId,
         };
         Client = new B2Client(B2Options);
         B2Map  = new ConcurrentDictionary <string, B2File>();
     }
     catch (Exception e)
     {
         Log.Error(e.ToString());
     }
 }
예제 #5
0
파일: B2Lib.cs 프로젝트: ijat/b2clone
        public async Task UploadToB2()
        {
            IDictionary <string, FileObject> dbFileObjects = B2Db.GetDbFileObjects();

            if (UploadList.Count == 0)
            {
                Log.Information("No local changes | Nothing to upload.");
                return;
            }

            Log.Information("Upload started");
            Log.Information("Total files to upload = {0}", UploadList.Count);
            Stopwatch stopWatch    = Stopwatch.StartNew();
            int       currentIndex = 0;

            await UploadList.ForEachAsync(Environment.ProcessorCount, async uploadObject =>
            {
                try
                {
                    // reset progress bar
                    // progressLastLength = 0;
                    // progressSpeedAverage.Clear();
                    // for (short i = 0; i<8; i++) progressSpeedAverage.Enqueue(0);
                    // progressLastUpdate = DateTime.Now;

                    B2File file        = null;
                    string b2Path      = Utils.GetB2Filename(uploadObject.FileObject.FilePath, uploadObject.ParentPath, uploadObject.B2Path);
                    string contentType = MimeTypeMap.GetMimeType(Path.GetExtension(uploadObject.FileObject.FilePath));

                    Log.Verbose("File = {0} | B2 Path = {1}", uploadObject.FileObject.FilePath, "/" + b2Path);

                    FileInfo fileInfo = new FileInfo(uploadObject.FileObject.FilePath);
                    if (fileInfo.Length < (1024 * 1024 * 100)) // more than 50MB
                    {
                        file = await UploadSmallFile(uploadObject, b2Path, contentType);
                    }
                    else
                    {
                        file = await UploadLargeFile(uploadObject, b2Path, contentType);
                    }

                    if (file != null)
                    {
                        uploadObject.FileObject.B2Files.Add(file.FileId, file);
                        uploadObject.FileObject.DateModified = DateTime.Now;
                        await B2Db.UpdateFileObject(uploadObject.FileObject);
                    }

                    Console.Write("Progress = {2}% [{0}/{1}]\t\t\t\r", ++currentIndex, UploadList.Count,
                                  Math.Round((Convert.ToDouble(currentIndex) / Convert.ToDouble(UploadList.Count)) * 100));
                }
                catch (Exception)
                {
                    RetryList.Add(uploadObject);
                    Log.Error("Error file = {0}", uploadObject.FileObject.FilePath);
                }
            });

            Log.Information("Upload finished | Execution time = {0} seconds | Error = {1}", stopWatch.Elapsed.TotalSeconds,
                            RetryList.Count);
        }