Beispiel #1
0
        public async Task BuildItemInfo(FSItem item)
        {
            var node = await amazon.Nodes.GetNodeExtended(item.Id);

            var info = new ACDDokanNetItemInfo
            {
                Id       = item.Id,
                TempLink = node.tempLink,
                Assets   = node.assets?.Select(i => new ACDDokanNetAssetInfo {
                    Id = i.id, TempLink = i.tempLink
                }).ToList()
            };

            if (node.video != null)
            {
                info.Video = new ACDDokanNetAssetInfoImage {
                    Width = node.video.width, Height = node.video.height
                };
            }

            if (node.image != null)
            {
                info.Image = new ACDDokanNetAssetInfoImage {
                    Width = node.image.width, Height = node.image.height
                };
            }

            string str = JsonConvert.SerializeObject(info);

            item.Info = Encoding.UTF8.GetBytes(str);
        }
Beispiel #2
0
        private Downloader StartDownload(FSItem item, string path)
        {
            try
            {
                var downloader = new Downloader(item, path);
                if (!downloaders.TryAdd(item.Path, downloader))
                {
                    return(downloaders[item.Path]);
                }

                var writer = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
                if (writer.Length == item.Length)
                {
                    writer.Close();
                    Downloader removed;
                    downloaders.TryRemove(item.Path, out removed);
                    return(Downloader.CreateCompleted(item, path, item.Length));
                }

                if (writer.Length > 0)
                {
                    Log.Warn($"File was not totally downloaded before. Should be {item.Length} but was {writer.Length}: {path}");
                }

                downloader.Downloaded = writer.Length;
                downloader.Task       = Task.Run(async() => await Download(item, writer, downloader));
                return(downloader);
            }
            catch (IOException e)
            {
                Log.Trace("File is already downloading: " + item.Id + "\r\n" + e);
                return(downloaders[item.Path]);
            }
        }
Beispiel #3
0
 public UploadInfo(FSItem item)
 {
     Id       = item.Id;
     Path     = item.Path;
     ParentId = item.ParentIds.First();
     Length   = item.Length;
 }
Beispiel #4
0
 public static Downloader CreateCompleted(FSItem item, string path, long length)
 {
     return(new Downloader(item, path)
     {
         Task = Task.FromResult <bool>(true),
         Downloaded = length
     });
 }
Beispiel #5
0
        public IBlockStream OpenReadWithDownload(FSItem item)
        {
            var path = Path.Combine(cachePath, item.Id);

            StartDownload(item, path);

            return(OpenReadCachedOnly(item));
        }
Beispiel #6
0
        private void AddUpload(FSItem item)
        {
            var info = new UploadInfo(item);

            var path = Path.Combine(cachePath, item.Id);

            WriteInfo(path + ".info", info);
            uploads.Add(info);
        }
Beispiel #7
0
        public void CreateDir(string filePath)
        {
            var dir     = Path.GetDirectoryName(filePath);
            var dirNode = GetItem(dir);

            var name = Path.GetFileName(filePath);
            var node = amazon.Nodes.CreateFolder(dirNode.Id, name).Result;

            itemsTreeCache.Add(FSItem.FromNode(filePath, node));
        }
Beispiel #8
0
        public NewFileBlockWriter OpenNew(FSItem item)
        {
            var path   = Path.Combine(cachePath, item.Id);
            var result = new NewFileBlockWriter(item, path);

            result.OnClose = () =>
            {
                AddUpload(item);
            };

            return(result);
        }
Beispiel #9
0
 public void Update(FSItem newitem)
 {
     lok.EnterWriteLock();
     try
     {
         pathToNode[newitem.Path] = newitem;
     }
     finally
     {
         lok.ExitWriteLock();
     }
 }
Beispiel #10
0
        public void AddOverwrite(FSItem item)
        {
            var info = new UploadInfo(item)
            {
                Overwrite = true
            };

            var path = Path.Combine(cachePath, item.Id);

            WriteInfo(path + ".info", info);
            uploads.Add(info);
        }
Beispiel #11
0
 public void AddItemOnly(FSItem item)
 {
     lok.EnterWriteLock();
     try
     {
         pathToNode[item.Path] = item;
     }
     finally
     {
         lok.ExitWriteLock();
     }
 }
Beispiel #12
0
 public FSItem(FSItem item)
 {
     IsUploading    = item.IsUploading;
     Path           = item.Path;
     Id             = item.Id;
     IsDir          = item.IsDir;
     Length         = item.Length;
     LastAccessTime = item.LastAccessTime;
     LastWriteTime  = item.LastWriteTime;
     CreationTime   = item.CreationTime;
     ParentIds      = new ConcurrentBag <string>(item.ParentIds);
 }
Beispiel #13
0
 public void MoveDir(string oldPath, FSItem newNode)
 {
     lok.EnterWriteLock();
     try
     {
         DeleteDir(oldPath);
         Add(newNode);
     }
     finally
     {
         lok.ExitWriteLock();
     }
 }
Beispiel #14
0
        public NewFileBlockWriter OpenTruncate(FSItem item)
        {
            var path   = Path.Combine(cachePath, item.Id);
            var result = new NewFileBlockWriter(item, path);

            result.SetLength(0);
            result.OnClose = () =>
            {
                AddOverwrite(item);
            };

            return(result);
        }
Beispiel #15
0
        public SmallFileBlockReaderWriter OpenReadWrite(FSItem item)
        {
            var path       = Path.Combine(cachePath, item.Id);
            var downloader = StartDownload(item, path);

            CacheEntry entry;

            if (access.TryGetValue(item.Id, out entry))
            {
                entry.AccessTime = DateTime.UtcNow;
            }

            Log.Trace("Opened ReadWrite cached: " + item.Id);
            return(new SmallFileBlockReaderWriter(downloader));
        }
Beispiel #16
0
 public void Add(FSItem item)
 {
     lok.EnterWriteLock();
     try
     {
         pathToNode[item.Path] = item;
         DirItem dirItem;
         if (pathToDirItem.TryGetValue(item.Dir, out dirItem))
         {
             dirItem.Items.Add(item.Path);
         }
     }
     finally
     {
         lok.ExitWriteLock();
     }
 }
Beispiel #17
0
        private FSItem WaitForReal(FSItem item, int timeout)
        {
            var timeouttime = DateTime.UtcNow.AddMilliseconds(timeout);

            while (item.IsUploading)
            {
                if (DateTime.UtcNow > timeouttime)
                {
                    throw new TimeoutException();
                }

                Thread.Sleep(100);
                item = GetItem(item.Path);
            }

            return(item);
        }
Beispiel #18
0
        public FileBlockReader OpenReadCachedOnly(FSItem item)
        {
            var path = Path.Combine(cachePath, item.Id);

            if (!File.Exists(path))
            {
                return(null);
            }

            CacheEntry entry;

            if (access.TryGetValue(item.Id, out entry))
            {
                entry.AccessTime = DateTime.UtcNow;
            }

            Log.Trace("Opened cached: " + item.Id);
            return(FileBlockReader.Open(path, item.Length));
        }
Beispiel #19
0
        private void CheckOldUploads()
        {
            var files = Directory.GetFiles(cachePath, "*.info");

            if (files.Length == 0)
            {
                return;
            }

            Log.Warn($"{files.Length} not uploaded files found. Resuming.");
            foreach (var info in files.Select(f => new FileInfo(f)).OrderBy(f => f.CreationTime))
            {
                var uploadinfo = JsonConvert.DeserializeObject <UploadInfo>(File.ReadAllText(info.FullName));
                var fileinfo   = new FileInfo(Path.Combine(info.DirectoryName, Path.GetFileNameWithoutExtension(info.Name)));
                var item       = FSItem.MakeUploading(uploadinfo.Path, fileinfo.Name, uploadinfo.ParentId, fileinfo.Length);
                OnUploadResumed(item);
                uploads.Add(uploadinfo);
            }
        }
Beispiel #20
0
        private FileInformation MakeFileInformation(FSItem i)
        {
            var result = new FileInformation
            {
                Length         = i.Length,
                FileName       = i.Name,
                Attributes     = i.IsDir ? FileAttributes.Directory : FileAttributes.Normal,
                LastAccessTime = i.LastAccessTime,
                LastWriteTime  = i.LastWriteTime,
                CreationTime   = i.CreationTime
            };

            if (ReadOnly)
            {
                result.Attributes |= FileAttributes.ReadOnly;
            }

            return(result);
        }
Beispiel #21
0
 private void DeleteItem(string filePath, FSItem item)
 {
     try
     {
         if (item.ParentIds.Count == 1)
         {
             amazon.Nodes.Trash(item.Id).Wait();
         }
         else
         {
             var dir     = Path.GetDirectoryName(filePath);
             var dirItem = GetItem(dir);
             amazon.Nodes.Remove(dirItem.Id, item.Id).Wait();
         }
     }
     catch (AggregateException ex)
     {
         var webex = ex.InnerException as HttpWebException;
         if (webex == null || (webex.StatusCode != HttpStatusCode.NotFound && webex.StatusCode != HttpStatusCode.Conflict))
         {
             throw ex.InnerException;
         }
     }
 }
Beispiel #22
0
        public async Task <IList <FSItem> > GetDirItems(string folderPath)
        {
            var cached = itemsTreeCache.GetDir(folderPath);

            if (cached != null)
            {
                // Log.Warn("Got cached dir:\r\n  " + string.Join("\r\n  ", cached));
                return((await Task.WhenAll(cached.Select(i => FetchNode(i)))).Where(i => i != null).ToList());
            }

            var folderNode = GetItem(folderPath);
            var nodes      = await amazon.Nodes.GetChildren(folderNode?.Id);

            var items  = new List <FSItem>(nodes.Count);
            var curdir = folderPath;

            if (curdir == "\\")
            {
                curdir = string.Empty;
            }

            foreach (var node in nodes.Where(n => FsItemKinds.Contains(n.kind)))
            {
                if (node.status != AmazonNodeStatus.AVAILABLE)
                {
                    continue;
                }

                var path = curdir + "\\" + node.name;
                items.Add(FSItem.FromNode(path, node));
            }

            // Log.Warn("Got real dir:\r\n  " + string.Join("\r\n  ", items.Select(i => i.Path)));
            itemsTreeCache.AddDirItems(folderPath, items);
            return(items);
        }
Beispiel #23
0
 public void AddExisting(FSItem item)
 {
     access.TryAdd(item.Id, new CacheEntry {
         Id = item.Id, AccessTime = DateTime.UtcNow
     });
 }
Beispiel #24
0
        private bool disposedValue = false; // To detect redundant calls

        public NewFileBlockWriter(FSItem item, string filePath)
        {
            this.item = item;

            writer = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
        }
Beispiel #25
0
        public void Delete(FSItem item)
        {
            var path = Path.Combine(cachePath, item.Id);

            File.Delete(path);
        }
Beispiel #26
0
        private async Task <FSItem> FetchNode(string itemPath)
        {
            if (itemPath == "\\" || itemPath == string.Empty)
            {
                return(FSItem.FromRoot(await amazon.Nodes.GetRoot()));
            }

            var cached = itemsTreeCache.GetItem(itemPath);

            if (cached != null)
            {
                if (cached.NotExistingDummy)
                {
                    // Log.Warn("NonExisting path from cache: " + itemPath);
                    return(null);
                }

                return(cached);
            }

            var    folders = new LinkedList <string>();
            var    curpath = itemPath;
            FSItem item    = null;

            do
            {
                folders.AddFirst(Path.GetFileName(curpath));
                curpath = Path.GetDirectoryName(curpath);
                if (curpath == "\\" || string.IsNullOrEmpty(curpath))
                {
                    break;
                }

                item = itemsTreeCache.GetItem(curpath);
            }while (item == null);
            if (item == null)
            {
                item = FSItem.FromRoot(await amazon.Nodes.GetRoot());
            }

            foreach (var name in folders)
            {
                if (curpath == "\\")
                {
                    curpath = string.Empty;
                }

                curpath = curpath + "\\" + name;

                var newnode = await amazon.Nodes.GetChild(item.Id, name);

                if (newnode == null || newnode.status != AmazonNodeStatus.AVAILABLE)
                {
                    itemsTreeCache.AddItemOnly(FSItem.MakeNotExistingDummy(curpath));

                    // Log.Error("NonExisting path from server: " + itemPath);
                    return(null);
                }

                item = FSItem.FromNode(curpath, newnode);
                itemsTreeCache.Add(item);
            }

            return(item);
        }
Beispiel #27
0
        public void MoveFile(string oldPath, string newPath, bool replace)
        {
            if (oldPath == newPath)
            {
                return;
            }

            Log.Trace($"Move: {oldPath} to {newPath} replace:{replace}");

            var oldDir  = Path.GetDirectoryName(oldPath);
            var oldName = Path.GetFileName(oldPath);
            var newDir  = Path.GetDirectoryName(newPath);
            var newName = Path.GetFileName(newPath);

            var item = GetItem(oldPath);

            item = WaitForReal(item, 25000);
            if (oldName != newName)
            {
                if (item.Length > 0 || item.IsDir)
                {
                    item = FSItem.FromNode(Path.Combine(oldDir, newName), amazon.Nodes.Rename(item.Id, newName).Result);
                }
                else
                {
                    item = new FSItem(item)
                    {
                        Path = Path.Combine(oldDir, newName)
                    };
                }

                if (item == null)
                {
                    throw new InvalidOperationException("Can not rename");
                }
            }

            if (oldDir != newDir)
            {
                var oldDirNodeTask = FetchNode(oldDir);
                var newDirNodeTask = FetchNode(newDir);
                Task.WaitAll(oldDirNodeTask, newDirNodeTask);
                if (item.Length > 0 || item.IsDir)
                {
                    item = FSItem.FromNode(newPath, amazon.Nodes.Move(item.Id, oldDirNodeTask.Result.Id, newDirNodeTask.Result.Id).Result);
                    if (item == null)
                    {
                        throw new InvalidOperationException("Can not move");
                    }
                }
                else
                {
                    item = new FSItem(item)
                    {
                        Path = newPath
                    };
                }
            }

            if (item.IsDir)
            {
                itemsTreeCache.MoveDir(oldPath, item);
            }
            else
            {
                itemsTreeCache.MoveFile(oldPath, item);
            }
        }
Beispiel #28
0
        private async Task Download(FSItem item, Stream writer, Downloader downloader)
        {
            Log.Trace("Started download: " + item.Id);
            var start = Stopwatch.StartNew();
            var buf   = new byte[4096];

            using (writer)
                try
                {
                    OnDownloadStarted?.Invoke(item.Id);
                    while (writer.Length < item.Length)
                    {
                        await amazon.Files.Download(item.Id, fileOffset : writer.Length, streammer : async(response) =>
                        {
                            var partial = response.StatusCode == System.Net.HttpStatusCode.PartialContent;
                            ContentRangeHeaderValue contentRange = null;
                            if (partial)
                            {
                                contentRange = response.Headers.GetContentRange();
                                if (contentRange.From != writer.Length)
                                {
                                    throw new InvalidOperationException("Content range does not match request");
                                }
                            }
                            using (var stream = response.GetResponseStream())
                            {
                                int red = 0;
                                do
                                {
                                    red = await stream.ReadAsync(buf, 0, buf.Length);
                                    if (writer.Length == 0)
                                    {
                                        Log.Trace("Got first part: " + item.Id + " in " + start.ElapsedMilliseconds);
                                    }

                                    writer.Write(buf, 0, red);
                                    downloader.Downloaded = writer.Length;
                                }while (red > 0);
                            }
                        });

                        if (writer.Length < item.Length)
                        {
                            await Task.Delay(500);
                        }
                    }

                    Log.Trace("Finished download: " + item.Id);
                    OnDownloaded?.Invoke(item.Id);

                    access.TryAdd(item.Id, new CacheEntry {
                        Id = item.Id, AccessTime = DateTime.UtcNow, Length = item.Length
                    });
                    TotalSize += item.Length;
                    if (TotalSize > CacheSize)
                    {
                        StartClear(TotalSize - CacheSize);
                    }
                }
                catch (Exception ex)
                {
                    OnDownloadFailed?.Invoke(item.Id);
                    Log.Error($"Download failed: {item.Id}\r\n{ex}");
                }
            finally
            {
                Downloader remove;
                downloaders.TryRemove(item.Path, out remove);
            }
        }
Beispiel #29
0
        public FSProvider(AmazonDrive amazon)
        {
            this.amazon     = amazon;
            SmallFilesCache = new SmallFilesCache(amazon);
            SmallFilesCache.OnDownloadStarted = (id) =>
            {
                Interlocked.Increment(ref downloadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
            };
            SmallFilesCache.OnDownloaded = (id) =>
            {
                Interlocked.Decrement(ref downloadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
            };
            SmallFilesCache.OnDownloadFailed = (id) =>
            {
                Interlocked.Decrement(ref downloadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
            };

            UploadService = new UploadService(2, amazon);
            UploadService.OnUploadFailed = (uploaditem, reason) =>
            {
                Interlocked.Decrement(ref uploadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);

                var olditemPath = Path.Combine(UploadService.CachePath, uploaditem.Id);
                File.Delete(olditemPath);

                switch (reason)
                {
                case FailReason.ZeroLength:
                    var item = GetItem(uploaditem.Path);
                    item?.MakeNotUploading();
                    return;

                case FailReason.Conflict:
                    return;
                }

                itemsTreeCache.DeleteFile(uploaditem.Path);
            };
            UploadService.OnUploadFinished = (item, node) =>
            {
                Interlocked.Decrement(ref uploadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);

                var newitem     = FSItem.FromNode(item.Path, node);
                var olditemPath = Path.Combine(UploadService.CachePath, item.Id);
                var newitemPath = Path.Combine(SmallFilesCache.CachePath, node.id);

                if (!File.Exists(newitemPath))
                {
                    File.Move(olditemPath, newitemPath);
                }
                else
                {
                    File.Delete(olditemPath);
                }

                SmallFilesCache.AddExisting(newitem);
                itemsTreeCache.Update(newitem);
            };
            UploadService.OnUploadResumed = item =>
            {
                itemsTreeCache.Add(item);
                Interlocked.Increment(ref uploadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
            };
            UploadService.Start();
        }
Beispiel #30
0
        public IBlockStream OpenFile(string filePath, FileMode mode, FileAccess fileAccess, FileShare share, FileOptions options)
        {
            var item = GetItem(filePath);

            if (fileAccess == FileAccess.Read)
            {
                if (item == null)
                {
                    return(null);
                }

                item = WaitForReal(item, 25000);

                Log.Trace($"Opening {filePath} for Read");

                if (item.Length < SmallFileSizeLimit)
                {
                    return(SmallFilesCache.OpenReadWithDownload(item));
                }

                var result = SmallFilesCache.OpenReadCachedOnly(item);
                if (result != null)
                {
                    return(result);
                }

                Interlocked.Increment(ref downloadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
                var buffered = new BufferedAmazonBlockReader(item, amazon);
                buffered.OnClose = () =>
                {
                    Interlocked.Decrement(ref downloadingCount);
                    OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
                };

                return(buffered);
            }

            if (item == null || item.Length == 0)
            {
#if TRACE
                Log.Trace($"Creating {filePath} as New because mode:{mode} and {((item == null) ? "item is null" : "length is 0")}");
#endif

                var dir     = Path.GetDirectoryName(filePath);
                var name    = Path.GetFileName(filePath);
                var dirItem = GetItem(dir);

                item = FSItem.MakeUploading(filePath, Guid.NewGuid().ToString(), dirItem.Id, 0);

                var file = UploadService.OpenNew(item);
                Interlocked.Increment(ref uploadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);

                itemsTreeCache.Add(item);

                return(file);
            }

            if (item == null)
            {
                return(null);
            }

            item = WaitForReal(item, 25000);

            if ((mode == FileMode.Create || mode == FileMode.Truncate) && item.Length > 0)
            {
#if TRACE
                Log.Trace($"Opening {filePath} as Truncate because mode:{mode} and length {item.Length}");
#endif
                item.Length = 0;
                SmallFilesCache.Delete(item);
                item.MakeUploading();
                var file = UploadService.OpenTruncate(item);
                Interlocked.Increment(ref uploadingCount);
                OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);

                return(file);
            }

            if (mode == FileMode.Open || mode == FileMode.Append || mode == FileMode.OpenOrCreate)
            {
#if TRACE
                Log.Trace($"Opening {filePath} as ReadWrite because mode:{mode} and length {item.Length}");
#endif
                if (item.Length < SmallFileSizeLimit)
                {
                    var file = SmallFilesCache.OpenReadWrite(item);
                    file.OnChangedAndClosed = (it, path) =>
                    {
                        it.LastWriteTime = DateTime.UtcNow;
                        Interlocked.Increment(ref uploadingCount);
                        OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);

                        if (!it.IsUploading)
                        {
                            it.MakeUploading();
                            var olditemPath = Path.Combine(SmallFilesCache.CachePath, item.Id);
                            var newitemPath = Path.Combine(UploadService.CachePath, item.Id);

                            if (File.Exists(newitemPath))
                            {
                                File.Delete(newitemPath);
                            }

                            SymbolicLink.CreateFile(GetRelativePath(olditemPath, Path.GetDirectoryName(newitemPath)), newitemPath);
                            SmallFilesCache.AddExisting(it);
                        }

                        UploadService.AddOverwrite(it);
                    };

                    return(file);
                }

                Log.Warn("File is too big for ReadWrite: " + filePath);
            }

            return(null);
        }