public async Task BuildItemInfo(FSItem item) { var node = await amazon.Nodes.GetNodeExtended(item.Id); var info = new ACDDokanNetItemInfo { Id = item.Id, TempLink = node.tempLink, Assets = node.assets?.Select(i => new ACDDokanNetAssetInfo { Id = i.id, TempLink = i.tempLink }).ToList() }; if (node.video != null) { info.Video = new ACDDokanNetAssetInfoImage { Width = node.video.width, Height = node.video.height }; } if (node.image != null) { info.Image = new ACDDokanNetAssetInfoImage { Width = node.image.width, Height = node.image.height }; } string str = JsonConvert.SerializeObject(info); item.Info = Encoding.UTF8.GetBytes(str); }
private Downloader StartDownload(FSItem item, string path) { try { var downloader = new Downloader(item, path); if (!downloaders.TryAdd(item.Path, downloader)) { return(downloaders[item.Path]); } var writer = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); if (writer.Length == item.Length) { writer.Close(); Downloader removed; downloaders.TryRemove(item.Path, out removed); return(Downloader.CreateCompleted(item, path, item.Length)); } if (writer.Length > 0) { Log.Warn($"File was not totally downloaded before. Should be {item.Length} but was {writer.Length}: {path}"); } downloader.Downloaded = writer.Length; downloader.Task = Task.Run(async() => await Download(item, writer, downloader)); return(downloader); } catch (IOException e) { Log.Trace("File is already downloading: " + item.Id + "\r\n" + e); return(downloaders[item.Path]); } }
public UploadInfo(FSItem item) { Id = item.Id; Path = item.Path; ParentId = item.ParentIds.First(); Length = item.Length; }
public static Downloader CreateCompleted(FSItem item, string path, long length) { return(new Downloader(item, path) { Task = Task.FromResult <bool>(true), Downloaded = length }); }
public IBlockStream OpenReadWithDownload(FSItem item) { var path = Path.Combine(cachePath, item.Id); StartDownload(item, path); return(OpenReadCachedOnly(item)); }
private void AddUpload(FSItem item) { var info = new UploadInfo(item); var path = Path.Combine(cachePath, item.Id); WriteInfo(path + ".info", info); uploads.Add(info); }
public void CreateDir(string filePath) { var dir = Path.GetDirectoryName(filePath); var dirNode = GetItem(dir); var name = Path.GetFileName(filePath); var node = amazon.Nodes.CreateFolder(dirNode.Id, name).Result; itemsTreeCache.Add(FSItem.FromNode(filePath, node)); }
public NewFileBlockWriter OpenNew(FSItem item) { var path = Path.Combine(cachePath, item.Id); var result = new NewFileBlockWriter(item, path); result.OnClose = () => { AddUpload(item); }; return(result); }
public void Update(FSItem newitem) { lok.EnterWriteLock(); try { pathToNode[newitem.Path] = newitem; } finally { lok.ExitWriteLock(); } }
public void AddOverwrite(FSItem item) { var info = new UploadInfo(item) { Overwrite = true }; var path = Path.Combine(cachePath, item.Id); WriteInfo(path + ".info", info); uploads.Add(info); }
public void AddItemOnly(FSItem item) { lok.EnterWriteLock(); try { pathToNode[item.Path] = item; } finally { lok.ExitWriteLock(); } }
public FSItem(FSItem item) { IsUploading = item.IsUploading; Path = item.Path; Id = item.Id; IsDir = item.IsDir; Length = item.Length; LastAccessTime = item.LastAccessTime; LastWriteTime = item.LastWriteTime; CreationTime = item.CreationTime; ParentIds = new ConcurrentBag <string>(item.ParentIds); }
public void MoveDir(string oldPath, FSItem newNode) { lok.EnterWriteLock(); try { DeleteDir(oldPath); Add(newNode); } finally { lok.ExitWriteLock(); } }
public NewFileBlockWriter OpenTruncate(FSItem item) { var path = Path.Combine(cachePath, item.Id); var result = new NewFileBlockWriter(item, path); result.SetLength(0); result.OnClose = () => { AddOverwrite(item); }; return(result); }
public SmallFileBlockReaderWriter OpenReadWrite(FSItem item) { var path = Path.Combine(cachePath, item.Id); var downloader = StartDownload(item, path); CacheEntry entry; if (access.TryGetValue(item.Id, out entry)) { entry.AccessTime = DateTime.UtcNow; } Log.Trace("Opened ReadWrite cached: " + item.Id); return(new SmallFileBlockReaderWriter(downloader)); }
public void Add(FSItem item) { lok.EnterWriteLock(); try { pathToNode[item.Path] = item; DirItem dirItem; if (pathToDirItem.TryGetValue(item.Dir, out dirItem)) { dirItem.Items.Add(item.Path); } } finally { lok.ExitWriteLock(); } }
private FSItem WaitForReal(FSItem item, int timeout) { var timeouttime = DateTime.UtcNow.AddMilliseconds(timeout); while (item.IsUploading) { if (DateTime.UtcNow > timeouttime) { throw new TimeoutException(); } Thread.Sleep(100); item = GetItem(item.Path); } return(item); }
public FileBlockReader OpenReadCachedOnly(FSItem item) { var path = Path.Combine(cachePath, item.Id); if (!File.Exists(path)) { return(null); } CacheEntry entry; if (access.TryGetValue(item.Id, out entry)) { entry.AccessTime = DateTime.UtcNow; } Log.Trace("Opened cached: " + item.Id); return(FileBlockReader.Open(path, item.Length)); }
private void CheckOldUploads() { var files = Directory.GetFiles(cachePath, "*.info"); if (files.Length == 0) { return; } Log.Warn($"{files.Length} not uploaded files found. Resuming."); foreach (var info in files.Select(f => new FileInfo(f)).OrderBy(f => f.CreationTime)) { var uploadinfo = JsonConvert.DeserializeObject <UploadInfo>(File.ReadAllText(info.FullName)); var fileinfo = new FileInfo(Path.Combine(info.DirectoryName, Path.GetFileNameWithoutExtension(info.Name))); var item = FSItem.MakeUploading(uploadinfo.Path, fileinfo.Name, uploadinfo.ParentId, fileinfo.Length); OnUploadResumed(item); uploads.Add(uploadinfo); } }
private FileInformation MakeFileInformation(FSItem i) { var result = new FileInformation { Length = i.Length, FileName = i.Name, Attributes = i.IsDir ? FileAttributes.Directory : FileAttributes.Normal, LastAccessTime = i.LastAccessTime, LastWriteTime = i.LastWriteTime, CreationTime = i.CreationTime }; if (ReadOnly) { result.Attributes |= FileAttributes.ReadOnly; } return(result); }
private void DeleteItem(string filePath, FSItem item) { try { if (item.ParentIds.Count == 1) { amazon.Nodes.Trash(item.Id).Wait(); } else { var dir = Path.GetDirectoryName(filePath); var dirItem = GetItem(dir); amazon.Nodes.Remove(dirItem.Id, item.Id).Wait(); } } catch (AggregateException ex) { var webex = ex.InnerException as HttpWebException; if (webex == null || (webex.StatusCode != HttpStatusCode.NotFound && webex.StatusCode != HttpStatusCode.Conflict)) { throw ex.InnerException; } } }
public async Task <IList <FSItem> > GetDirItems(string folderPath) { var cached = itemsTreeCache.GetDir(folderPath); if (cached != null) { // Log.Warn("Got cached dir:\r\n " + string.Join("\r\n ", cached)); return((await Task.WhenAll(cached.Select(i => FetchNode(i)))).Where(i => i != null).ToList()); } var folderNode = GetItem(folderPath); var nodes = await amazon.Nodes.GetChildren(folderNode?.Id); var items = new List <FSItem>(nodes.Count); var curdir = folderPath; if (curdir == "\\") { curdir = string.Empty; } foreach (var node in nodes.Where(n => FsItemKinds.Contains(n.kind))) { if (node.status != AmazonNodeStatus.AVAILABLE) { continue; } var path = curdir + "\\" + node.name; items.Add(FSItem.FromNode(path, node)); } // Log.Warn("Got real dir:\r\n " + string.Join("\r\n ", items.Select(i => i.Path))); itemsTreeCache.AddDirItems(folderPath, items); return(items); }
public void AddExisting(FSItem item) { access.TryAdd(item.Id, new CacheEntry { Id = item.Id, AccessTime = DateTime.UtcNow }); }
private bool disposedValue = false; // To detect redundant calls public NewFileBlockWriter(FSItem item, string filePath) { this.item = item; writer = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); }
public void Delete(FSItem item) { var path = Path.Combine(cachePath, item.Id); File.Delete(path); }
private async Task <FSItem> FetchNode(string itemPath) { if (itemPath == "\\" || itemPath == string.Empty) { return(FSItem.FromRoot(await amazon.Nodes.GetRoot())); } var cached = itemsTreeCache.GetItem(itemPath); if (cached != null) { if (cached.NotExistingDummy) { // Log.Warn("NonExisting path from cache: " + itemPath); return(null); } return(cached); } var folders = new LinkedList <string>(); var curpath = itemPath; FSItem item = null; do { folders.AddFirst(Path.GetFileName(curpath)); curpath = Path.GetDirectoryName(curpath); if (curpath == "\\" || string.IsNullOrEmpty(curpath)) { break; } item = itemsTreeCache.GetItem(curpath); }while (item == null); if (item == null) { item = FSItem.FromRoot(await amazon.Nodes.GetRoot()); } foreach (var name in folders) { if (curpath == "\\") { curpath = string.Empty; } curpath = curpath + "\\" + name; var newnode = await amazon.Nodes.GetChild(item.Id, name); if (newnode == null || newnode.status != AmazonNodeStatus.AVAILABLE) { itemsTreeCache.AddItemOnly(FSItem.MakeNotExistingDummy(curpath)); // Log.Error("NonExisting path from server: " + itemPath); return(null); } item = FSItem.FromNode(curpath, newnode); itemsTreeCache.Add(item); } return(item); }
public void MoveFile(string oldPath, string newPath, bool replace) { if (oldPath == newPath) { return; } Log.Trace($"Move: {oldPath} to {newPath} replace:{replace}"); var oldDir = Path.GetDirectoryName(oldPath); var oldName = Path.GetFileName(oldPath); var newDir = Path.GetDirectoryName(newPath); var newName = Path.GetFileName(newPath); var item = GetItem(oldPath); item = WaitForReal(item, 25000); if (oldName != newName) { if (item.Length > 0 || item.IsDir) { item = FSItem.FromNode(Path.Combine(oldDir, newName), amazon.Nodes.Rename(item.Id, newName).Result); } else { item = new FSItem(item) { Path = Path.Combine(oldDir, newName) }; } if (item == null) { throw new InvalidOperationException("Can not rename"); } } if (oldDir != newDir) { var oldDirNodeTask = FetchNode(oldDir); var newDirNodeTask = FetchNode(newDir); Task.WaitAll(oldDirNodeTask, newDirNodeTask); if (item.Length > 0 || item.IsDir) { item = FSItem.FromNode(newPath, amazon.Nodes.Move(item.Id, oldDirNodeTask.Result.Id, newDirNodeTask.Result.Id).Result); if (item == null) { throw new InvalidOperationException("Can not move"); } } else { item = new FSItem(item) { Path = newPath }; } } if (item.IsDir) { itemsTreeCache.MoveDir(oldPath, item); } else { itemsTreeCache.MoveFile(oldPath, item); } }
private async Task Download(FSItem item, Stream writer, Downloader downloader) { Log.Trace("Started download: " + item.Id); var start = Stopwatch.StartNew(); var buf = new byte[4096]; using (writer) try { OnDownloadStarted?.Invoke(item.Id); while (writer.Length < item.Length) { await amazon.Files.Download(item.Id, fileOffset : writer.Length, streammer : async(response) => { var partial = response.StatusCode == System.Net.HttpStatusCode.PartialContent; ContentRangeHeaderValue contentRange = null; if (partial) { contentRange = response.Headers.GetContentRange(); if (contentRange.From != writer.Length) { throw new InvalidOperationException("Content range does not match request"); } } using (var stream = response.GetResponseStream()) { int red = 0; do { red = await stream.ReadAsync(buf, 0, buf.Length); if (writer.Length == 0) { Log.Trace("Got first part: " + item.Id + " in " + start.ElapsedMilliseconds); } writer.Write(buf, 0, red); downloader.Downloaded = writer.Length; }while (red > 0); } }); if (writer.Length < item.Length) { await Task.Delay(500); } } Log.Trace("Finished download: " + item.Id); OnDownloaded?.Invoke(item.Id); access.TryAdd(item.Id, new CacheEntry { Id = item.Id, AccessTime = DateTime.UtcNow, Length = item.Length }); TotalSize += item.Length; if (TotalSize > CacheSize) { StartClear(TotalSize - CacheSize); } } catch (Exception ex) { OnDownloadFailed?.Invoke(item.Id); Log.Error($"Download failed: {item.Id}\r\n{ex}"); } finally { Downloader remove; downloaders.TryRemove(item.Path, out remove); } }
public FSProvider(AmazonDrive amazon) { this.amazon = amazon; SmallFilesCache = new SmallFilesCache(amazon); SmallFilesCache.OnDownloadStarted = (id) => { Interlocked.Increment(ref downloadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); }; SmallFilesCache.OnDownloaded = (id) => { Interlocked.Decrement(ref downloadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); }; SmallFilesCache.OnDownloadFailed = (id) => { Interlocked.Decrement(ref downloadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); }; UploadService = new UploadService(2, amazon); UploadService.OnUploadFailed = (uploaditem, reason) => { Interlocked.Decrement(ref uploadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); var olditemPath = Path.Combine(UploadService.CachePath, uploaditem.Id); File.Delete(olditemPath); switch (reason) { case FailReason.ZeroLength: var item = GetItem(uploaditem.Path); item?.MakeNotUploading(); return; case FailReason.Conflict: return; } itemsTreeCache.DeleteFile(uploaditem.Path); }; UploadService.OnUploadFinished = (item, node) => { Interlocked.Decrement(ref uploadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); var newitem = FSItem.FromNode(item.Path, node); var olditemPath = Path.Combine(UploadService.CachePath, item.Id); var newitemPath = Path.Combine(SmallFilesCache.CachePath, node.id); if (!File.Exists(newitemPath)) { File.Move(olditemPath, newitemPath); } else { File.Delete(olditemPath); } SmallFilesCache.AddExisting(newitem); itemsTreeCache.Update(newitem); }; UploadService.OnUploadResumed = item => { itemsTreeCache.Add(item); Interlocked.Increment(ref uploadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); }; UploadService.Start(); }
public IBlockStream OpenFile(string filePath, FileMode mode, FileAccess fileAccess, FileShare share, FileOptions options) { var item = GetItem(filePath); if (fileAccess == FileAccess.Read) { if (item == null) { return(null); } item = WaitForReal(item, 25000); Log.Trace($"Opening {filePath} for Read"); if (item.Length < SmallFileSizeLimit) { return(SmallFilesCache.OpenReadWithDownload(item)); } var result = SmallFilesCache.OpenReadCachedOnly(item); if (result != null) { return(result); } Interlocked.Increment(ref downloadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); var buffered = new BufferedAmazonBlockReader(item, amazon); buffered.OnClose = () => { Interlocked.Decrement(ref downloadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); }; return(buffered); } if (item == null || item.Length == 0) { #if TRACE Log.Trace($"Creating {filePath} as New because mode:{mode} and {((item == null) ? "item is null" : "length is 0")}"); #endif var dir = Path.GetDirectoryName(filePath); var name = Path.GetFileName(filePath); var dirItem = GetItem(dir); item = FSItem.MakeUploading(filePath, Guid.NewGuid().ToString(), dirItem.Id, 0); var file = UploadService.OpenNew(item); Interlocked.Increment(ref uploadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); itemsTreeCache.Add(item); return(file); } if (item == null) { return(null); } item = WaitForReal(item, 25000); if ((mode == FileMode.Create || mode == FileMode.Truncate) && item.Length > 0) { #if TRACE Log.Trace($"Opening {filePath} as Truncate because mode:{mode} and length {item.Length}"); #endif item.Length = 0; SmallFilesCache.Delete(item); item.MakeUploading(); var file = UploadService.OpenTruncate(item); Interlocked.Increment(ref uploadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); return(file); } if (mode == FileMode.Open || mode == FileMode.Append || mode == FileMode.OpenOrCreate) { #if TRACE Log.Trace($"Opening {filePath} as ReadWrite because mode:{mode} and length {item.Length}"); #endif if (item.Length < SmallFileSizeLimit) { var file = SmallFilesCache.OpenReadWrite(item); file.OnChangedAndClosed = (it, path) => { it.LastWriteTime = DateTime.UtcNow; Interlocked.Increment(ref uploadingCount); OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount); if (!it.IsUploading) { it.MakeUploading(); var olditemPath = Path.Combine(SmallFilesCache.CachePath, item.Id); var newitemPath = Path.Combine(UploadService.CachePath, item.Id); if (File.Exists(newitemPath)) { File.Delete(newitemPath); } SymbolicLink.CreateFile(GetRelativePath(olditemPath, Path.GetDirectoryName(newitemPath)), newitemPath); SmallFilesCache.AddExisting(it); } UploadService.AddOverwrite(it); }; return(file); } Log.Warn("File is too big for ReadWrite: " + filePath); } return(null); }