public async Task BuildItemInfo(FSItem item)
{
var node = await amazon.Nodes.GetNodeExtended(item.Id);
var info = new ACDDokanNetItemInfo
{
Id = item.Id,
TempLink = node.tempLink,
Assets = node.assets?.Select(i => new ACDDokanNetAssetInfo {
Id = i.id, TempLink = i.tempLink
}).ToList()
};
if (node.video != null)
{
info.Video = new ACDDokanNetAssetInfoImage {
Width = node.video.width, Height = node.video.height
};
}
if (node.image != null)
{
info.Image = new ACDDokanNetAssetInfoImage {
Width = node.image.width, Height = node.image.height
};
}
string str = JsonConvert.SerializeObject(info);
item.Info = Encoding.UTF8.GetBytes(str);
}
private Downloader StartDownload(FSItem item, string path)
{
try
{
var downloader = new Downloader(item, path);
if (!downloaders.TryAdd(item.Path, downloader))
{
return(downloaders[item.Path]);
}
var writer = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
if (writer.Length == item.Length)
{
writer.Close();
Downloader removed;
downloaders.TryRemove(item.Path, out removed);
return(Downloader.CreateCompleted(item, path, item.Length));
}
if (writer.Length > 0)
{
Log.Warn($"File was not totally downloaded before. Should be {item.Length} but was {writer.Length}: {path}");
}
downloader.Downloaded = writer.Length;
downloader.Task = Task.Run(async() => await Download(item, writer, downloader));
return(downloader);
}
catch (IOException e)
{
Log.Trace("File is already downloading: " + item.Id + "\r\n" + e);
return(downloaders[item.Path]);
}
}
public UploadInfo(FSItem item)
{
Id = item.Id;
Path = item.Path;
ParentId = item.ParentIds.First();
Length = item.Length;
}
public static Downloader CreateCompleted(FSItem item, string path, long length)
{
return(new Downloader(item, path)
{
Task = Task.FromResult <bool>(true),
Downloaded = length
});
}
public IBlockStream OpenReadWithDownload(FSItem item)
{
var path = Path.Combine(cachePath, item.Id);
StartDownload(item, path);
return(OpenReadCachedOnly(item));
}
private void AddUpload(FSItem item)
{
var info = new UploadInfo(item);
var path = Path.Combine(cachePath, item.Id);
WriteInfo(path + ".info", info);
uploads.Add(info);
}
public void CreateDir(string filePath)
{
var dir = Path.GetDirectoryName(filePath);
var dirNode = GetItem(dir);
var name = Path.GetFileName(filePath);
var node = amazon.Nodes.CreateFolder(dirNode.Id, name).Result;
itemsTreeCache.Add(FSItem.FromNode(filePath, node));
}
public NewFileBlockWriter OpenNew(FSItem item)
{
var path = Path.Combine(cachePath, item.Id);
var result = new NewFileBlockWriter(item, path);
result.OnClose = () =>
{
AddUpload(item);
};
return(result);
}
public void Update(FSItem newitem)
{
lok.EnterWriteLock();
try
{
pathToNode[newitem.Path] = newitem;
}
finally
{
lok.ExitWriteLock();
}
}
public void AddOverwrite(FSItem item)
{
var info = new UploadInfo(item)
{
Overwrite = true
};
var path = Path.Combine(cachePath, item.Id);
WriteInfo(path + ".info", info);
uploads.Add(info);
}
public void AddItemOnly(FSItem item)
{
lok.EnterWriteLock();
try
{
pathToNode[item.Path] = item;
}
finally
{
lok.ExitWriteLock();
}
}
public FSItem(FSItem item)
{
IsUploading = item.IsUploading;
Path = item.Path;
Id = item.Id;
IsDir = item.IsDir;
Length = item.Length;
LastAccessTime = item.LastAccessTime;
LastWriteTime = item.LastWriteTime;
CreationTime = item.CreationTime;
ParentIds = new ConcurrentBag <string>(item.ParentIds);
}
public void MoveDir(string oldPath, FSItem newNode)
{
lok.EnterWriteLock();
try
{
DeleteDir(oldPath);
Add(newNode);
}
finally
{
lok.ExitWriteLock();
}
}
public NewFileBlockWriter OpenTruncate(FSItem item)
{
var path = Path.Combine(cachePath, item.Id);
var result = new NewFileBlockWriter(item, path);
result.SetLength(0);
result.OnClose = () =>
{
AddOverwrite(item);
};
return(result);
}
public SmallFileBlockReaderWriter OpenReadWrite(FSItem item)
{
var path = Path.Combine(cachePath, item.Id);
var downloader = StartDownload(item, path);
CacheEntry entry;
if (access.TryGetValue(item.Id, out entry))
{
entry.AccessTime = DateTime.UtcNow;
}
Log.Trace("Opened ReadWrite cached: " + item.Id);
return(new SmallFileBlockReaderWriter(downloader));
}
public void Add(FSItem item)
{
lok.EnterWriteLock();
try
{
pathToNode[item.Path] = item;
DirItem dirItem;
if (pathToDirItem.TryGetValue(item.Dir, out dirItem))
{
dirItem.Items.Add(item.Path);
}
}
finally
{
lok.ExitWriteLock();
}
}
private FSItem WaitForReal(FSItem item, int timeout)
{
var timeouttime = DateTime.UtcNow.AddMilliseconds(timeout);
while (item.IsUploading)
{
if (DateTime.UtcNow > timeouttime)
{
throw new TimeoutException();
}
Thread.Sleep(100);
item = GetItem(item.Path);
}
return(item);
}
public FileBlockReader OpenReadCachedOnly(FSItem item)
{
var path = Path.Combine(cachePath, item.Id);
if (!File.Exists(path))
{
return(null);
}
CacheEntry entry;
if (access.TryGetValue(item.Id, out entry))
{
entry.AccessTime = DateTime.UtcNow;
}
Log.Trace("Opened cached: " + item.Id);
return(FileBlockReader.Open(path, item.Length));
}
private void CheckOldUploads()
{
var files = Directory.GetFiles(cachePath, "*.info");
if (files.Length == 0)
{
return;
}
Log.Warn($"{files.Length} not uploaded files found. Resuming.");
foreach (var info in files.Select(f => new FileInfo(f)).OrderBy(f => f.CreationTime))
{
var uploadinfo = JsonConvert.DeserializeObject <UploadInfo>(File.ReadAllText(info.FullName));
var fileinfo = new FileInfo(Path.Combine(info.DirectoryName, Path.GetFileNameWithoutExtension(info.Name)));
var item = FSItem.MakeUploading(uploadinfo.Path, fileinfo.Name, uploadinfo.ParentId, fileinfo.Length);
OnUploadResumed(item);
uploads.Add(uploadinfo);
}
}
private FileInformation MakeFileInformation(FSItem i)
{
var result = new FileInformation
{
Length = i.Length,
FileName = i.Name,
Attributes = i.IsDir ? FileAttributes.Directory : FileAttributes.Normal,
LastAccessTime = i.LastAccessTime,
LastWriteTime = i.LastWriteTime,
CreationTime = i.CreationTime
};
if (ReadOnly)
{
result.Attributes |= FileAttributes.ReadOnly;
}
return(result);
}
private void DeleteItem(string filePath, FSItem item)
{
try
{
if (item.ParentIds.Count == 1)
{
amazon.Nodes.Trash(item.Id).Wait();
}
else
{
var dir = Path.GetDirectoryName(filePath);
var dirItem = GetItem(dir);
amazon.Nodes.Remove(dirItem.Id, item.Id).Wait();
}
}
catch (AggregateException ex)
{
var webex = ex.InnerException as HttpWebException;
if (webex == null || (webex.StatusCode != HttpStatusCode.NotFound && webex.StatusCode != HttpStatusCode.Conflict))
{
throw ex.InnerException;
}
}
}
public async Task <IList <FSItem> > GetDirItems(string folderPath)
{
var cached = itemsTreeCache.GetDir(folderPath);
if (cached != null)
{
// Log.Warn("Got cached dir:\r\n " + string.Join("\r\n ", cached));
return((await Task.WhenAll(cached.Select(i => FetchNode(i)))).Where(i => i != null).ToList());
}
var folderNode = GetItem(folderPath);
var nodes = await amazon.Nodes.GetChildren(folderNode?.Id);
var items = new List <FSItem>(nodes.Count);
var curdir = folderPath;
if (curdir == "\\")
{
curdir = string.Empty;
}
foreach (var node in nodes.Where(n => FsItemKinds.Contains(n.kind)))
{
if (node.status != AmazonNodeStatus.AVAILABLE)
{
continue;
}
var path = curdir + "\\" + node.name;
items.Add(FSItem.FromNode(path, node));
}
// Log.Warn("Got real dir:\r\n " + string.Join("\r\n ", items.Select(i => i.Path)));
itemsTreeCache.AddDirItems(folderPath, items);
return(items);
}
public void AddExisting(FSItem item)
{
access.TryAdd(item.Id, new CacheEntry {
Id = item.Id, AccessTime = DateTime.UtcNow
});
}
private bool disposedValue = false; // To detect redundant calls
public NewFileBlockWriter(FSItem item, string filePath)
{
this.item = item;
writer = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
}
public void Delete(FSItem item)
{
var path = Path.Combine(cachePath, item.Id);
File.Delete(path);
}
private async Task <FSItem> FetchNode(string itemPath)
{
if (itemPath == "\\" || itemPath == string.Empty)
{
return(FSItem.FromRoot(await amazon.Nodes.GetRoot()));
}
var cached = itemsTreeCache.GetItem(itemPath);
if (cached != null)
{
if (cached.NotExistingDummy)
{
// Log.Warn("NonExisting path from cache: " + itemPath);
return(null);
}
return(cached);
}
var folders = new LinkedList <string>();
var curpath = itemPath;
FSItem item = null;
do
{
folders.AddFirst(Path.GetFileName(curpath));
curpath = Path.GetDirectoryName(curpath);
if (curpath == "\\" || string.IsNullOrEmpty(curpath))
{
break;
}
item = itemsTreeCache.GetItem(curpath);
}while (item == null);
if (item == null)
{
item = FSItem.FromRoot(await amazon.Nodes.GetRoot());
}
foreach (var name in folders)
{
if (curpath == "\\")
{
curpath = string.Empty;
}
curpath = curpath + "\\" + name;
var newnode = await amazon.Nodes.GetChild(item.Id, name);
if (newnode == null || newnode.status != AmazonNodeStatus.AVAILABLE)
{
itemsTreeCache.AddItemOnly(FSItem.MakeNotExistingDummy(curpath));
// Log.Error("NonExisting path from server: " + itemPath);
return(null);
}
item = FSItem.FromNode(curpath, newnode);
itemsTreeCache.Add(item);
}
return(item);
}
public void MoveFile(string oldPath, string newPath, bool replace)
{
if (oldPath == newPath)
{
return;
}
Log.Trace($"Move: {oldPath} to {newPath} replace:{replace}");
var oldDir = Path.GetDirectoryName(oldPath);
var oldName = Path.GetFileName(oldPath);
var newDir = Path.GetDirectoryName(newPath);
var newName = Path.GetFileName(newPath);
var item = GetItem(oldPath);
item = WaitForReal(item, 25000);
if (oldName != newName)
{
if (item.Length > 0 || item.IsDir)
{
item = FSItem.FromNode(Path.Combine(oldDir, newName), amazon.Nodes.Rename(item.Id, newName).Result);
}
else
{
item = new FSItem(item)
{
Path = Path.Combine(oldDir, newName)
};
}
if (item == null)
{
throw new InvalidOperationException("Can not rename");
}
}
if (oldDir != newDir)
{
var oldDirNodeTask = FetchNode(oldDir);
var newDirNodeTask = FetchNode(newDir);
Task.WaitAll(oldDirNodeTask, newDirNodeTask);
if (item.Length > 0 || item.IsDir)
{
item = FSItem.FromNode(newPath, amazon.Nodes.Move(item.Id, oldDirNodeTask.Result.Id, newDirNodeTask.Result.Id).Result);
if (item == null)
{
throw new InvalidOperationException("Can not move");
}
}
else
{
item = new FSItem(item)
{
Path = newPath
};
}
}
if (item.IsDir)
{
itemsTreeCache.MoveDir(oldPath, item);
}
else
{
itemsTreeCache.MoveFile(oldPath, item);
}
}
private async Task Download(FSItem item, Stream writer, Downloader downloader)
{
Log.Trace("Started download: " + item.Id);
var start = Stopwatch.StartNew();
var buf = new byte[4096];
using (writer)
try
{
OnDownloadStarted?.Invoke(item.Id);
while (writer.Length < item.Length)
{
await amazon.Files.Download(item.Id, fileOffset : writer.Length, streammer : async(response) =>
{
var partial = response.StatusCode == System.Net.HttpStatusCode.PartialContent;
ContentRangeHeaderValue contentRange = null;
if (partial)
{
contentRange = response.Headers.GetContentRange();
if (contentRange.From != writer.Length)
{
throw new InvalidOperationException("Content range does not match request");
}
}
using (var stream = response.GetResponseStream())
{
int red = 0;
do
{
red = await stream.ReadAsync(buf, 0, buf.Length);
if (writer.Length == 0)
{
Log.Trace("Got first part: " + item.Id + " in " + start.ElapsedMilliseconds);
}
writer.Write(buf, 0, red);
downloader.Downloaded = writer.Length;
}while (red > 0);
}
});
if (writer.Length < item.Length)
{
await Task.Delay(500);
}
}
Log.Trace("Finished download: " + item.Id);
OnDownloaded?.Invoke(item.Id);
access.TryAdd(item.Id, new CacheEntry {
Id = item.Id, AccessTime = DateTime.UtcNow, Length = item.Length
});
TotalSize += item.Length;
if (TotalSize > CacheSize)
{
StartClear(TotalSize - CacheSize);
}
}
catch (Exception ex)
{
OnDownloadFailed?.Invoke(item.Id);
Log.Error($"Download failed: {item.Id}\r\n{ex}");
}
finally
{
Downloader remove;
downloaders.TryRemove(item.Path, out remove);
}
}
public FSProvider(AmazonDrive amazon)
{
this.amazon = amazon;
SmallFilesCache = new SmallFilesCache(amazon);
SmallFilesCache.OnDownloadStarted = (id) =>
{
Interlocked.Increment(ref downloadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
};
SmallFilesCache.OnDownloaded = (id) =>
{
Interlocked.Decrement(ref downloadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
};
SmallFilesCache.OnDownloadFailed = (id) =>
{
Interlocked.Decrement(ref downloadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
};
UploadService = new UploadService(2, amazon);
UploadService.OnUploadFailed = (uploaditem, reason) =>
{
Interlocked.Decrement(ref uploadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
var olditemPath = Path.Combine(UploadService.CachePath, uploaditem.Id);
File.Delete(olditemPath);
switch (reason)
{
case FailReason.ZeroLength:
var item = GetItem(uploaditem.Path);
item?.MakeNotUploading();
return;
case FailReason.Conflict:
return;
}
itemsTreeCache.DeleteFile(uploaditem.Path);
};
UploadService.OnUploadFinished = (item, node) =>
{
Interlocked.Decrement(ref uploadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
var newitem = FSItem.FromNode(item.Path, node);
var olditemPath = Path.Combine(UploadService.CachePath, item.Id);
var newitemPath = Path.Combine(SmallFilesCache.CachePath, node.id);
if (!File.Exists(newitemPath))
{
File.Move(olditemPath, newitemPath);
}
else
{
File.Delete(olditemPath);
}
SmallFilesCache.AddExisting(newitem);
itemsTreeCache.Update(newitem);
};
UploadService.OnUploadResumed = item =>
{
itemsTreeCache.Add(item);
Interlocked.Increment(ref uploadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
};
UploadService.Start();
}
public IBlockStream OpenFile(string filePath, FileMode mode, FileAccess fileAccess, FileShare share, FileOptions options)
{
var item = GetItem(filePath);
if (fileAccess == FileAccess.Read)
{
if (item == null)
{
return(null);
}
item = WaitForReal(item, 25000);
Log.Trace($"Opening {filePath} for Read");
if (item.Length < SmallFileSizeLimit)
{
return(SmallFilesCache.OpenReadWithDownload(item));
}
var result = SmallFilesCache.OpenReadCachedOnly(item);
if (result != null)
{
return(result);
}
Interlocked.Increment(ref downloadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
var buffered = new BufferedAmazonBlockReader(item, amazon);
buffered.OnClose = () =>
{
Interlocked.Decrement(ref downloadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
};
return(buffered);
}
if (item == null || item.Length == 0)
{
#if TRACE
Log.Trace($"Creating {filePath} as New because mode:{mode} and {((item == null) ? "item is null" : "length is 0")}");
#endif
var dir = Path.GetDirectoryName(filePath);
var name = Path.GetFileName(filePath);
var dirItem = GetItem(dir);
item = FSItem.MakeUploading(filePath, Guid.NewGuid().ToString(), dirItem.Id, 0);
var file = UploadService.OpenNew(item);
Interlocked.Increment(ref uploadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
itemsTreeCache.Add(item);
return(file);
}
if (item == null)
{
return(null);
}
item = WaitForReal(item, 25000);
if ((mode == FileMode.Create || mode == FileMode.Truncate) && item.Length > 0)
{
#if TRACE
Log.Trace($"Opening {filePath} as Truncate because mode:{mode} and length {item.Length}");
#endif
item.Length = 0;
SmallFilesCache.Delete(item);
item.MakeUploading();
var file = UploadService.OpenTruncate(item);
Interlocked.Increment(ref uploadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
return(file);
}
if (mode == FileMode.Open || mode == FileMode.Append || mode == FileMode.OpenOrCreate)
{
#if TRACE
Log.Trace($"Opening {filePath} as ReadWrite because mode:{mode} and length {item.Length}");
#endif
if (item.Length < SmallFileSizeLimit)
{
var file = SmallFilesCache.OpenReadWrite(item);
file.OnChangedAndClosed = (it, path) =>
{
it.LastWriteTime = DateTime.UtcNow;
Interlocked.Increment(ref uploadingCount);
OnStatisticsUpdated?.Invoke(downloadingCount, uploadingCount);
if (!it.IsUploading)
{
it.MakeUploading();
var olditemPath = Path.Combine(SmallFilesCache.CachePath, item.Id);
var newitemPath = Path.Combine(UploadService.CachePath, item.Id);
if (File.Exists(newitemPath))
{
File.Delete(newitemPath);
}
SymbolicLink.CreateFile(GetRelativePath(olditemPath, Path.GetDirectoryName(newitemPath)), newitemPath);
SmallFilesCache.AddExisting(it);
}
UploadService.AddOverwrite(it);
};
return(file);
}
Log.Warn("File is too big for ReadWrite: " + filePath);
}
return(null);
}
