static private void AddNewDriveItem(FileMetadata_Info newdata)
{
ItemInfo value;
if (newdata == null)
{
return;
}
if (newdata.status == "AVAILABLE")
{
// exist item
if (DriveTree.TryGetValue(newdata.id, out value))
{
if (value.info != null)
{
foreach (var p in value.info.parents)
{
ItemInfo value2;
if (DriveTree.TryGetValue(p, out value2))
{
ItemInfo outvalue;
while (!value2.children.TryRemove(value.info.id, out outvalue))
{
if (!value2.children.TryGetValue(value.info.id, out outvalue))
{
break;
}
}
}
}
}
value.info = newdata;
}
else
{
DriveTree[newdata.id] = new ItemInfo(newdata);
}
foreach (var p in newdata.parents)
{
if (DriveTree.TryGetValue(p, out value))
{
value.children[newdata.id] = DriveTree[newdata.id];
}
else
{
DriveTree[p] = new ItemInfo(null);
DriveTree[p].children[newdata.id] = DriveTree[newdata.id];
}
}
if (newdata.isRoot ?? false)
{
root_id = newdata.id;
}
}
else if (newdata.status == "TRASH" || newdata.status == "PURGED")
{
// deleted item
DeleteDriveItem(newdata);
}
}
public AmazonDriveSeekableStream(AmazonDrive Drive, FileMetadata_Info downitem) : base()
{
this.Drive = Drive;
targetItem = downitem;
FileSize = targetItem.OrignalLength ?? 0;
if (FileSize <= 0)
{
return;
}
slots = new SlotMaster(Drive, downitem, cts.Token);
_Position = 0;
if (downitem.contentProperties?.size > ConfigAPI.FilenameChangeTrickSize && !Regex.IsMatch(downitem.name, "^[\x20-\x7e]*$"))
{
OrgFilename = downitem.name;
Config.Log.LogOut("AmazonDriveSeekableStream : <BIG FILE> temporary filename change");
Config.Log.LogOut("AmazonDriveSeekableStream : orgnal name : " + OrgFilename);
Drive.renameItem(downitem.id, ConfigAPI.temporaryFilename + downitem.id).Wait();
}
lastslot = ((downitem.OrignalLength ?? 0) - 1) / AmazonDriveStreamConfig.slotsize;
lockslot1 = AmazonDriveStreamConfig.lockslotfirstnum;
lockslot2 = lastslot - AmazonDriveStreamConfig.lockslotlastnum;
if (lockslot2 < lockslot1)
{
lockslot2 = lockslot1;
}
slots.CreateTask(0);
slots.CreateTask(lockslot2);
slots.CreateTask(lockslot1);
slots.CreateTask(lockslot1 + AmazonDriveStreamConfig.preforwardnum);
}
public SlotTask(AmazonDrive Drive, FileMetadata_Info targetItem, CancellationToken ct = default(CancellationToken))
{
cts = CancellationTokenSource.CreateLinkedTokenSource(ct, Internal_cts.Token);
this.Drive = Drive;
this.targetItem = targetItem;
lastslot = ((targetItem.OrignalLength ?? 0) - 1) / AmazonDriveStreamConfig.slotsize;
}
public static async Task <bool> EncryptFilename(string uploadfilename, string enckey, string checkpoint, CancellationToken ct = default(CancellationToken))
{
int retry = 30;
while (--retry > 0)
{
var child = (await GetChanges(checkpoint, ct).ConfigureAwait(false)).Where(x => x.name.Contains(uploadfilename)).LastOrDefault();
if (child?.status == "AVAILABLE")
{
Config.Log.LogOut("EncryptFilename");
using (var ms = new MemoryStream())
{
byte[] plain = Encoding.UTF8.GetBytes(enckey);
ms.Write(plain, 0, plain.Length);
ms.Position = 0;
using (var enc = new CryptCTR.AES256CTR_CryptStream(ms, child.id))
{
byte[] buf = new byte[ms.Length];
enc.Read(buf, 0, buf.Length);
string cryptname = "";
foreach (var c in buf)
{
cryptname += (char)('\u2800' + c);
}
int retry2 = 30;
FileMetadata_Info reItem = null;
while (--retry2 > 0)
{
try
{
if (reItem == null)
{
reItem = await Drive.renameItem(id : child.id, newname : cryptname, ct : ct).ConfigureAwait(false);
}
}
catch { }
var child2 = (await GetChanges(checkpoint: ChangeCheckpoint, ct: ct).ConfigureAwait(false)).Where(x => x.name.Contains(cryptname)).LastOrDefault();
if (reItem != null && child2?.status == "AVAILABLE")
{
AmazonDriveTree[reItem.id].IsEncrypted = CryptMethods.Method1_CTR;
AmazonDriveTree[reItem.id].ReloadCryptedMethod1();
break;
}
await Task.Delay(2000).ConfigureAwait(false);
}
return(true);
}
}
}
await Task.Delay(2000).ConfigureAwait(false);
}
return(false);
}
public static string DecryptFilename(FileMetadata_Info downloaditem)
{
using (var ms = new MemoryStream())
{
string cryptname = downloaditem?.name;
if (string.IsNullOrEmpty(cryptname))
{
return(null);
}
byte[] buf = new byte[cryptname.Length];
int i = 0;
foreach (var c in cryptname)
{
if (c < '\u2800' || c > '\u28ff')
{
return(null);
}
buf[i++] = (byte)(c - '\u2800');
}
ms.Write(buf, 0, i);
ms.Position = 0;
using (var dec = new CryptCTR.AES256CTR_CryptStream(ms, downloaditem.id))
{
byte[] plain = new byte[i];
dec.Read(plain, 0, plain.Length);
var plainname = Encoding.UTF8.GetString(plain);
if (plainname.IndexOfAny(Path.GetInvalidFileNameChars()) < 0)
{
if (Regex.IsMatch(plainname ?? "", ".*?\\.[a-z0-9]{8}$"))
{
return(plainname);
}
else
{
return(null);
}
}
else
{
return(null);
}
}
}
}
static private void DeleteDriveItem(FileMetadata_Info deldata)
{
ItemInfo value;
if (deldata == null)
{
return;
}
if (DriveTree.TryGetValue(deldata.id, out value))
{
var children = value.children.Values.ToArray();
foreach (var child in children)
{
DeleteDriveItem(child.info);
}
ItemInfo outitem;
while (!DriveTree.TryRemove(deldata.id, out outitem))
{
if (!DriveTree.TryGetValue(deldata.id, out outitem))
{
break;
}
}
}
foreach (var p in deldata.parents)
{
if (DriveTree.TryGetValue(p, out value))
{
ItemInfo outvalue;
while (!value.children.TryRemove(deldata.id, out outvalue))
{
if (!value.children.TryGetValue(deldata.id, out outvalue))
{
break;
}
}
}
}
}
static string FindItemsID(string[] path_str, FileMetadata_Info root = null)
{
if (path_str.Length == 0)
{
return(root?.id);
}
while (path_str.Length > 0 && string.IsNullOrEmpty(path_str.First()))
{
path_str = path_str.Skip(1).ToArray();
}
if (path_str.Length == 0)
{
return(root?.id);
}
if (root == null)
{
root = DriveData.AmazonDriveTree[DriveData.AmazonDriveRootID].info;
}
var children = DriveData.AmazonDriveTree[root.id].children.Select(x => x.Value);
foreach (var c in children)
{
if (c.DisplayName == path_str[0])
{
if (c.info.kind == "FOLDER")
{
return(FindItemsID(path_str.Skip(1).ToArray(), c.info));
}
else
{
return(null);
}
}
}
return(null);
}
public RemoteItemInfo(FileMetadata_Info info, string path, string name)
{
this.info = info;
this.path = path;
this.name = name;
}
static FileMetadata_Info[] FindItems(string[] path_str, bool recursive = false, FileMetadata_Info root = null)
{
List <FileMetadata_Info> ret = new List <FileMetadata_Info>();
if (root == null)
{
root = DriveData.AmazonDriveTree[DriveData.AmazonDriveRootID].info;
}
if (!(path_str?.Length > 0))
{
ret.Add(root);
return(ret.ToArray());
}
while (path_str.Length > 0 && string.IsNullOrEmpty(path_str.First()))
{
path_str = path_str.Skip(1).ToArray();
}
if (path_str.Length == 0)
{
ret.Add(root);
return(ret.ToArray());
}
var children = DriveData.AmazonDriveTree[root.id].children.Select(x => x.Value);
foreach (var c in children)
{
if (c.DisplayName == path_str[0]
||
((path_str[0].Contains('*') || path_str[0].Contains('?')) &&
Regex.IsMatch(c.DisplayName, Regex.Escape(path_str[0]).Replace("\\*", ".*").Replace("\\?", "."))))
{
if (c.info.kind == "FOLDER")
{
ret.AddRange(FindItems((recursive && path_str[0] == "*") ? path_str : path_str.Skip(1).ToArray(), recursive, c.info));
}
else
{
if (path_str[0] == c.DisplayName
||
(((path_str[0].Contains('*') || path_str[0].Contains('?')) &&
Regex.IsMatch(c.DisplayName, Regex.Escape(path_str[0]).Replace("\\*", ".*").Replace("\\?", ".")))))
{
ret.Add(c.info);
}
}
}
}
if (recursive)
{
ret.Sort((x, y) => (DriveData.GetFullPathfromId(x.id)).CompareTo(DriveData.GetFullPathfromId(y.id)));
return(ret.ToArray());
}
else
{
ret.Sort((x, y) => x.name.CompareTo(y.name));
return(ret.ToArray());
}
}
public SlotMaster(AmazonDrive Drive, FileMetadata_Info downitem, CancellationToken ct = default(CancellationToken))
{
cts = CancellationTokenSource.CreateLinkedTokenSource(cts_internal.Token, ct);
this.Drive = Drive;
targetItem = downitem;
lastslot = ((downitem.OrignalLength ?? 0) - 1) / AmazonDriveStreamConfig.slotsize;
lockslot1 = AmazonDriveStreamConfig.lockslotfirstnum;
lockslot2 = lastslot - AmazonDriveStreamConfig.lockslotlastnum;
if (lockslot2 < lockslot1)
{
lockslot2 = lockslot1;
}
int extraslot = 0;
Task.Run(() =>
{
foreach (var newitem in SlotBuffer.GetConsumingEnumerable(cts.Token))
{
try
{
slot.GetOrAdd(newitem.Key, newitem.Value);
while (slot.Count > AmazonDriveStreamConfig.slotbacklog + extraslot)
{
Task.Delay(100, cts.Token).Wait(cts.Token);
}
if (cts.Token.IsCancellationRequested)
{
return;
}
}
catch { }
}
}, cts.Token);
Task.Run(() =>
{
while (!cts.Token.IsCancellationRequested)
{
try
{
const int slotnumc = AmazonDriveStreamConfig.slotbacklog;
// slotが多すぎるのでいらないものから消す
if (slot.Count > slotnumc)
{
var pos = slot
.OrderByDescending(x => x.Value.Age)
.OrderBy(x => x.Value.ReadAge)
.First().Key;
var s = StartLock;
if (s != null)
{
pos = s.Value;
}
//Config.Log.LogOut(string.Format("AmazonDriveStream : Removing slots current pos {0}", pos));
var deleteitem = slot
.Where(x => !(x.Key >= StartLock && x.Key <= EndLock))
.Where(x => x.Key > lockslot1 && x.Key < lockslot2);
deleteitem = deleteitem
.Where(x => x.Key <pos - AmazonDriveStreamConfig.slotkeepold || x.Key> pos + AmazonDriveStreamConfig.slotbacklog * 2)
.OrderByDescending(x => x.Value.ReadAge)
.Take(slot.Count - slotnumc).ToArray();
foreach (var item in deleteitem)
{
MemoryStreamSlot o;
if (slot.TryRemove(item.Key, out o))
{
//Config.Log.LogOut(string.Format("AmazonDriveStream : Remove slot {0} pos {1:#,0} len {2:#,0}", item.Key, o.Offset, o.Length));
if (!(item.Key >= StartLock && item.Key <= EndLock))
{
o.Dispose();
}
else
{
slot.GetOrAdd(item.Key, o);
}
}
}
extraslot = slot.Count - slotnumc;
}
// 終了したタスクを除去する
if (Tasks.Any(x => x.Done))
{
var deleteitem = Tasks.Where(x => x.Done).ToArray();
foreach (var item in deleteitem)
{
SlotTask o;
if (Tasks.TryTake(out o))
{
//Config.Log.LogOut(string.Format("AmazonDriveStream : Remove end Task slot {0}", o.ReadingSlotno));
o.Dispose();
}
}
}
// 走りすぎているスレットを消す
if (accesslog.Count() > 0)
{
var min_point = accesslog.OrderByDescending(x => x.Value).Take(1).Min(x => x.Key);
var max_point = min_point + AmazonDriveStreamConfig.slotbacklog;
min_point = Math.Max(min_point - AmazonDriveStreamConfig.slotnearby * 2, 0);
max_point = Math.Min(max_point + AmazonDriveStreamConfig.slotnearby * 2, lastslot);
//Config.Log.LogOut(string.Format("AmazonDriveStream : min_point {0}", min_point));
//Config.Log.LogOut(string.Format("AmazonDriveStream : max_point {0}", max_point));
if (min_point < lockslot2 && Tasks.Any(x => x.ReadingSlotno < min_point && x.ReadingSlotno > lockslot1))
{
var deleteitem = Tasks.Where(x => x.ReadingSlotno <min_point && x.ReadingSlotno> lockslot1).ToList();
SlotTask o;
while (deleteitem.Count > 0 && Tasks.TryTake(out o))
{
if (deleteitem.Contains(o))
{
//Config.Log.LogOut(string.Format("AmazonDriveStream : Remove1 Task slot {0} too far({1})", o.ReadingSlotno, min_point));
deleteitem.Remove(o);
o.Dispose();
}
else
{
Tasks.Add(o);
}
}
}
if (Tasks.Count == 1)
{
Tasks.First().leadThread = true;
}
else
{
foreach (var item in Tasks)
{
item.leadThread = false;
}
}
if (max_point > lockslot1 && Tasks.Any(x => x.ReadingSlotno > max_point && x.ReadingSlotno < lockslot2))
{
var deleteitem = Tasks.Where(x => x.ReadingSlotno > max_point && x.ReadingSlotno < lockslot2).ToList();
SlotTask o;
while (deleteitem.Count > 0 && Tasks.TryTake(out o))
{
if (deleteitem.Contains(o))
{
if (o.leadThread)
{
//Config.Log.LogOut(string.Format("AmazonDriveStream : LeadThread Task slot {0} too far({1})", o.ReadingSlotno, min_point));
deleteitem.Remove(o);
Tasks.Add(o);
}
else
{
//Config.Log.LogOut(string.Format("AmazonDriveStream : Remove2 Task slot {0} too far({1})", o.ReadingSlotno, min_point));
deleteitem.Remove(o);
o.Dispose();
}
}
else
{
Tasks.Add(o);
}
}
}
}
}
catch { }
//Config.Log.LogOut(string.Format("AmazonDriveStream : Tasks {0} slots {1}", Tasks.Count, slot.Count));
//Config.Log.LogOut(string.Format("AmazonDriveStream : slot {0}", string.Join(",",Tasks.Select(x=>x.ReadingSlotno.ToString()))));
Task.Delay(500, cts.Token).Wait(cts.Token);
}
}, cts.Token);
}
public ItemInfo(FileMetadata_Info thisdata)
{
info = thisdata;
}
public async Task <Stream> downloadFile(FileMetadata_Info target, long?from = null, long?to = null, string enckey = null, bool autodecrypt = true, CancellationToken ct = default(CancellationToken))
{
string id = target.id;
string filename = target.name;
CryptMethods Encrypted = CryptMethods.Method0_Plain;
if (enckey != null)
{
Encrypted = CryptMethods.Method1_CTR;
}
else
{
if (filename.StartsWith(Config.CarotDAV_CryptNameHeader))
{
Encrypted = CryptMethods.Method2_CBC_CarotDAV;
enckey = "";
}
else if (Regex.IsMatch(filename, ".*?\\.[a-z0-9]{8}\\.enc$"))
{
Encrypted = CryptMethods.Method1_CTR;
enckey = Path.GetFileNameWithoutExtension(filename);
}
else if (Regex.IsMatch(filename, "^[\u2800-\u28ff]+$"))
{
enckey = DriveData.DecryptFilename(target);
if (enckey != null)
{
Encrypted = CryptMethods.Method1_CTR;
}
}
if (enckey == null)
{
Encrypted = CryptMethods.Method0_Plain;
}
}
if (!autodecrypt)
{
Encrypted = CryptMethods.Method0_Plain;
}
Config.Log.LogOut("\t[downloadFile] " + id);
string error_str = "";
var client = new HttpClient();
client.Timeout = TimeSpan.FromDays(1);
try
{
long?fix_from = from, fix_to = to;
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Authkey.access_token);
if (from != null || to != null)
{
if (Encrypted == CryptMethods.Method2_CBC_CarotDAV)
{
if (fix_from != null)
{
// ひとつ前のブロックから要求する
fix_from -= CryptCarotDAV.BlockSizeByte;
if (fix_from < CryptCarotDAV.BlockSizeByte)
{
// 先頭ブロックを取得するときはファイルの先頭から
fix_from = 0;
}
else
{
// ブロックにアライメントを合わせる
fix_from -= ((fix_from - 1) % CryptCarotDAV.BlockSizeByte + 1);
// 途中のブロックを要求された場合は、ヘッダをスキップ
fix_from += CryptCarotDAV.CryptHeaderByte;
}
}
if (fix_to != null)
{
if (fix_to >= target.OrignalLength)
{
// 末尾まで読み込むときは、ハッシュチェックのために最後まで読み込む
fix_to = null;
}
else
{
// オリジナルの位置を、暗号化済みの位置に変更
fix_to += CryptCarotDAV.CryptHeaderByte;
}
}
if (fix_from != null || fix_to != null)
{
client.DefaultRequestHeaders.Range = new RangeHeaderValue(fix_from, fix_to);
}
}
else
{
client.DefaultRequestHeaders.Range = new RangeHeaderValue(from, to);
}
}
string url = Config.contentUrl + "nodes/" + id + "/content?download=false";
var response = await client.GetAsync(
url,
HttpCompletionOption.ResponseHeadersRead,
ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
if (Encrypted == CryptMethods.Method1_CTR)
{
return(new CryptCTR.AES256CTR_CryptStream(new ThrottleDownloadStream(new HashStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), new MD5CryptoServiceProvider()), ct),
enckey,
from ?? 0));
}
else if (Encrypted == CryptMethods.Method2_CBC_CarotDAV)
{
return(new CryptCarotDAV.CryptCarotDAV_DecryptStream(new ThrottleDownloadStream(new HashStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), new MD5CryptoServiceProvider()), ct),
from ?? 0,
fix_from ?? 0,
target.contentProperties?.size ?? -1
));
}
else
{
return(new ThrottleDownloadStream(new HashStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), new MD5CryptoServiceProvider()), ct));
}
}
catch (HttpRequestException ex)
{
error_str = ex.Message;
Config.Log.LogOut("\t[downloadFile] " + error_str);
throw;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
error_str = ex.ToString();
Config.Log.LogOut("\t[downloadFile] " + error_str);
throw;
}
throw new SystemException("fileDownload failed. " + error_str);
}