public AmazonDriveBaseStream(AmazonDrive Drive, FileMetadata_Info downitem, bool autodecrypt = true, JobControler.Job parentJob = null) : base() { this.Drive = Drive; targetItem = downitem; FileSize = targetItem.OrignalLength ?? 0; this.autodecrypt = autodecrypt; if (parentJob == null) { downloadJob = JobControler.CreateNewJob( type: JobControler.JobClass.Download, info: new JobControler.Job.SubInfo { type = JobControler.Job.SubInfo.SubType.DownloadFile, size = downitem.contentProperties?.size ?? 0, }); downloadJob.DisplayName = downitem.name; downloadJob.ProgressStr = "wait for download"; downloadJob.Progress = -1; ct = downloadJob.ct; } else { ct = parentJob.ct; } if (FileSize < 0) { return; } if (downitem.contentProperties?.size > ConfigAPI.FilenameChangeTrickSize && !Regex.IsMatch(downitem.name, "^[\x20-\x7e]*$")) { Interlocked.Increment(ref Config.AmazonDriveTempCount); OrgFilename = targetItem.name; Config.Log.LogOut("AmazonDriveBaseStream : <BIG FILE> temporary filename change"); Config.Log.LogOut("AmazonDriveBaseStream : orgnal name : " + OrgFilename); Drive.renameItem(targetItem.id, ConfigAPI.temporaryFilename + targetItem.id).Wait(); } InitStream(); if (parentJob == null) { JobControler.Run(downloadJob, (j) => { downloadJob.ProgressStr = "download..."; downloadJob.Wait(ct: ct); downloadJob.Progress = 1; downloadJob.ProgressStr = "done."; }); } }
private static Job InitServer(Job master) { var loadJob = JobControler.CreateNewJob(JobClass.Normal, depends: master); loadJob.WeekDepend = true; loadJob.DisplayName = "Load server list"; JobControler.Run(loadJob, (j) => { j.Progress = -1; j.ProgressStr = "Loading..."; RemoteServerFactory.Restore(); while (!RemoteServerFactory.ServerList.Values.All(x => x.IsReady)) { loadJob.Ct.ThrowIfCancellationRequested(); Task.Delay(500).Wait(loadJob.Ct); } j.Progress = 1; j.ProgressStr = "Done."; }); return(loadJob); }
public void StartDownload(long slotno, ConcurrentDictionary <long, MemoryStreamSlot> slot, BlockingCollection <KeyValuePair <long, MemoryStreamSlot> > SlotBuffer) { int timeout = (int)(1000 * (double)SeekableStreamConfig.shortbuflen / (128 * 1024)); long start = slotno * SeekableStreamConfig.slotsize; long length = SeekableStreamConfig.slotsize; if (start + length > (targetItem.Size ?? 0)) { length = (targetItem.Size ?? 0) - start; } if (length <= 0) { return; } done = false; readslot = slotno; //CancellationTokenSource cancel_cts = new CancellationTokenSource(); //var cts_1 = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancel_cts.Token); //Config.Log.LogOut(string.Format("AmazonDriveStream : start to download slot {0} offset {1:#,0} - ", slotno, start)); var cjob = new JobForToken(cts.Token); djob?.Cancel(); djob = targetItem.DownloadItemRawJob(start, hidden: true, prevJob: cjob as Job); var job = JobControler.CreateNewJob <Stream>(JobClass.RemoteDownload, depends: djob); job.DisplayName = "Download slot :" + slotno + " item : " + targetItem.Name; job.ProgressStr = "wait for prepare..."; job.ForceHidden = true; job.DoAlways = true; var cts_1 = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, job.Ct); JobControler.Run <Stream>(job, (j) => { var result = j.ResultOfDepend[0]; if (result == null || !result.TryGetTarget(out var stream)) { return; } using (stream) { if ((j as Job <Stream>).IsError) { (j as Job <Stream>).Cancel(); StartDownload(slotno, slot, SlotBuffer); return; } if ((j as Job <Stream>).IsCanceled) { done = true; return; } while (slotno <= lastslot) { //Config.Log.LogOut(string.Format("AmazonDriveStream : download slot {0}", slotno)); cts_1.Token.ThrowIfCancellationRequested(); // すでに取得済みかチェック MemoryStreamSlot o; if (slot.TryGetValue(slotno, out o)) { cts.Cancel(); (j as Job <Stream>).Cancel(); } readslot = slotno; byte[] buffer = new byte[(length > SeekableStreamConfig.shortbuflen) ? SeekableStreamConfig.shortbuflen : length]; var mem = new MemoryStream(); var stime = DateTime.Now; int loopdelay = 0; while (length > 0) { cts_1.Token.ThrowIfCancellationRequested(); var tret = stream.ReadAsync(buffer, 0, buffer.Length).ContinueWith(task2 => { if (!task2.Wait(timeout, cts_1.Token)) { cts_1.Token.ThrowIfCancellationRequested(); //Config.Log.LogOut(string.Format("AmazonDriveStream : wait timeout slot {0}", slotno)); throw new IOException("transfer timeout1"); } var ret = task2.Result; mem.Write(buffer, 0, ret); length -= ret; loopdelay += 5; if ((DateTime.Now - stime).TotalMilliseconds > timeout + loopdelay) { //Config.Log.LogOut(string.Format("AmazonDriveStream : transfer timeout slot {0}", slotno)); throw new IOException("transfer timeout2"); } if (buffer.Length > length) { buffer = new byte[length]; } }, cts_1.Token, TaskContinuationOptions.AttachedToParent | TaskContinuationOptions.NotOnCanceled, TaskScheduler.Default) .Wait(timeout, cts_1.Token); if (!tret) { //Config.Log.LogOut(string.Format("AmazonDriveStream : wait2 timeout slot {0}", slotno)); throw new IOException("transfer timeout3"); } } cts_1.Token.ThrowIfCancellationRequested(); var newslot = new MemoryStreamSlot(mem, start); if (leadThread) { while (leadThread) { cts_1.Token.ThrowIfCancellationRequested(); if (SlotBuffer.TryAdd(new KeyValuePair <long, MemoryStreamSlot>(slotno, newslot), 500, cts_1.Token)) { break; } } if (!leadThread) { cts_1.Token.ThrowIfCancellationRequested(); if (slot.GetOrAdd(slotno, newslot) != newslot) { cts.Cancel(); (j as Job <Stream>).Cancel(); } } } else { if (slot.GetOrAdd(slotno, newslot) != newslot) { cts.Cancel(); (j as Job <Stream>).Cancel(); } } cts_1.Token.ThrowIfCancellationRequested(); start = ++slotno * SeekableStreamConfig.slotsize; length = SeekableStreamConfig.slotsize; if (start + length > (targetItem.Size ?? 0)) { length = (targetItem.Size ?? 0) - start; } if (length <= 0) { return; } } } });
private void DoSearch() { if (radioButton_selected.Checked && SelectedItems == null) { return; } button_seach.Enabled = false; button_cancel.Enabled = true; button_showresult.Enabled = false; var typeAll = radioButton_typeAll.Checked; var typeFolder = radioButton_typeFolder.Checked; var typeFile = radioButton_typeFile.Checked; var selectall = radioButton_SearchAll.Checked; var selectitem = radioButton_selected.Checked; var selecttree = radioButton_SerachFolder.Checked; var treepath = textBox_SearchFolder.Text; var searchStr = comboBox_name.Text; var strstarts = radioButton_startswith.Checked; var strends = radioButton_endswith.Checked; var strcontain = radioButton_contain.Checked; var strregex = checkBox_regex.Checked; var strcase = checkBox_case.Checked; var SizeOver = checkBox_Over.Checked; var SizeUnder = checkBox_Under.Checked; var Over = numericUpDown_over.Value; var Under = numericUpDown_under.Value; var modifiedFromEnable = dateTimePicker_modifiedFrom.Checked; var modifiedToEnable = dateTimePicker_modifiedTo.Checked; var createdFromEnable = dateTimePicker_createdFrom.Checked; var createdToEnable = dateTimePicker_createdTo.Checked; var modifiedFrom = dateTimePicker_modifiedFrom.Value; var modifiedTo = dateTimePicker_modifiedTo.Value; var createdFrom = dateTimePicker_createdFrom.Value; var createdTo = dateTimePicker_createdTo.Value; progressBar1.Style = ProgressBarStyle.Marquee; label_result.Text = "wait for system..."; SearchJob?.Cancel(); SearchJob = JobControler.CreateNewJob(); SearchJob.DisplayName = "Search"; JobControler.Run(SearchJob, (j) => { j.ProgressStr = "Create index..."; j.Progress = -1; TSviewCloudConfig.Config.Log.LogOut("[Search] start"); var sw = new System.Diagnostics.Stopwatch(); try { List <IRemoteItem> initselection = new List <IRemoteItem>(); List <IRemoteItem> selection = new List <IRemoteItem>(); if (selectall) { initselection.AddRange(RemoteServerFactory.ServerList.Values.Select(x => x[""])); } if (selectitem) { initselection.AddRange(SelectedItems); } if (selecttree) { initselection.Add(RemoteServerFactory.PathToItem(treepath).Result); } synchronizationContext.Post((o) => { label_result.Text = o as string; }, "Prepare items..."); TSviewCloudConfig.Config.Log.LogOut("[Search] Create index"); sw.Start(); Parallel.ForEach( initselection, new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Math.Ceiling((Environment.ProcessorCount * 0.75) * 1.0)) }, () => new List <IRemoteItem>(), (x, state, local) => { if (x == null) { return(local); } var item = RemoteServerFactory.PathToItem(x.FullPath).Result; if (item == null) { return(local); } local.AddRange(GetItems(item)); return(local); }, (result) => { lock (selection) selection.AddRange(result); } ); synchronizationContext.Post((o) => { label_result.Text = o as string; }, "Prepare items done."); sw.Stop(); var itemsearch_time = sw.Elapsed; var search = selection.AsParallel(); if (typeFile) { search = search.Where(x => x.ItemType == RemoteItemType.File); } if (typeFolder) { search = search.Where(x => x.ItemType == RemoteItemType.Folder); } if (strregex) { if (!strcase) { search = search.Where(x => Regex.IsMatch(x.Name ?? "", searchStr)); } else { search = search.Where(x => Regex.IsMatch(x.Name ?? "", searchStr, RegexOptions.IgnoreCase)); } } else { if (!strcase) { if (strstarts) { search = search.Where(x => (x.Name?.StartsWith(searchStr) ?? searchStr == "")); } if (strends) { search = search.Where(x => (x.Name?.EndsWith(searchStr) ?? searchStr == "")); } if (strcontain) { search = search.Where(x => (x.Name?.IndexOf(searchStr) >= 0)); } } else { search = search.Where(x => ( System.Globalization.CultureInfo.CurrentCulture.CompareInfo.IndexOf( x.Name ?? "", searchStr, System.Globalization.CompareOptions.IgnoreCase | System.Globalization.CompareOptions.IgnoreKanaType | System.Globalization.CompareOptions.IgnoreWidth | System.Globalization.CompareOptions.IgnoreNonSpace | System.Globalization.CompareOptions.IgnoreSymbols ) >= 0)); } } if (SizeOver) { search = search.Where(x => (x.Size ?? 0) > Over); } if (SizeUnder) { search = search.Where(x => (x.Size ?? 0) < Under); } if (modifiedFromEnable) { search = search.Where(x => x.ModifiedDate > modifiedFrom); } if (modifiedToEnable) { search = search.Where(x => x.ModifiedDate < modifiedTo); } if (createdFromEnable) { search = search.Where(x => x.CreatedDate > createdFrom); } if (createdToEnable) { search = search.Where(x => x.CreatedDate < createdTo); } synchronizationContext.Post((o) => { label_result.Text = o as string; }, "Search..."); j.ProgressStr = "Search..."; TSviewCloudConfig.Config.Log.LogOut("[Search] Search"); sw.Restart(); SearchResult = search.ToArray(); sw.Stop(); var filteritem_time = sw.Elapsed; j.Progress = 1; j.ProgressStr = "Found : " + SearchResult.Count().ToString(); synchronizationContext.Post((o) => { label_result.Text = o as string; button_seach.Enabled = true; button_cancel.Enabled = false; button_showresult.Enabled = true; progressBar1.Style = ProgressBarStyle.Continuous; SearchJob = null; }, string.Format("Found : {0}, Index {1} search {2}", SearchResult.Count(), itemsearch_time, filteritem_time)); TSviewCloudConfig.Config.Log.LogOut("[Search] found " + SearchResult.Count().ToString()); } catch { TSviewCloudConfig.Config.Log.LogOut("[Search] Abort"); synchronizationContext.Post((o) => { label_result.Text = "abort"; button_seach.Enabled = true; button_cancel.Enabled = false; progressBar1.Style = ProgressBarStyle.Continuous; SearchJob = null; }, null); } }); }
private static int RunUpload(string[] targetArgs, string[] paramArgs) { string remotePath; string localPath; if (targetArgs.Length < 3) { Console.Error.WriteLine("upload needs more 2 arguments."); Console.Error.WriteLine("upload (localpath) (remotetarget)"); return(0); } remotePath = targetArgs[2]; remotePath = remotePath.Replace('\\', '/'); localPath = targetArgs[1]; if (!localPath.Contains(':') && !localPath.StartsWith(@"\\")) { localPath = Path.GetFullPath(localPath); } if (!localPath.StartsWith(@"\\")) { localPath = ItemControl.GetLongFilename(localPath); } Console.Error.WriteLine("upload"); Console.Error.WriteLine("remote: " + remotePath); Console.Error.WriteLine("local: " + ItemControl.GetOrgFilename(localPath)); bool createFolder = false; foreach (var p in paramArgs) { switch (p) { case "createfolder": Console.Error.WriteLine("(--createfolder: create folders)"); createFolder = true; break; } } var job = JobControler.CreateNewJob(JobClass.ControlMaster); job.DisplayName = "Upload"; JobControler.Run(job, (j) => { try { var j2 = InitServer(j); j2.Wait(ct: j.Ct); var target = FindItems(remotePath, ct: j.Ct); IRemoteItem remote = null; if (target.Count() != 1 && !createFolder) { Console.Error.WriteLine("upload needs 1 remote target item."); j.ResultAsObject = 2; return; } if (target.Count() == 0 && createFolder) { Console.Error.WriteLine("Create new folders."); remote = CreateFolders(remotePath, j); if (remote == null) { Console.Error.WriteLine("make folder failed."); j.ResultAsObject = 3; return; } } else { remote = target.First(); } ConsoleJobDisp.Run(); if (File.Exists(localPath)) { ItemControl.UploadFiles(remote, new[] { localPath }, true, j); } else if (Directory.Exists(localPath)) { if (!localPath.EndsWith(":\\") && localPath.EndsWith("\\")) { localPath = localPath.TrimEnd('\\'); } ItemControl.UploadFolder(remote, localPath, true, j); } else { Console.Error.WriteLine("upload localitem not found."); j.ResultAsObject = 2; return; } while (JobControler.JobTypeCount(JobClass.Upload) > 0) { JobControler.JobList().Where(x => x.JobType == JobClass.Upload).FirstOrDefault()?.Wait(ct: j.Ct); } var SaveConfigJob = JobControler.CreateNewJob(TSviewCloudPlugin.JobClass.Save); SaveConfigJob.DisplayName = "Save server list"; JobControler.Run(SaveConfigJob, (j3) => { j3.Progress = -1; j3.ProgressStr = "Save..."; TSviewCloudConfig.Config.Save(); RemoteServerFactory.Save(); j3.Progress = 1; j3.ProgressStr = "Done."; }); SaveConfigJob.Wait(); job.ResultAsObject = 0; } catch (OperationCanceledException) { job.ResultAsObject = -1; } catch (Exception ex) { Console.Error.WriteLine("error: " + ex.ToString()); job.ResultAsObject = 1; } }); try { job.Wait(ct: job.Ct); } catch (OperationCanceledException) { } TSviewCloudConfig.Config.ApplicationExit = true; Console.Out.Flush(); return((job.ResultAsObject as int?) ?? -1); }
private static int RunDownload(string [] targetArgs, string[] paramArgs) { string remotePath; string localPath; if (targetArgs.Length < 3) { Console.Error.WriteLine("download needs more 2 arguments."); Console.Error.WriteLine("download (remotepath) (localpath)"); return(0); } remotePath = targetArgs[1]; remotePath = remotePath.Replace('\\', '/'); localPath = targetArgs[2]; if (!localPath.Contains(':') && !localPath.StartsWith(@"\\")) { localPath = Path.GetFullPath(localPath); } if (!localPath.StartsWith(@"\\")) { localPath = ItemControl.GetLongFilename(localPath); } Console.Error.WriteLine("download"); Console.Error.WriteLine("remote: " + remotePath); Console.Error.WriteLine("local: " + ItemControl.GetOrgFilename(localPath)); ReloadType reload = ReloadType.Reload; foreach (var p in paramArgs) { switch (p) { case "cache": Console.Error.WriteLine("(--cache: no reload)"); reload = ReloadType.Cache; break; } } var job = JobControler.CreateNewJob(JobClass.ControlMaster); job.DisplayName = "Download"; JobControler.Run(job, (j) => { try { var j2 = InitServer(j); j2.Wait(ct: j.Ct); var target = FindItems(remotePath, ct: j.Ct, reload: reload); if (target.Count() < 1) { j.ResultAsObject = 2; return; } string remotePathBase = null; if (remotePath.IndexOfAny(new[] { '*', '?' }) < 0) { remotePathBase = ItemControl.GetCommonPath(target); } else { remotePathBase = GetBasePath(remotePath); } target = RemoveDup(target); ConsoleJobDisp.Run(); var j3 = target .Where(x => x.ItemType == RemoteItemType.File) .Select(x => RemoteServerFactory.PathToItem(x.FullPath, ReloadType.Reload).Result) .Select(x => ItemControl.DownloadFile(Path.Combine(localPath, ItemControl.GetLocalFullPath(x.FullPath, remotePathBase)), x, j, true)); var j4 = target .Where(x => x.ItemType == RemoteItemType.Folder) .Select(x => RemoteServerFactory.PathToItem(x.FullPath, ReloadType.Reload).Result) .Select(x => ItemControl.DownloadFolder(Path.GetDirectoryName(Path.Combine(localPath, ItemControl.GetLocalFullPath(x.FullPath, remotePathBase))), new[] { x }, j, true)); foreach (var jx in j3.Concat(j4).ToArray()) { j.Ct.ThrowIfCancellationRequested(); jx.Wait(ct: j.Ct); } job.ResultAsObject = 0; } catch (OperationCanceledException) { job.ResultAsObject = -1; } catch (Exception ex) { Console.Error.WriteLine("error: " + ex.ToString()); job.ResultAsObject = 1; } }); try { job.Wait(ct: job.Ct); } catch (OperationCanceledException) { } TSviewCloudConfig.Config.ApplicationExit = true; Console.Out.Flush(); return((job.ResultAsObject as int?) ?? -1); }
static int RunList(string[] targetArgs, string[] paramArgs) { string remotepath = null; IEnumerable <IRemoteItem> target = null; if (targetArgs.Length > 1) { remotepath = targetArgs[1]; remotepath = remotepath.Replace('\\', '/'); } bool recursive = false; bool showhash = false; ReloadType reload = ReloadType.Reload; foreach (var p in paramArgs) { switch (p) { case "recursive": Console.Error.WriteLine("(--recursive: recursive mode)"); recursive = true; break; case "hash": Console.Error.WriteLine("(--hash: show hash)"); showhash = true; break; case "cache": Console.Error.WriteLine("(--cache: no reload)"); reload = ReloadType.Cache; break; } } var job = JobControler.CreateNewJob(JobClass.ControlMaster); job.DisplayName = "ListItem"; JobControler.Run(job, (j) => { try { var j2 = InitServer(j); j2.Wait(ct: j.Ct); target = FindItems(remotepath, recursive: recursive, ct: j.Ct, reload: reload); if (target.Count() < 1) { j.ResultAsObject = 2; return; } if (remotepath?.Contains("**") ?? true) { recursive = true; } Console.Error.WriteLine("Found : " + target.Count()); foreach (var item in target.OrderBy(x => x.FullPath)) { string detail = ""; if (showhash) { detail = "\t" + item.Hash; } if (recursive) { Console.WriteLine(item.FullPath + "\t" + item.Name + detail); } else { if (item.IsRoot) { Console.WriteLine(item.FullPath + "\t" + item.Name + detail); } else { Console.WriteLine(item.Name + ((item.ItemType == RemoteItemType.Folder) ? "/" : "") + detail); } } } job.ResultAsObject = 0; } catch (OperationCanceledException) { job.ResultAsObject = -1; } catch (Exception ex) { Console.Error.WriteLine("error: " + ex.ToString()); job.ResultAsObject = 1; } }); try { job.Wait(ct: job.Ct); } catch (OperationCanceledException) { } TSviewCloudConfig.Config.ApplicationExit = true; Console.Out.Flush(); return((job.ResultAsObject as int?) ?? -1); }