public void LoadTables() { foreach (Type type in RecordsAssembly.GetTypes()) { TableAttribute attribute = type.GetCustomAttribute <TableAttribute>(); if (attribute != null) { IList tables = (IList)Activator.CreateInstance(typeof(List <>).MakeGenericType(type)); var records = SmartFile.ConvertToRecord(attribute.Path, type); foreach (var record in records) { tables.Add(record); } if (tables.Count > 0) { var field = ITableManager.Instance.GetCache(type); field.SetValue(null, tables); } } } }
void downloader_DownloadCompleted(object sender, FileDownloadCompleteEventArgs e) { this.Text = "文件" + this.m_File.FileName + "文件下载完成100%"; Downloader.Logger(this.Text); byte[] buffer = new byte[e.MemoryStream.Length]; e.MemoryStream.Read(buffer, 0, buffer.Length); DownloadHelper.Instance.WriteSmartFile(this.m_File, buffer); this.files.RemoveAt(0); if (this.files.Count > 0) { m_File = this.files[0]; this.Text = "准备下载文件" + this.m_File.FileName + "..."; this.downloader.DownloadFile(m_File.FileName); } else { DownloadHelper.Instance.WriteLocalUpdateXml(); if (LoginContext.Singleton.Account != null) //已经成功登录 { SLContext.Instance.Shell = new UI.MainPage(); App.Navigation(SLContext.Instance.Shell); } else //未能成功登录 { App.Navigation(PlugContext.LoginForm as UserControl); } } }
public void SmartFileTests() { //Create a test file string tmpFile = System.IO.Path.GetTempFileName(); StreamWriter sw = File.AppendText(tmpFile); sw.WriteLine("Testing, over the river and through the woods"); sw.Close(); SmartFile s = new SmartFile(tmpFile); Assert.IsTrue(s.isValidFile,"Bad File Condition"); Debug.WriteLine(s.hashString); string tmpFile2 = System.IO.Path.GetTempFileName(); StreamWriter sw2 = File.AppendText(tmpFile2); sw2.WriteLine("Testing, over the river and through the woods"); sw2.Close(); SmartFile s2 = new SmartFile(tmpFile2); Assert.IsTrue(s2.isValidFile, "Bad File Condition"); Debug.WriteLine(s2.hashString); //File.AppendText("Testing, over the river and through the woods"); //Assert.IsTrue( // File.Delete(tmpFile2); File.Delete(tmpFile); }
public void UpdateTags(SmartFile sf) { foreach (AutoRankRule rule in rules) { rule.UpdateTag(sf); } }
public void Autorank_Legibility() { AutoRankRule_FilenameReadability autoRankRuler = new AutoRankRule_FilenameReadability(); SmartFile newFile = new SmartFile("C:\\inc\\Riftchan_Notes032714.txt"); autoRankRuler.UpdateTag(newFile); }
public Task(SmartFile.ClientHandler client, String APIendPoint, DateTime created, int status, string type) { this.client = client; this.APIendPoint = APIendPoint; this.created = created; this.status = status; this.type = type; }
private bool InAKeepFolder(SmartFile sf) { if (sf.LastFolderName.StartsWith("1")) { return true; } return false; }
private bool InUnsortedSubFolder(SmartFile sf) { if (sf.fullFileName.ToLower().Contains("unsort")) { return true; } return false; }
public bool IsExits(string url) { SmartFile _smartDb = new SmartFile(OutputFile); if (_smartDb.Lines != null && _smartDb.Lines.Count() > 0 && _smartDb.Lines.Contains(url)) { return(true); } return(false); }
public void Autorank_VowelTest() { AutoRankRule_FilenameReadability autoRankRuler = new AutoRankRule_FilenameReadability(); string[] filenames = Directory.GetFiles("C:\\inc", "*", SearchOption.AllDirectories); foreach (string curFile in filenames) { SmartFile newSmartFile = new SmartFile(curFile); autoRankRuler.UpdateTag(newSmartFile); } }
//Parse file and add necessary tag(s) public override void UpdateTag(SmartFile sf) { //if (sf.fileName.Contains(" - Copy")) if (sf.fileName.ToLower().Contains("copy")) { sf.tags[RuleName] = new Tag(RuleResult, "'Copy' Filename",modifier_IsACopy); return; } sf.tags[RuleName] = new Tag(RuleResult, "not a copy", modifier_NotACopy); return; }
public void Initialize(string smartFilePath, Assembly recordsAssembly) { this.RecordsAssembly = recordsAssembly; if (File.Exists(smartFilePath) == false) { SmartFile = new SmartFile(smartFilePath, new Dictionary <string, SmartTable>()); } else { SmartFile = new SmartFile(smartFilePath); } }
public List <KeyValuePair <string, object[]> > GetPropertiesValues() { List <KeyValuePair <string, object[]> > results = new List <KeyValuePair <string, object[]> >(); foreach (var file in SmartFile.GetSmartTables()[SelectedTable].Files) { var jObject = (JObject)JsonConvert.DeserializeObject(file.Value); object[] properties = Array.ConvertAll <JToken, object>(jObject.PropertyValues().ToArray(), x => x.ToString()); results.Add(new KeyValuePair <string, object[]>(file.FileName, properties)); } return(results); }
private void AddResult() { SmartFile _file = new SmartFile(OutputFile); if (IsAccept(Url) && !IsExits(Url)) { Page = new WebPage(Url); DateTime _time = DateTime.Now; _file.AddLineBefore("{" + _time + "}" + "{Title}{" + Page.Title.ToStandard() + "}"); _file.Save(); _file.AddLineBefore("{" + _time + "}" + "{Url}{" + Url + "}"); _file.Save(); } }
public string[] GetProperties() { var files = SmartFile.GetSmartTables()[SelectedTable].Files; if (files.Count == 0) { return(new string[0]); } var objRef = files[0]; var jObject = (JObject)JsonConvert.DeserializeObject(objRef.Value); return(Array.ConvertAll(jObject.Properties().ToArray(), x => x.Name)); }
private List<SmartFile> GetAllFilesInfolders(string[] searchPaths) { List<SmartFile> sfl = new List<SmartFile>(); foreach (string curPath in searchPaths) { string[] filenames = Directory.GetFiles(curPath, "*", SearchOption.AllDirectories); foreach (string curFile in filenames) { SmartFile newSmartFile = new SmartFile(curFile); sfl.Add(newSmartFile); } } return sfl; }
public override void UpdateTag(SmartFile sf) { if (InAKeepFolder(sf)) { sf.tags[RuleName] = new Tag(RuleName, "In a Keeper folder", modifier_KeepFolder); } if (InUnsortedFolder(sf)) { sf.tags[RuleName] = new Tag(RuleName, "In Unsorted Folder", modifier_UnsortedFolder); } if (InUnsortedSubFolder(sf)) { sf.tags[RuleName] = new Tag(RuleName, "In Unsorted Subfolder", modifier_UnsortedSubFolder); } }
//Check ratio of vowels to letters (confirm a file is *probably* human readable private bool ProbablyLegible(SmartFile sf) { //Require at least 1 character after f (rules out Hexidecimal naming) Match m = Regex.Match(sf.ShortFilename, "([g-zG-Z+])"); if (m.Success == false) { return false; } //Vowel range between x and y int vowelCount = CountVowels(sf.ShortFilename); int letterCount = CountLetters(sf.ShortFilename); double vowelRatio = (double)vowelCount / ((double)letterCount + vowelCount); if ((vowelRatio >= 0.24f) && (vowelRatio <= 0.62f)) { return true; } else { return false; } }
public SmartCrawler LoadCrawler(string pathConfig) { SmartFile _smartFile = new SmartFile(pathConfig); string _line = _smartFile.Lines.FirstOrDefault(); string[] _items = _line.Split(new string[] { "|" }, StringSplitOptions.None); string _crawlerName = _items[0].Trim(); string _crawlerUrl = _items[1].Trim(); string _crawlerOutput = _items[2].Trim().Trim(); int _dept = int.Parse(_items[3].Trim()); string[] _crawlerKeyWords = _items[4].Split(new string[] { ";" }, StringSplitOptions.None) .Where(t => t != null && t.Length > 0).ToArray(); string[] _crawlerIgnoreKeyWords = _items[5].Split(new string[] { ";" }, StringSplitOptions.None) .Where(t => t != null && t.Length > 0).ToArray(); var _crawler = new SmartCrawler(_crawlerName, _crawlerUrl, CrawlerOutputFolder + _crawlerOutput, _crawlerKeyWords, _crawlerIgnoreKeyWords, _dept); return(_crawler); }
public SmartCrawler(string crawlerTemplate, string crawlerOutput) { SmartFile _file = new SmartFile(crawlerTemplate); string[] _items = _file.Lines[0].Split(new string[] { "|" }, StringSplitOptions.None); Name = _items[0]; //CrawlerName Url = _items[1]; //Url OutputFile = _items[2]; Depth = int.Parse(_items[3]); UrlKeyWords = _items[4].Split(new string[] { ";" }, StringSplitOptions.None).Where(t => t.Length > 0).ToArray(); InitSchedules(); CountRun = 0; Step = 0; if (!string.IsNullOrWhiteSpace(_items[5])) { IgnoreKeywords = _items[5].Split(new string[] { ";" }, StringSplitOptions.None).Where(t => t.Length > 0).ToArray(); } if (OutputFile == null || OutputFile == string.Empty) { OutputFile = @"C:\TmpC#\Crawlers\" + Name + ".txt"; } UrlSmart = new SmartUrl(Url); Page = new WebPage(Url); OutputFile = crawlerOutput + OutputFile; if (!File.Exists(OutputFile)) { File.Create(OutputFile); File.WriteAllText(OutputFile, "Crawler " + Name); } AddResult(); NextUrls = new List <string>(); NextUrls = Page.HtmlSmart.InternalLinks.Where(t => t.IsUrl() && IsAccept(t)).Distinct().ToList(); }
public void Initialize() { while (SLContext.Instance.UpdateXml == null) { System.Threading.Thread.Sleep(50); } this.Text = "正在准备下载系统运行所必须的程序及资源..."; this.files = new List <SmartFile>(); IList <SmartFile> files2 = DownloadHelper.Instance.GetDownList(); foreach (SmartFile item in files2) { this.files.Add(item); } if (this.files.Count > 0) { this.downloader = new Downloader(); this.downloader.DownloadCompleted += new EventHandler <FileDownloadCompleteEventArgs>(downloader_DownloadCompleted); this.downloader.DownloadProgressChanged += new EventHandler <FileDownloadProgressEventArgs>(downloader_DownloadProgressChanged); this.downloader.ErrorAccured += new EventHandler <FileDownloadErrorAccured>(downloader_ErrorAccured); m_File = this.files[0]; this.Text = "准备下载文件" + this.m_File.FileName + "..."; this.downloader.DownloadFile(m_File.FileName); } else { if (LoginContext.Singleton.Account != null) //已经成功登录 { App.StartMain(); } else //未能成功登录 { App.Navigation(PlugContext.LoginForm as UserControl); } } }
//Parse file and add necessary tag(s) public override void UpdateTag(SmartFile sf) { //Any alphabet characters? if (UnreadableFilename(sf)) { sf.tags[RuleName] = null; //Kill old tag first sf.tags[RuleName] = new Tag(RuleName, "Unreadable Filename", unreadableFilenameModifier); return; } //Probably human readable? if (ProbablyLegible(sf)) { sf.tags[RuleName] = null; sf.tags[RuleName] = new Tag(RuleName, "Probably Legible Filename", probablyLegibleFilenameModifier); return; } //Unknown sf.tags[RuleName] = null; sf.tags[RuleName] = new Tag(RuleName, "Unkown Filename Type", unknownFilenameModifier); }
public void LoadCrawlerDb() { SmartFile _outFile = new SmartFile(OutputFile); TextDb = _outFile.Content; }
public void DropDatabase() { File.Delete(SmartFile.Path); SmartFile = new SmartFile(SmartFile.Path, new Dictionary <string, SmartTable>()); }
public TableView(SmartFile smartFile) { this.SmartFile = smartFile; }
public DupFileEvent(SmartFile s) { smartFile = s; }
private void OnFileDeleted(SmartFile sf) { if (FileDeleted != null) { FileDeleted(sf); } }
//Attempt to add a file to our master 'duplicates' list. private void TryAddDuplicate(SmartFile newFile) { List<SmartFile> dupeList; //If hash exists open dictionary, if not create a new entry if (duplicateFiles.ContainsKey(newFile.hash64)) { dupeList = duplicateFiles[newFile.hash64]; } else { dupeList = new List<SmartFile>(); //create new list duplicateFiles.Add(newFile.hash64,dupeList); //add list to master dupe dictionary } //Add the duplicate file if it doesn't already exist if (!dupeList.Contains(newFile)) { dupeList.Add(newFile); //Autotag/autorank AutoRank ar = new AutoRank(); ar.UpdateTags(newFile); OnDuplicateFileFound(newFile); } }
//Discover files and add to fileSizeList dictionary. THREAD BLOCKING private void DiscoverFiles(string pathName) { // FileInfo and parsing etc string[] filenames = Directory.GetFiles(pathName, "*", SearchOption.AllDirectories); OnProgressChanged(0, filenames.Length.ToString() + " files acquired", state); int numFiles = filenames.Length; int curFileNum = 0; //statistics foreach (string curFile in filenames) { //Ignore all bittorrent sync folder items if (curFile.Contains("\\.SyncArchive\\")) { continue;} if (curFile.Contains(".SyncIgnore")) { continue; } //Generate Smartfile and do fun stuff with it SmartFile newSmartFile = new SmartFile(curFile); //Add the list if it dosen't exist already if (!fileSizeList.ContainsKey(newSmartFile.FileSize)) { fileSizeList.Add(newSmartFile.FileSize,new List<SmartFile>()); } fileSizeList[newSmartFile.FileSize].Add(newSmartFile); OnProgressChanged((double)((double)curFileNum / (double)numFiles)*100f, curFileNum.ToString() + " files processed", state); curFileNum++; } }
public void MoreTests() { SmartFile sf = new SmartFile("C:\\inc\\0914\\34\\1408837834350.jpg"); Debug.WriteLine(sf.LastFolderName); }
public string[] GetDirectoriesNames() { return(SmartFile.GetSmartTables().Keys.ToArray()); }
public void DeleteFile(SmartFile sf) { if (sf == null) { return; } //Find in duplicate lists if (duplicateFiles[sf.hash64].Exists((SmartFile sf2) => sf2.fullFileName == sf.fullFileName)) { duplicateFiles[sf.hash64].Find((SmartFile sf2) => sf2.fullFileName == sf.fullFileName).Delete(); duplicateFiles[sf.hash64].Remove(sf); OnFileDeleted(sf); } }
public bool IsIdenticalTo(SmartFile otherFile) { if (Array.Equals(this.hash, otherFile.hash)) { return true; } else { return false; } }
public Boolean modifyGroup(String user, SmartFile.Permission permissions) { return false; }
internal void Add(ITable table) { SmartFile.AddTable(table); }
internal void Remove(ITable table) { SmartFile.RemoveTable(table); }
private void OnDuplicateFileFound(SmartFile sf) { if (DuplicateFileFound != null) { DuplicateFileFound(sf); } }
internal void Update(ITable table) { SmartFile.UpdateTable(table); }
private void DupFileDeleted(SmartFile sf) { //find file in list }
public void Save() { SmartFile.Save(); }
//Returns true if there are no readable characters. (\w) private bool UnreadableFilename(SmartFile sf) { Match m = Regex.Match(sf.ShortFilename, "(\\w+)"); return !m.Success; }
private void toolStripPasteItems_Click(object sender, EventArgs e) { if (Clipboard.GetData("FileDrop") == null) { return;} // TreeNode selectedNode = treeView1.SelectedNode; if (selectedNode != null) { SmartFile file = (SmartFile) selectedNode.Tag; string fullPath = file.FullFolderName; string[] files = (string[]) Clipboard.GetData("FileDrop"); foreach (string curFile in files) { SmartFile s = new SmartFile(curFile); if (s.isValidFile) { s.Move(fullPath); } //System.IO.Directory.Move(curFile,fullPath } PopulateDupList(); } }
//public abstract int GetRank(SmartFile sf); public abstract void UpdateTag(SmartFile sf);