public void LoadTables()
{
foreach (Type type in RecordsAssembly.GetTypes())
{
TableAttribute attribute = type.GetCustomAttribute <TableAttribute>();
if (attribute != null)
{
IList tables = (IList)Activator.CreateInstance(typeof(List <>).MakeGenericType(type));
var records = SmartFile.ConvertToRecord(attribute.Path, type);
foreach (var record in records)
{
tables.Add(record);
}
if (tables.Count > 0)
{
var field = ITableManager.Instance.GetCache(type);
field.SetValue(null, tables);
}
}
}
}
void downloader_DownloadCompleted(object sender, FileDownloadCompleteEventArgs e)
{
this.Text = "文件" + this.m_File.FileName + "文件下载完成100%";
Downloader.Logger(this.Text);
byte[] buffer = new byte[e.MemoryStream.Length];
e.MemoryStream.Read(buffer, 0, buffer.Length);
DownloadHelper.Instance.WriteSmartFile(this.m_File, buffer);
this.files.RemoveAt(0);
if (this.files.Count > 0)
{
m_File = this.files[0];
this.Text = "准备下载文件" + this.m_File.FileName + "...";
this.downloader.DownloadFile(m_File.FileName);
}
else
{
DownloadHelper.Instance.WriteLocalUpdateXml();
if (LoginContext.Singleton.Account != null) //已经成功登录
{
SLContext.Instance.Shell = new UI.MainPage();
App.Navigation(SLContext.Instance.Shell);
}
else //未能成功登录
{
App.Navigation(PlugContext.LoginForm as UserControl);
}
}
}
public void SmartFileTests()
{
//Create a test file
string tmpFile = System.IO.Path.GetTempFileName();
StreamWriter sw = File.AppendText(tmpFile);
sw.WriteLine("Testing, over the river and through the woods");
sw.Close();
SmartFile s = new SmartFile(tmpFile);
Assert.IsTrue(s.isValidFile,"Bad File Condition");
Debug.WriteLine(s.hashString);
string tmpFile2 = System.IO.Path.GetTempFileName();
StreamWriter sw2 = File.AppendText(tmpFile2);
sw2.WriteLine("Testing, over the river and through the woods");
sw2.Close();
SmartFile s2 = new SmartFile(tmpFile2);
Assert.IsTrue(s2.isValidFile, "Bad File Condition");
Debug.WriteLine(s2.hashString);
//File.AppendText("Testing, over the river and through the woods");
//Assert.IsTrue(
//
File.Delete(tmpFile2);
File.Delete(tmpFile);
}
public void UpdateTags(SmartFile sf)
{
foreach (AutoRankRule rule in rules)
{
rule.UpdateTag(sf);
}
}
public void Autorank_Legibility()
{
AutoRankRule_FilenameReadability autoRankRuler = new AutoRankRule_FilenameReadability();
SmartFile newFile = new SmartFile("C:\\inc\\Riftchan_Notes032714.txt");
autoRankRuler.UpdateTag(newFile);
}
public Task(SmartFile.ClientHandler client, String APIendPoint, DateTime created, int status, string type)
{
this.client = client;
this.APIendPoint = APIendPoint;
this.created = created;
this.status = status;
this.type = type;
}
private bool InAKeepFolder(SmartFile sf)
{
if (sf.LastFolderName.StartsWith("1"))
{
return true;
}
return false;
}
private bool InUnsortedSubFolder(SmartFile sf)
{
if (sf.fullFileName.ToLower().Contains("unsort"))
{
return true;
}
return false;
}
public bool IsExits(string url)
{
SmartFile _smartDb = new SmartFile(OutputFile);
if (_smartDb.Lines != null && _smartDb.Lines.Count() > 0 && _smartDb.Lines.Contains(url))
{
return(true);
}
return(false);
}
public void Autorank_VowelTest()
{
AutoRankRule_FilenameReadability autoRankRuler = new AutoRankRule_FilenameReadability();
string[] filenames = Directory.GetFiles("C:\\inc", "*", SearchOption.AllDirectories);
foreach (string curFile in filenames)
{
SmartFile newSmartFile = new SmartFile(curFile);
autoRankRuler.UpdateTag(newSmartFile);
}
}
//Parse file and add necessary tag(s)
public override void UpdateTag(SmartFile sf)
{
//if (sf.fileName.Contains(" - Copy"))
if (sf.fileName.ToLower().Contains("copy"))
{
sf.tags[RuleName] = new Tag(RuleResult, "'Copy' Filename",modifier_IsACopy);
return;
}
sf.tags[RuleName] = new Tag(RuleResult, "not a copy", modifier_NotACopy);
return;
}
public void Initialize(string smartFilePath, Assembly recordsAssembly)
{
this.RecordsAssembly = recordsAssembly;
if (File.Exists(smartFilePath) == false)
{
SmartFile = new SmartFile(smartFilePath, new Dictionary <string, SmartTable>());
}
else
{
SmartFile = new SmartFile(smartFilePath);
}
}
public List <KeyValuePair <string, object[]> > GetPropertiesValues()
{
List <KeyValuePair <string, object[]> > results = new List <KeyValuePair <string, object[]> >();
foreach (var file in SmartFile.GetSmartTables()[SelectedTable].Files)
{
var jObject = (JObject)JsonConvert.DeserializeObject(file.Value);
object[] properties = Array.ConvertAll <JToken, object>(jObject.PropertyValues().ToArray(), x => x.ToString());
results.Add(new KeyValuePair <string, object[]>(file.FileName, properties));
}
return(results);
}
private void AddResult()
{
SmartFile _file = new SmartFile(OutputFile);
if (IsAccept(Url) && !IsExits(Url))
{
Page = new WebPage(Url);
DateTime _time = DateTime.Now;
_file.AddLineBefore("{" + _time + "}" + "{Title}{" + Page.Title.ToStandard() + "}");
_file.Save();
_file.AddLineBefore("{" + _time + "}" + "{Url}{" + Url + "}");
_file.Save();
}
}
public string[] GetProperties()
{
var files = SmartFile.GetSmartTables()[SelectedTable].Files;
if (files.Count == 0)
{
return(new string[0]);
}
var objRef = files[0];
var jObject = (JObject)JsonConvert.DeserializeObject(objRef.Value);
return(Array.ConvertAll(jObject.Properties().ToArray(), x => x.Name));
}
private List<SmartFile> GetAllFilesInfolders(string[] searchPaths)
{
List<SmartFile> sfl = new List<SmartFile>();
foreach (string curPath in searchPaths)
{
string[] filenames = Directory.GetFiles(curPath, "*", SearchOption.AllDirectories);
foreach (string curFile in filenames)
{
SmartFile newSmartFile = new SmartFile(curFile);
sfl.Add(newSmartFile);
}
}
return sfl;
}
public override void UpdateTag(SmartFile sf)
{
if (InAKeepFolder(sf))
{
sf.tags[RuleName] = new Tag(RuleName, "In a Keeper folder", modifier_KeepFolder);
}
if (InUnsortedFolder(sf))
{
sf.tags[RuleName] = new Tag(RuleName, "In Unsorted Folder", modifier_UnsortedFolder);
}
if (InUnsortedSubFolder(sf))
{
sf.tags[RuleName] = new Tag(RuleName, "In Unsorted Subfolder", modifier_UnsortedSubFolder);
}
}
//Check ratio of vowels to letters (confirm a file is *probably* human readable
private bool ProbablyLegible(SmartFile sf)
{
//Require at least 1 character after f (rules out Hexidecimal naming)
Match m = Regex.Match(sf.ShortFilename, "([g-zG-Z+])");
if (m.Success == false) { return false; }
//Vowel range between x and y
int vowelCount = CountVowels(sf.ShortFilename);
int letterCount = CountLetters(sf.ShortFilename);
double vowelRatio = (double)vowelCount / ((double)letterCount + vowelCount);
if ((vowelRatio >= 0.24f) && (vowelRatio <= 0.62f))
{
return true;
}
else
{
return false;
}
}
public SmartCrawler LoadCrawler(string pathConfig)
{
SmartFile _smartFile = new SmartFile(pathConfig);
string _line = _smartFile.Lines.FirstOrDefault();
string[] _items = _line.Split(new string[] { "|" }, StringSplitOptions.None);
string _crawlerName = _items[0].Trim();
string _crawlerUrl = _items[1].Trim();
string _crawlerOutput = _items[2].Trim().Trim();
int _dept = int.Parse(_items[3].Trim());
string[] _crawlerKeyWords = _items[4].Split(new string[] { ";" }, StringSplitOptions.None)
.Where(t => t != null && t.Length > 0).ToArray();
string[] _crawlerIgnoreKeyWords = _items[5].Split(new string[] { ";" }, StringSplitOptions.None)
.Where(t => t != null && t.Length > 0).ToArray();
var _crawler = new SmartCrawler(_crawlerName, _crawlerUrl, CrawlerOutputFolder + _crawlerOutput, _crawlerKeyWords, _crawlerIgnoreKeyWords, _dept);
return(_crawler);
}
public SmartCrawler(string crawlerTemplate, string crawlerOutput)
{
SmartFile _file = new SmartFile(crawlerTemplate);
string[] _items = _file.Lines[0].Split(new string[] { "|" }, StringSplitOptions.None);
Name = _items[0]; //CrawlerName
Url = _items[1]; //Url
OutputFile = _items[2];
Depth = int.Parse(_items[3]);
UrlKeyWords = _items[4].Split(new string[] { ";" }, StringSplitOptions.None).Where(t => t.Length > 0).ToArray();
InitSchedules();
CountRun = 0;
Step = 0;
if (!string.IsNullOrWhiteSpace(_items[5]))
{
IgnoreKeywords = _items[5].Split(new string[] { ";" }, StringSplitOptions.None).Where(t => t.Length > 0).ToArray();
}
if (OutputFile == null || OutputFile == string.Empty)
{
OutputFile = @"C:\TmpC#\Crawlers\" + Name + ".txt";
}
UrlSmart = new SmartUrl(Url);
Page = new WebPage(Url);
OutputFile = crawlerOutput + OutputFile;
if (!File.Exists(OutputFile))
{
File.Create(OutputFile);
File.WriteAllText(OutputFile, "Crawler " + Name);
}
AddResult();
NextUrls = new List <string>();
NextUrls = Page.HtmlSmart.InternalLinks.Where(t => t.IsUrl() && IsAccept(t)).Distinct().ToList();
}
public void Initialize()
{
while (SLContext.Instance.UpdateXml == null)
{
System.Threading.Thread.Sleep(50);
}
this.Text = "正在准备下载系统运行所必须的程序及资源...";
this.files = new List <SmartFile>();
IList <SmartFile> files2 = DownloadHelper.Instance.GetDownList();
foreach (SmartFile item in files2)
{
this.files.Add(item);
}
if (this.files.Count > 0)
{
this.downloader = new Downloader();
this.downloader.DownloadCompleted += new EventHandler <FileDownloadCompleteEventArgs>(downloader_DownloadCompleted);
this.downloader.DownloadProgressChanged += new EventHandler <FileDownloadProgressEventArgs>(downloader_DownloadProgressChanged);
this.downloader.ErrorAccured += new EventHandler <FileDownloadErrorAccured>(downloader_ErrorAccured);
m_File = this.files[0];
this.Text = "准备下载文件" + this.m_File.FileName + "...";
this.downloader.DownloadFile(m_File.FileName);
}
else
{
if (LoginContext.Singleton.Account != null) //已经成功登录
{
App.StartMain();
}
else //未能成功登录
{
App.Navigation(PlugContext.LoginForm as UserControl);
}
}
}
//Parse file and add necessary tag(s)
public override void UpdateTag(SmartFile sf)
{
//Any alphabet characters?
if (UnreadableFilename(sf))
{
sf.tags[RuleName] = null; //Kill old tag first
sf.tags[RuleName] = new Tag(RuleName, "Unreadable Filename", unreadableFilenameModifier);
return;
}
//Probably human readable?
if (ProbablyLegible(sf))
{
sf.tags[RuleName] = null;
sf.tags[RuleName] = new Tag(RuleName, "Probably Legible Filename", probablyLegibleFilenameModifier);
return;
}
//Unknown
sf.tags[RuleName] = null;
sf.tags[RuleName] = new Tag(RuleName, "Unkown Filename Type", unknownFilenameModifier);
}
public void LoadCrawlerDb()
{
SmartFile _outFile = new SmartFile(OutputFile);
TextDb = _outFile.Content;
}
public void DropDatabase()
{
File.Delete(SmartFile.Path);
SmartFile = new SmartFile(SmartFile.Path, new Dictionary <string, SmartTable>());
}
public TableView(SmartFile smartFile)
{
this.SmartFile = smartFile;
}
public DupFileEvent(SmartFile s)
{
smartFile = s;
}
private void OnFileDeleted(SmartFile sf)
{
if (FileDeleted != null)
{
FileDeleted(sf);
}
}
//Attempt to add a file to our master 'duplicates' list.
private void TryAddDuplicate(SmartFile newFile)
{
List<SmartFile> dupeList;
//If hash exists open dictionary, if not create a new entry
if (duplicateFiles.ContainsKey(newFile.hash64))
{
dupeList = duplicateFiles[newFile.hash64];
}
else
{
dupeList = new List<SmartFile>(); //create new list
duplicateFiles.Add(newFile.hash64,dupeList); //add list to master dupe dictionary
}
//Add the duplicate file if it doesn't already exist
if (!dupeList.Contains(newFile))
{
dupeList.Add(newFile);
//Autotag/autorank
AutoRank ar = new AutoRank();
ar.UpdateTags(newFile);
OnDuplicateFileFound(newFile);
}
}
//Discover files and add to fileSizeList dictionary. THREAD BLOCKING
private void DiscoverFiles(string pathName)
{
// FileInfo and parsing etc
string[] filenames = Directory.GetFiles(pathName, "*", SearchOption.AllDirectories);
OnProgressChanged(0, filenames.Length.ToString() + " files acquired", state);
int numFiles = filenames.Length; int curFileNum = 0; //statistics
foreach (string curFile in filenames)
{
//Ignore all bittorrent sync folder items
if (curFile.Contains("\\.SyncArchive\\")) { continue;}
if (curFile.Contains(".SyncIgnore")) { continue; }
//Generate Smartfile and do fun stuff with it
SmartFile newSmartFile = new SmartFile(curFile);
//Add the list if it dosen't exist already
if (!fileSizeList.ContainsKey(newSmartFile.FileSize))
{
fileSizeList.Add(newSmartFile.FileSize,new List<SmartFile>());
}
fileSizeList[newSmartFile.FileSize].Add(newSmartFile);
OnProgressChanged((double)((double)curFileNum / (double)numFiles)*100f, curFileNum.ToString() + " files processed", state); curFileNum++;
}
}
public void MoreTests()
{
SmartFile sf = new SmartFile("C:\\inc\\0914\\34\\1408837834350.jpg");
Debug.WriteLine(sf.LastFolderName);
}
public string[] GetDirectoriesNames()
{
return(SmartFile.GetSmartTables().Keys.ToArray());
}
public void DeleteFile(SmartFile sf)
{
if (sf == null) { return; }
//Find in duplicate lists
if (duplicateFiles[sf.hash64].Exists((SmartFile sf2) => sf2.fullFileName == sf.fullFileName))
{
duplicateFiles[sf.hash64].Find((SmartFile sf2) => sf2.fullFileName == sf.fullFileName).Delete();
duplicateFiles[sf.hash64].Remove(sf);
OnFileDeleted(sf);
}
}
public bool IsIdenticalTo(SmartFile otherFile)
{
if (Array.Equals(this.hash, otherFile.hash))
{
return true;
}
else
{
return false;
}
}
public Boolean modifyGroup(String user, SmartFile.Permission permissions)
{
return false;
}
internal void Add(ITable table)
{
SmartFile.AddTable(table);
}
internal void Remove(ITable table)
{
SmartFile.RemoveTable(table);
}
private void OnDuplicateFileFound(SmartFile sf)
{
if (DuplicateFileFound != null)
{
DuplicateFileFound(sf);
}
}
internal void Update(ITable table)
{
SmartFile.UpdateTable(table);
}
private void DupFileDeleted(SmartFile sf)
{
//find file in list
}
public void Save()
{
SmartFile.Save();
}
//Returns true if there are no readable characters. (\w)
private bool UnreadableFilename(SmartFile sf)
{
Match m = Regex.Match(sf.ShortFilename, "(\\w+)");
return !m.Success;
}
private void toolStripPasteItems_Click(object sender, EventArgs e)
{
if (Clipboard.GetData("FileDrop") == null) { return;}
//
TreeNode selectedNode = treeView1.SelectedNode;
if (selectedNode != null)
{
SmartFile file = (SmartFile) selectedNode.Tag;
string fullPath = file.FullFolderName;
string[] files = (string[]) Clipboard.GetData("FileDrop");
foreach (string curFile in files)
{
SmartFile s = new SmartFile(curFile);
if (s.isValidFile)
{
s.Move(fullPath);
}
//System.IO.Directory.Move(curFile,fullPath
}
PopulateDupList();
}
}
//public abstract int GetRank(SmartFile sf); public abstract void UpdateTag(SmartFile sf);
