public void SourceAAPL_SubmissionInfo_Success()
{
FileDictionary fileDict = new FileDictionary();
ISource secSource = new SECSource();
ISourceInitParams initParams = secSource.CreateInitParams();
initParams.Dictionary = fileDict;
secSource.Init(initParams);
ISourceSubmissionsInfoParams subInfoParams = new SECSourceSubmissionsInfoParams();
subInfoParams.CompanyCode = ConfigurationManager.AppSettings["SEC_AAPL_CODE"];
subInfoParams.RegulatorCode = ConfigurationManager.AppSettings["SECCode"];
ISourceItemInfo itemInfo = secSource.CreateSourceItemInfo();
itemInfo.Name = ConfigurationManager.AppSettings["SEC_CIK_AAPL_SUBMISSION_FOLDER"];
subInfoParams.Items.Add(itemInfo);
ISourceSubmissionsInfoResult subInfoResult = secSource.GetSubmissionsInfo(subInfoParams);
Assert.IsTrue(subInfoResult.Success, "Failed to get submission info");
Assert.AreNotEqual(subInfoResult.Submissions, null, "Submissions list is NULL");
Assert.IsNotEmpty(subInfoResult.Submissions, "Submissions list is empty");
Assert.AreEqual(subInfoResult.Submissions.Count, 1, "Invalid submissions info items in the list");
Assert.AreEqual(subInfoResult.Submissions[0].Report, "aapl-20170701.xml", "Invalid report file specified");
Assert.AreEqual(subInfoResult.Submissions[0].Name, ConfigurationManager.AppSettings["SEC_CIK_AAPL_SUBMISSION_FOLDER"], "Invalid submission name returned");
}
void IFileData.Load(FileLoader loader)
{
loader.CheckSignature(_signature);
uint padding = loader.ReadUInt32();
uint Version = loader.ReadUInt32();
SetVersionInfo(Version);
ByteOrder = loader.ReadUInt16();
Alignment = (uint)loader.ReadByte();
TargetAddressSize = (uint)loader.ReadByte();
uint Padding = loader.ReadUInt32(); //Usually name offset for file with other switch formats
ushort Padding2 = loader.ReadUInt16();
ushort BlockOffset = loader.ReadUInt16(); //Goes to ASST section which seems to have block headers
uint RelocationTableOffset = loader.ReadUInt32();
uint EndOfFile = loader.ReadUInt32();
ulong FileCount = loader.ReadUInt64();
ulong FileInfoOffset = loader.ReadUInt64();
FileDictionary = loader.LoadDict();
ulong unk = loader.ReadUInt64();
Name = loader.LoadString();
ulong BlockOffset2 = loader.ReadUInt64(); //Same offset?
FileList = loader.LoadCustom(() =>
{
Dictionary <string, ASST> asstList = new Dictionary <string, ASST>();
for (int i = 0; i < (int)FileCount; i++)
{
asstList.Add(FileDictionary.GetKey(i), loader.Load <ASST>());
}
return(asstList);
}, (long)FileInfoOffset);
}
public void SourceAAPL_Extract_Success()
{
FileDictionary fileDict = new FileDictionary();
ISource secSource = new SECSource();
ISourceInitParams initParams = secSource.CreateInitParams();
initParams.Dictionary = fileDict;
secSource.Init(initParams);
ISourceExtractParams extractParams = secSource.CreateExtractParams();
extractParams.CompanyCode = ConfigurationManager.AppSettings["SEC_AAPL_CODE"];
extractParams.RegulatorCode = ConfigurationManager.AppSettings["SECCode"];
ISourceItemInfo itemInfo = secSource.CreateSourceItemInfo();
itemInfo.Name = ConfigurationManager.AppSettings["SEC_CIK_AAPL_SUBMISSION_FOLDER"];
extractParams.Items.Add(itemInfo);
ISourceExtractResult result = secSource.ExtractReports(extractParams);
Assert.AreNotEqual(result.Items, null, "List of file items in filing is NULL");
Assert.AreNotEqual(result.Items.Count, 0, "List of file items in filing is empty");
Assert.AreEqual(result.Items.Count, 75, string.Format("Incorrect number of file extracted - {0}", result.Items.Count));
}
public IStorage OpenFile(string filename)
{
if (!FileDictionary.TryGetValue(filename, out SaveFileEntry file))
{
throw new FileNotFoundException();
}
return(OpenFile(file));
}
public FileIndexViewerWindow()
{
InitializeComponent();
FileDictionary fileDirectoryOriginal = ApplicationState.Instance.ProjectFile.OriginalImage.FileDictionary;
dataGridView1.DataSource = fileDirectoryOriginal.Files;
FileDictionary fileDirectoryModified = ApplicationState.Instance.ProjectFile.ModifiedImage.FileDictionary;
dataGridView2.DataSource = fileDirectoryModified.Files;
}
public void SaveFile(bool threaded)
{
if (threaded)
{
Methods.Run(() => FileDictionary.Save(this.XmlSerialize()), "FileDictionary.Save", true, true);
}
else
{
FileDictionary.Save(this.XmlSerialize());
}
}
/// <summary>
/// Read every file recursevily in path1 and store in dictionary of md5-hash, path.
/// Read every file recursevily in path2 and store in dictionary of md5-has, path.
/// List the difference between path1 and path2 by comparing md5-hashes.
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
FileDictionary[] dictionaries = new FileDictionary[2];
Parallel.For(0, 2, i =>
{
Console.WriteLine(i);
dictionaries[i] = ScanDirectory(args[i]);
});
var result = dictionaries[0] - dictionaries[1];
ShowDictionary(result);
}
public void GetFilesInDirectory(FileInfo[] files)
{
foreach (var file in files)
{
if (FileDictionary.ContainsKey(file.Name))
{
continue;
}
FileDictionary.Add(file.Name, new Dictionary <string, string> {
{ file.Directory.FullName, file.Name }
});
}
}
static void RunFile()
{
GC.Collect();
long baseSize, curSize;
var sw = new System.Diagnostics.Stopwatch();
var h = new FileDictionary <StringableString, StringableString>(@"C:\TMP\hash", 2);
h.Prepare();
baseSize = Environment.WorkingSet;
sw.Start();
InsertTest(h);
sw.Stop();
curSize = Environment.WorkingSet;
Console.WriteLine($"FileDictionary: Add {sw.ElapsedMilliseconds} msec");
Console.WriteLine($"FileDictionary: heap {curSize - baseSize} byte ({(((double)curSize - (double)baseSize) / 1024 / 1024):0.0}) MB");
sw.Reset();
sw.Start();
FindTest(h);
sw.Stop();
Console.WriteLine($"Dictionary: Find {sw.ElapsedMilliseconds} msec");
sw.Reset();
sw.Start();
IterationTest(h);
sw.Stop();
Console.WriteLine($"Dictionary: Iteration {sw.ElapsedMilliseconds} msec");
sw.Reset();
sw.Start();
IterationTest2(h);
sw.Stop();
Console.WriteLine($"Dictionary: Iteration2 {sw.ElapsedMilliseconds} msec");
sw.Reset();
sw.Start();
IterationTest3(h);
sw.Stop();
Console.WriteLine($"Dictionary: Iteration3 {sw.ElapsedMilliseconds} msec");
sw.Reset();
sw.Start();
RemoveTest(h);
sw.Stop();
Console.WriteLine($"Dictionary: Remove {sw.ElapsedMilliseconds} msec");
}
public DirectoryEntryType GetEntryType(string path)
{
path = PathTools.Normalize(path);
if (DirDictionary.ContainsKey(path))
{
return(DirectoryEntryType.Directory);
}
if (FileDictionary.ContainsKey(path))
{
return(DirectoryEntryType.File);
}
throw new FileNotFoundException(path);
}
public ProjectViewModel(string basePath, string name, string language, string sourceCodeType, string moduleName)
{
_basePath = basePath;
_name = name;
_language = language;
_modules = new List <ProjectViewModel> {
this
};
this._sourceCodeType = sourceCodeType;
_moduleName = moduleName;
_uploadCommand = new RelayCommand(BeginUpload, () => FileDictionary["inheritance"].Exists);
foreach (string dataType in DataTypes)
{
FileDictionary.Add(dataType, new FileViewModel(dataType));
}
}
private static void DoScan(FileDictionary fileDictionary, MD5 md5, string directoryPath)
{
foreach (var filePath in Directory.GetFiles(directoryPath))
{
if (IncludeFile(filePath))
{
string md5hash = CalcMd5Hash(md5, filePath);
fileDictionary.Add(md5hash, filePath);
}
}
foreach (var dirPath in Directory.GetDirectories(directoryPath))
{
DoScan(fileDictionary, md5, dirPath);
}
}
private void fileIndexViewerToolStripMenuItem_Click(object sender, EventArgs e)
{
if (ApplicationState.Instance.ProjectFile == null)
{
MessageBox.Show("There is no project maaaan.");
return;
}
FileDictionary fileDirectory = ApplicationState.Instance.ProjectFile.ModifiedImage.FileDictionary;
if (fileDirectory == null)
{
MessageBox.Show("There are no file indices maaaan.");
return;
}
FileIndexViewerWindow fileIndexViewerWindow = new FileIndexViewerWindow();
fileIndexViewerWindow.ShowDialog();
}
public IFile OpenFile(string path, OpenMode mode)
{
path = PathTools.Normalize(path);
if (!FileDictionary.TryGetValue(path, out SaveFileEntry file))
{
throw new FileNotFoundException();
}
if (file.BlockIndex < 0)
{
return(new NullFile());
}
AllocationTableStorage storage = OpenFatBlock(file.BlockIndex, file.FileSize);
return(new SaveDataFile(storage, 0, file.FileSize, mode));
}
private void SaveFile()
{
try
{
foreach (var item in FileDictionary)
{
BaseService.SaveFile(System.IO.File.ReadAllBytes(item.Value), item.Key);
}
FileDictionary.Clear();
}
catch (Exception ex)
{
throw ex;
}
finally
{
}
}
static void Main(string[] args)
{
var dict = new FileDictionary(@"C:\Users\dorse\source\repos\WordStats\WordStats\bin\Debug\Dictionary.txt");
//string[] words = { "hello", "he", "hell", "hole" };
//foreach (string s in words)
//{
// dict.Add(s);
//}
WordFinder wf = new WordFinder(dict);
var l = wf.GetWords(Console.ReadLine());
l = GetList(3, l);
l = l.OrderBy(x => x.Length).ToList();
foreach (string s in l)
{
Console.WriteLine(s);
}
Console.ReadKey();
}
public void SourceAAPL_Validate_Success()
{
FileDictionary fileDict = new FileDictionary();
ISource secSource = new SECSource();
ISourceInitParams initParams = secSource.CreateInitParams();
initParams.Dictionary = fileDict;
secSource.Init(initParams);
// preparing params
ISourceValidateParams vldParams = secSource.CreateValidateParams();
vldParams.CompanyCode = ConfigurationManager.AppSettings["SEC_AAPL_CODE"];
vldParams.RegulatorCode = ConfigurationManager.AppSettings["SECCode"];
vldParams.UpdateFromDate = DateTime.Now - TimeSpan.FromDays(90);
vldParams.UpdateToDate = DateTime.Now;
ISourceFilingsListResult result = secSource.GetFilingsList(vldParams);
Assert.AreNotEqual(result.Filings, null, "List of delta items is NULL");
Assert.AreNotEqual(result.Filings.Count, 0, "List of delta items is empty");
}
public ConverterEngine(
string sourceDirectory,
string searchPattern,
string destDirectory,
bool isCaseSensitive = true)
{
Contract.RequiresArgumentNotNull(sourceDirectory, nameof(sourceDirectory));
Contract.RequiresArgumentNotNull(searchPattern, nameof(searchPattern));
Contract.RequiresArgumentNotNull(destDirectory, nameof(destDirectory));
this.sourceDirectory = sourceDirectory;
this.searchPattern = searchPattern;
this.destDirectory = destDirectory;
this.pathComparer = GetPathComparer(isCaseSensitive);
this.files =
new[]
{
ConverterFileStatus.None,
ConverterFileStatus.Processing,
ConverterFileStatus.ProcessedSuccessfully,
ConverterFileStatus.Failed
}
.ToDictionary(status => status, status => new FileSubdictionary(this.pathComparer));
FileDictionary SelectFiles(params ConverterFileStatus[] statuses) =>
statuses
.ToDictionary(status => status, status => this.files[status]);
this.completedFiles = SelectFiles(
ConverterFileStatus.ProcessedSuccessfully,
ConverterFileStatus.Failed);
this.uncompletedFiles = SelectFiles(
ConverterFileStatus.None,
ConverterFileStatus.Processing);
}
public virtual void Open()
{
try
{
//清除文件列表
FileDictionary.Clear();
foreach (var sys_WorkSet in WorkSetList.OrderBy(w => w.WorkSet_ReadSort))
{
if (string.IsNullOrEmpty(sys_WorkSet.WorkSet_Trigger))
{
if (sys_WorkSet.WorkSet_Control != null && sys_WorkSet.WorkSet_Control.GetType().Equals(typeof(GridControl)))
{
GridControl gd = (GridControl)sys_WorkSet.WorkSet_Control;
if (gd != null)
{
List <MyParameter> QueryParameters = InitializeBind(sys_WorkSet.WorkSet_Id, true);
SetSource(gd, BaseService.Open(sys_WorkSet.WorkSet_Name, QueryParameters));
}
}
if (sys_WorkSet.WorkSet_Control != null && sys_WorkSet.WorkSet_Control.GetType().Equals(typeof(ChartControl)))
{
ChartControl chart = (ChartControl)sys_WorkSet.WorkSet_Control;
if (chart != null)
{
List <MyParameter> QueryParameters = InitializeBind(sys_WorkSet.WorkSet_Id, true);
chart.DataSource = BaseService.Open(sys_WorkSet.WorkSet_Name, QueryParameters);
}
}
}
}
}
catch (Exception ex)
{
throw ex;
}
}
public void Unpo()
{
const int KOUNT = 1000;
var baseDir = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "../../work");
var h = new FileDictionary <StringableString, StringableString>(Path.Combine(baseDir, "dic"), 2);
h.Prepare();
h.Clear();
for (int i = 0; i < KOUNT; ++i)
{
h.Add(new StringableString($"K{i:000}"), new StringableString($"V{i:000}"));
}
Assert.Equal(KOUNT, h.Count);
for (int i = 0; i < KOUNT; ++i)
{
var v = h[new StringableString($"K{i:000}")];
Assert.Equal($"V{i:000}", v.Raw);
Assert.Throws <KeyNotFoundException>(() => h[new StringableString($"X{i:000}")]);
}
for (int i = 0; i < KOUNT; ++i)
{
Assert.True(h.ContainsKey(new StringableString($"K{i:000}")));
Assert.False(h.ContainsKey(new StringableString($"V{i:000}")));
}
for (int i = 0; i < KOUNT; ++i)
{
StringableString s;
Assert.True(h.TryGetValue(new StringableString($"K{i:000}"), out s));
Assert.False(h.TryGetValue(new StringableString($"うんぽ{i:000}"), out s));
}
int c = 0;
foreach (var k in h.Keys)
{
var v = h[k];
var ks = "V" + k.Raw.Substring(1);
Assert.Equal(ks, v.Raw);
c++;
}
Assert.Equal(KOUNT, c);
c = 0;
foreach (var k in h.Values)
{
c++;
}
Assert.Equal(KOUNT, c);
c = 0;
foreach (var k in h)
{
c++;
var ks = k.Key.Raw.Substring(1);
var vs = k.Value.Raw.Substring(1);
Assert.Equal(ks, vs);
}
Assert.Equal(KOUNT, c);
for (int i = KOUNT; i >= 0; --i)
{
h.Remove(new StringableString($"K{i:000}"));
Assert.Equal(i, h.Count);
}
Assert.Empty(h);
for (int i = 0; i < KOUNT; ++i)
{
var k = new StringableString($"K{i:000}");
Assert.False(h.ContainsKey(k));
StringableString s;
Assert.False(h.TryGetValue(k, out s));
Assert.Throws <KeyNotFoundException>(() => h[k]);
}
}
private static void ResetFileDictionary(FileDictionary files) => files.ForEach(item => item.Value.Clear());
public DictionaryBasedSplitter(TaskScheduler scheduler)
{
this.dictionary = new FileDictionary();
this.matrix = new SparseMatrixForWordPairs(scheduler);
this.dictionary.rawWordsEvent += matrix.HandleCoOcurrentWordsAsync;
}
private static FileDictionary ScanDirectory(string directoryPath)
{
var fileDictionary = new FileDictionary();
using (MD5 md5 = MD5.Create())
{
DoScan(fileDictionary, md5, directoryPath);
}
return fileDictionary;
}
public bool FileExists(string path)
{
path = PathTools.Normalize(path);
return(FileDictionary.ContainsKey(path));
}
private static void ShowDictionary(FileDictionary result)
{
result.DumpToConsole();
}
public DictionaryBasedSplitter()
{
this.dictionary = new FileDictionary();
this.matrix = new SparseCoOccurrenceMatrix();
this.dictionary.rawWordsEvent += matrix.HandleCoOcurrentWordsAsync;
}
static void Main(string[] args)
{
FileDictionary fd = new FileDictionary();
List <CompanyInfo> companies = fd.GetCompaniesByRegulator("SEC");
// class template
string classTemplate10Q = Resources.Class10QTemplate;
string classTemplate10K = Resources.Class10KTemplate;
string xmlTemplate = Resources.CompanyTagsTemplate;
string projectRoot = "..\\..\\..\\DMFX.SECParser\\";
string comp10QCatalogRegister = "new TypeCatalog(";
string comp10KCatalogRegister = "new TypeCatalog(";
string csvNames = "";
foreach (var c in companies)
{
// 10-Q report classes
{
Console.WriteLine("Generating: " + c.Code + " (" + c.Name + ")");
string className = c.Code.Replace(".", "_");
string classFileName = Path.Combine(projectRoot, "SEC10Q", c.Code + ".cs");
comp10QCatalogRegister += string.Format("typeof(SEC10Q.{0}),\r\n", className);
csvNames += string.Format("result.Add(\"{0}\");\r\n", c.Code);
if (!File.Exists(classFileName))
{
string content = classTemplate10Q.Replace("{{CODE}}", className);
File.WriteAllText(classFileName, content);
}
}
// 10-K report classes
{
Console.WriteLine("Generating: " + c.Code + " (" + c.Name + ")");
string className = c.Code.Replace(".", "_");
string classFileName = Path.Combine(projectRoot, "SEC10K", c.Code + ".cs");
comp10KCatalogRegister += string.Format("typeof(SEC10K.{0}),\r\n", className);
csvNames += string.Format("result.Add(\"{0}\");\r\n", c.Code);
if (!File.Exists(classFileName))
{
string content = classTemplate10K.Replace("{{CODE}}", className);
File.WriteAllText(classFileName, content);
}
}
// Parser XML resources
{
string xmlFileName = Path.Combine(projectRoot, "Resources", c.Code + ".xml");
if (!File.Exists(xmlFileName))
{
string content = xmlTemplate.Replace("{{CODE}}", c.Code.ToLowerInvariant());
File.WriteAllText(xmlFileName, content);
}
}
}
comp10QCatalogRegister += ")";
comp10KCatalogRegister += ")";
}
private static bool FilesValid()
{
long measuredSize = 0;
List <string> files = new List <string>();
string[] paths = { "Content", "maps", "Scripts" };
string[] includeExt = { ".dat", ".poke", ".lua", ".trainer" };
// TODO
if (Runvalidation)
{
Logger.Log(Logger.LogTypes.Debug, "FileValidation.vb: WARNING! FILE VALIDATION IS RUNNING!");
foreach (string subFolder in paths)
{
foreach (string file in System.IO.Directory.GetFiles(GameController.GamePath + "\\" + subFolder, "*.*", SearchOption.AllDirectories))
{
if (file.Contains("\\Content\\Localization\\") == false)
{
string ext = System.IO.Path.GetExtension(file);
if (includeExt.Contains(ext.ToLower()))
{
files.Add(file.Remove(0, GameController.GamePath.Length + 1));
}
}
}
}
string s = "";
foreach (string f in files)
{
long i = new System.IO.FileInfo(GameController.GamePath + "\\" + f).Length;
string hash = GetMd5FromFile(GameController.GamePath + "\\" + f);
FileDictionary.Add((GameController.GamePath + "\\" + f).ToLower(), new ValidationStorage(i, hash));
measuredSize += i;
if (!string.IsNullOrEmpty(s))
{
s += ",";
}
s += f + ":" + hash;
}
System.IO.File.WriteAllText(GameController.GamePath + "\\meta", s);
Logger.Log(Logger.LogTypes.Debug, "FileValidation.vb: Meta created! Expected Size: " + measuredSize + "|MetaHash: " + StringObfuscation.Obfuscate(GetMd5FromFile(GameController.GamePath + "\\meta")));
// TODO
//Core.GameInstance.Exit();
}
else
{
if (System.IO.File.Exists(GameController.GamePath + "\\meta") == true)
{
if (GetMd5FromFile(GameController.GamePath + "\\meta") == StringObfuscation.DeObfuscate(Metahash))
{
files = System.IO.File.ReadAllText(GameController.GamePath + "\\meta").Split(Convert.ToChar(",")).ToList();
Logger.Debug("Meta loaded. Files to check: " + files.Count);
}
else
{
Logger.Log(Logger.LogTypes.Warning, "FileValidation.vb: Failed to load Meta (Hash incorrect)! File Validation will fail!");
}
}
else
{
Logger.Log(Logger.LogTypes.Warning, "FileValidation.vb: Failed to load Meta (File not found)! File Validation will fail!");
}
foreach (string f in files)
{
string fileName = f.Split(Convert.ToChar(":"))[0];
string fileHash = f.Split(Convert.ToChar(":"))[1];
if (System.IO.File.Exists(GameController.GamePath + "\\" + fileName))
{
long i = new System.IO.FileInfo(GameController.GamePath + "\\" + fileName).Length;
FileDictionary.Add((GameController.GamePath + "\\" + fileName).ToLower(), new ValidationStorage(i, fileHash));
measuredSize += i;
}
}
}
if (measuredSize == Expectedsize)
{
return(true);
}
return(false);
}
