public HastyClient(Uri serverUrl, string realm, string username, string password, CommandDefinitions definitions, string baseDir, object target, ILog log)
{
this.username = username;
this.password = password;
this.realm = realm;
this.log = log.CreateLog(typeof(HastyClient));
var storage = new StreamStorage(baseDir, log);
preferences = new Preferences(baseDir, log);
streamStorage = storage;
this.definitions = definitions;
defaultTarget = target;
SetStatus(ConnectionStatus.Idle);
SetState(ConnectionState.Establishing);
connectionMaintainer = new ConnectionMaintainer(serverUrl, realm, log);
connectionMaintainer.OnPacketRead += OnPacketRead;
connectionMaintainer.OnDisconnect += OnDisconnect;
connectionMaintainer.OnConnecting += OnConnecting;
connectionMaintainer.OnConnected += OnMaintainerConnected;
ulong userAssignedChannelId;
var foundUserAssignedChannel = preferences.GetUserChannel(serverUrl.ToString(), realm, out userAssignedChannelId);
if (foundUserAssignedChannel)
{
Subscribe((uint)userAssignedChannelId, target);
}
connectionMaintainer.Start();
}
public static void Extract(Stream inFileStream, string outDir, bool verify, Keyset keyset, Output Out, bool isDecryptedNca = false)
{
using (var file = new StreamStorage(inFileStream, false))
{
var nca = new Nca(keyset, file, false, isDecryptedNca);
Out.Log(nca.Print());
if (verify)
{
Out.Log($"ValidateMasterHashes...\r\n");
nca.ValidateMasterHashes();
}
for (var i = 0; i < 3; ++i)
{
if (nca.Sections[i] != null)
{
if (verify)
{
nca.VerifySection(i, Out);
}
nca.ExtractSection(i, Path.Combine(outDir, i.ToString()), IntegrityCheckLevel.None, Out);
}
}
}
}
public void A_stream_storage_instance_with_stream_factories()
{
Func <Stream> primaryStreamFactory = () => new MemoryStream();
Func <Stream> tempStreamFactory = () =>
{
HasCalledTempStreamFactory = true;
return(new MemoryStream());
};
Action <Stream> primaryStreamCleanUp = s => s.Close();
Action <Stream> tempStreamCleanUp = s => s.Close();
Storage = new StreamStorage(primaryStreamFactory,
tempStreamFactory,
primaryStreamCleanUp,
tempStreamCleanUp);
Table = "Test";
for (int i = 0; i < 110; i++)
{
Storage.Store(Table, i.ToString(), BitConverter.GetBytes(i));
}
for (int i = 11; i < 33; i++)
{
Storage.Remove(Table, i.ToString());
}
Storage.CleanUp();
}
public ITemporaryStreamStorage CreateTemporaryStreamStorage(CancellationToken cancellationToken = default(CancellationToken))
{
var result = new StreamStorage();
serviceList.Add(result);
return(result);
}
public void Given_Non_Valid_Id_Get_Returns_Null()
{
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItem = storage.Get("id");
Assert.IsNull(gotItem);
}
public UnitTestPagedstream()
{
string fobname = "fob.bin";
if (File.Exists(fobname))
{
File.Delete(fobname);
}
ss = new StreamStorage(fobname);
}
public void Given_Existing_10_Items_GetLatest_Returns_All_10_Items()
{
var items = BuildItems(count: 10);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(null, null, null, null).ToList();
Assert.That(items.Count, Is.EqualTo(gotItems.Count));
}
public void ItemStorage_Can_Save_10_New_Items()
{
var items = BuildItems(count: 10);
var storage = new StreamStorage(ConnectionString, DatabaseName);
storage.Save(items);
var savedItems = this.Items.AsQueryable().ToList();
Assert.That(items.Count, Is.EqualTo(savedItems.Count));
}
public void Given_Existing_Item_Get_Returns_It_By_Id()
{
var item = BuildItems(count: 1).Single();
this.Items.Insert(item);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItem = storage.Get(item.Id);
Assert.IsNotNull(gotItem);
Assert.AreEqual(item.Url, gotItem.Url);
}
public void Given_Existing_10_Items_Top_Returns_Most_Recent_Item()
{
var items = BuildItems(count: 10);
var latestItem = items.Last();
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItem = storage.Top();
Assert.That(latestItem.Id, Is.EqualTo(gotItem.Id));
}
public void Given_Existing_10_Items_GetLatest_Returns_All_10_Items()
{
var noLimit = int.MaxValue;
var items = BuildItems(count: 10);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(noLimit).ToList();
Assert.That(items.Count, Is.EqualTo(gotItems.Count));
}
public void Given_Existing_10_Items_GetLatest_Returns_Items_Newer_Than_Given_Date()
{
var items = BuildItems(count: 10).OrderBy(i => i.Published);
var fromDate = items.First().Published;
var numberOfItems = items.Count(i => i.Published > fromDate);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(null, fromDate, null, null).ToList();
Assert.AreEqual(numberOfItems, gotItems.Count);
}
public void Given_Existing_10_Items_GetLatest_Returns_Limited_Number_Of_5_Items()
{
var limit = 5;
var items = BuildItems(count: 10);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(limit).ToList();
Assert.AreEqual(limit, gotItems.Count);
}
public void Given_Existing_10_Items_GetLatest_Returns_Them_Sorted_By_Date_Descending()
{
var noLimit = int.MaxValue;
var items = BuildItems(count: 10);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(noLimit).ToList();
Assert.That(gotItems, Is.Ordered.Descending.By("Published"));
}
public void A_stream_storage_instance()
{
if (DataStream == null)
{
DataStream = new MemoryStream();
}
Func <Stream> primaryStreamFactory = () => new MemoryStream();
Func <Stream> tempStreamFactory = () => new MemoryStream();
Storage = new StreamStorage(primaryStreamFactory, tempStreamFactory, null, null);
Table = "Test";
}
public void Given_Existing_10_Items_GetOlder_Returns_Items_Older_Than_Given_Newest_Item()
{
var items = BuildItems(count: 10);
var newestItem = items.Last();
var numberOfItems = items.Count - 1;
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetOlder(newestItem, items.Count).ToList();
Assert.AreEqual(numberOfItems, gotItems.Count);
}
static void Main(string[] args)
{
string path = "../../Databases/";
Console.WriteLine("Start Task10_StreamStorage");
string s_path = path + "storage.bin";
bool toload = true;
if (toload && File.Exists(s_path))
{
File.Delete(s_path);
}
Polar.PagedStreams.StreamStorage storage = new StreamStorage(s_path);
string[] files = null;
int[] numbers = null;
if (toload)
{
var s1 = storage.CreateStream(out var n1);
files = Directory.GetFiles(@"C:\home\syp\Syp2017\portraits");
numbers = new int[files.Length];
for (int i = 0; i < files.Length; i++)
{
Console.WriteLine($"geting {files[i]}");
Stream destination = storage.CreateStream(out var n);
numbers[i] = n;
File.Open(files[i], FileMode.Open, FileAccess.Read).CopyTo(destination);
}
}
string dir_new = path + "copies/";
for (int i = 0; i < files.Length; i++)
{
Console.WriteLine($"puting {files[i]}");
Stream source = storage[numbers[i]];
source.Position = 0L;
string filename = files[i].Split('\\').Last();
Stream dest = File.Create(dir_new + filename);
source.CopyTo(dest);
dest.Close();
}
storage.Close();
}
public static CnmtExtended GetCnmtExtended(string folderPath, Keyset keyset, Output Out)
{
var dirDecrypted = new DirectoryInfo(folderPath);
foreach (var inFile in dirDecrypted.GetFiles("*.cnmt.nca"))
{
Out.Log($"{inFile}\r\n");
var ncaStorage = new StreamStorage(new FileStream(inFile.FullName, FileMode.Open, FileAccess.Read),
false);
var DecryptedHeader = new byte[0xC00];
ncaStorage.Read(DecryptedHeader, 0, 0xC00, 0);
var Header = new NcaHeader(new BinaryReader(new MemoryStream(DecryptedHeader)), keyset);
for (var i = 0; i < 4; ++i)
{
var section = NcaParseSection.ParseSection(Header, i);
if (section == null || section.Header.Type != SectionType.Pfs0)
{
continue;
}
IStorage sectionStorage = ncaStorage.Slice(section.Offset, section.Size, false);
IStorage pfs0Storage = sectionStorage.Slice(section.Header.Sha256Info.DataOffset,
section.Header.Sha256Info.DataSize, false);
var Pfs0Header = new PartitionFileSystemHeader(new BinaryReader(pfs0Storage.AsStream()));
var FileDict = Pfs0Header.Files.ToDictionary(x => x.Name, x => x);
foreach (var file in FileDict)
{
if (file.Key.EndsWith(".cnmt"))
{
IStorage fileStorage = pfs0Storage.Slice(Pfs0Header.HeaderSize + file.Value.Offset,
file.Value.Size, false);
var metadata = new Cnmt(fileStorage.AsStream());
if (metadata.ExtendedData != null)
{
ncaStorage.Dispose();
return(metadata.ExtendedData);
}
}
}
}
ncaStorage.Dispose();
}
return(null);
}
public void ItemStorage_Can_Save_5_Modified_Items_And_Retrieve_Them_Back()
{
this.Items.InsertBatch(BuildItems(count: 5));
var items = this.Items.AsQueryable().ToList();
foreach (var item in items)
{
item.Tags = new[] { "test" };
}
var storage = new StreamStorage(ConnectionString, DatabaseName);
storage.Save(items);
var savedItems = this.Items.AsQueryable().ToList();
foreach (var item in savedItems)
{
Assert.IsTrue(item.Tags.Contains("test"));
}
}
public void ItemStorage_Can_Save_5_New_And_5_Modified_Items()
{
this.Items.InsertBatch(BuildItems(count: 5));
var existingItems = this.Items.AsQueryable().ToList();
foreach (var item in existingItems)
{
item.Tags = new[] { "test" };
}
var newItems = BuildItems(count: 5);
var items = existingItems.Union(newItems).ToList();
var storage = new StreamStorage(ConnectionString, DatabaseName);
storage.Save(items);
var savedItems = this.Items.AsQueryable().ToList();
Assert.That(items.Count(), Is.EqualTo(savedItems.Count));
}
public static void Process(IFile inFile, IFile outFile, bool verifyBeforeDecrypting, Keyset keyset, Output Out)
{
using (var file = new StreamStorage(inFile.AsStream(), false))
{
var nca = new Nca(keyset, file, false);
Out.Log(nca.Print());
if (verifyBeforeDecrypting)
{
Out.Log($"ValidateMasterHashes...\r\n");
nca.ValidateMasterHashes();
//nca.ParseNpdm();
for (var i = 0; i < 3; ++i)
{
if (nca.Sections[i] != null)
{
nca.VerifySection(i, Out);
}
}
}
Out.Log($"Decripting...\r\n");
nca.OpenDecryptedNca().CopyToStream(outFile.AsStream());
}
}
public FileStorageEngine(string filename)
{
CompactionFrequency = 100;
_storage = new StreamStorage(
() => new FileStream(filename, FileMode.OpenOrCreate),
() => new FileStream(filename + ".tmp", FileMode.OpenOrCreate),
s => { s.Close(); File.Delete(filename); },
s => { s.Close(); File.Delete(filename + ".tmp"); });
_subscription = _input.Connect(config =>
{
config.AddConsumerOf <Request <Startup> >()
.UsingConsumer(Startup)
.HandleOnFiber(_fiber);
config.AddConsumerOf <RemoveValue>()
.UsingConsumer(RemoveKeyFromSession)
.HandleOnFiber(_fiber);
config.AddConsumerOf <Request <RetrieveValue> >()
.UsingConsumer(RetrieveValue)
.HandleOnFiber(_fiber);
config.AddConsumerOf <Request <ListValuesForType> >()
.UsingConsumer(RetrieveListFromType)
.HandleOnFiber(_fiber);
config.AddConsumerOf <StoreValue>()
.UsingConsumer(StoreValue)
.HandleOnFiber(_fiber);
config.AddConsumerOf <CompactionAction>()
.UsingConsumer(CompactDatabase)
.HandleOnFiber(_fiber);
});
}
public StreamStorageEngine(Func <Stream> primaryStreamFactory, Func <Stream> tempStreamFactory,
Action <Stream> primaryStreamCleanUp, Action <Stream> tempStreamCleanUp)
{
CompactionFrequency = 250;
_storage = new StreamStorage(primaryStreamFactory, tempStreamFactory, primaryStreamCleanUp, tempStreamCleanUp);
}
public static void Main(string[] Args)
{
usbnsp:
Console.Clear();
Initialize();
try
{
var pat = new KLST_PATTERN_MATCH {
DeviceID = @"USB\VID_057E&PID_3000"
};
var lst = new LstK(0, ref pat);
lst.MoveNext(out var dinfo);
USB = new UsbK(dinfo);
}
catch
{
Error.Log("No USB connection was found. Make sure you have Goldleaf open before running Goldtree.");
}
try
{
Command c = new Command(CommandId.ConnectionRequest);
USB.Write(c);
Log.Log("Attempting to connect to Goldleaf via USB...");
Command rc = USB.Read();
if (rc.MagicOk())
{
if (rc.IsCommandId(CommandId.ConnectionResponse))
{
Log.Log("Connection was established with Goldleaf.");
Log.Log("Select the NSP to send to Goldleaf on the dialog.");
OpenFileDialog fd = new OpenFileDialog()
{
Title = "Select NSP to send to Goldleaf via USB",
Filter = "NSP / Nintendo Submission Package (*.nsp)|*.nsp",
Multiselect = false,
};
if (fd.ShowDialog() == DialogResult.OK)
{
string nsp = fd.FileName;
string nspname = Path.GetFileName(nsp);
c = new Command(CommandId.NSPName);
USB.Write(c);
USB.Write((uint)nspname.Length);
USB.Write(nspname);
Log.Log("Selected NSP's name was sent to Goldleaf. Waiting for install approval from Goldleaf...");
Command rc2 = USB.Read();
if (rc2.MagicOk())
{
if (rc2.IsCommandId(CommandId.Start))
{
Log.Log("Goldleaf is ready for the installation. Preparing everything...");
try
{
FileStream fs = new FileStream(nsp, FileMode.Open);
StreamStorage ist = new StreamStorage(fs, true);
Pfs pnsp = new Pfs(ist);
ist.Dispose();
fs.Close();
fs = new FileStream(nsp, FileMode.Open);
uint filecount = (uint)pnsp.Files.Length;
c = new Command(CommandId.NSPData);
USB.Write(c);
USB.Write(filecount);
int tikidx = -1;
int certidx = -1;
int tmpidx = 0;
foreach (var file in pnsp.Files)
{
ulong offset = (ulong)pnsp.HeaderSize + (ulong)file.Offset;
ulong size = (ulong)file.Size;
uint len = (uint)file.Name.Length;
USB.Write(len);
USB.Write(file.Name);
USB.Write(offset);
USB.Write(size);
if (Path.GetExtension(file.Name).Replace(".", "").ToLower() == "tik")
{
tikidx = tmpidx;
}
else if (Path.GetExtension(file.Name).Replace(".", "").ToLower() == "cert")
{
certidx = tmpidx;
}
tmpidx++;
}
BinaryReader br = new BinaryReader(fs);
while (true)
{
Command ccmd = USB.Read();
if (ccmd.MagicOk())
{
if (ccmd.IsCommandId(CommandId.NSPContent))
{
USB.Read(out uint idx);
Log.Log("Sending content \'" + pnsp.Files[idx].Name + "\'... (" + (idx + 1) + " of " + pnsp.Files.Length + ")");
PfsFileEntry ent = pnsp.Files[idx];
long rsize = 0x800000;
long coffset = pnsp.HeaderSize + ent.Offset;
long toread = ent.Size;
long tmpread = 1;
byte[] bufb;
while ((tmpread > 0) && (toread > 0) && (coffset < (coffset + ent.Size)))
{
if (rsize >= ent.Size)
{
rsize = ent.Size;
}
int tor = (int)Math.Min(rsize, toread);
br.BaseStream.Position = coffset;
bufb = br.ReadBytes(tor);
tmpread = bufb.Length;
USB.Write(bufb);
coffset += tmpread;
toread -= tmpread;
}
Log.Log("Content was sent to Goldleaf.");
}
else if (ccmd.IsCommandId(CommandId.NSPTicket))
{
Log.Log("Sending ticket file...");
PfsFileEntry tik = pnsp.Files[tikidx];
br.BaseStream.Seek(pnsp.HeaderSize + tik.Offset, SeekOrigin.Begin);
byte[] file = br.ReadBytes((int)tik.Size);
USB.Write(file);
Log.Log("Ticket was sent to Goldleaf.");
}
else if (ccmd.IsCommandId(CommandId.Finish))
{
break;
}
else
{
USB = null;
Error.Log("An invalid command was received. Are you sure Goldleaf is active?");
}
}
else
{
USB = null;
Error.Log("An invalid command was received. Are you sure Goldleaf is active?");
}
}
}
catch
{
Error.Log("An error ocurred opening the selected NSP. Are you sure it's a valid NSP?");
}
}
else if (rc2.IsCommandId(CommandId.Finish))
{
USB = null;
Error.Log("Goldleaf has canceled the installation.");
}
else
{
USB = null;
Error.Log("An invalid command was received. Are you sure Goldleaf is active?");
}
}
else
{
USB = null;
Error.Log("An invalid command was received. Are you sure Goldleaf is active?");
}
}
else
{
Error.Log("The dialog was closed without selecting a NSP, or another error ocurred. Reopen Goldleaf and Goldtree and try again.");
}
}
else if (rc.IsCommandId(CommandId.Finish))
{
USB = null;
Error.Log("Goldleaf has canceled the installation.");
}
else
{
USB = null;
Error.Log("An invalid command was received. Are you sure Goldleaf is active?");
}
}
else
{
USB = null;
Error.Log("An invalid command was received. Are you sure Goldleaf is active?");
}
}
catch
{
Error.Log("An error ocurred selecting the NSP to be sent.");
}
Log.Log("The installation has finished. Press ENTER to close Goldtree, or any other key to start another USB installation.", true);
ConsoleKeyInfo ki = Console.ReadKey();
if (ki.Key == ConsoleKey.Enter)
{
if (USB != null)
{
Command cmd = new Command(CommandId.Finish);
USB.Write(cmd);
}
Environment.Exit(0);
}
else
{
goto usbnsp;
}
}
public static void Process(string folderPath, Keyset keyset, Output Out)
{
var cnmtExtended = CnmtNca.GetCnmtExtended(folderPath, keyset, Out);
if (cnmtExtended == null)
{
Out.Log(
"[Info] Skiped fragemt trimming as no patch Cnmt was found\r\n");
return;
}
if (cnmtExtended.FragmentSets.Length == 0)
{
Out.Log(
"[Info] Skiped fragemt trimming as no DeltaApplyInfos in the patch Cnmt were found\r\n");
return;
}
var DeltaContentID = 0;
foreach (var deltaApplyInfo in cnmtExtended.FragmentSets)
{
long offset = 0;
for (var deltaApplyInfoId = 0; deltaApplyInfoId < deltaApplyInfo.FragmentCount; ++deltaApplyInfoId)
{
var matching = $"{Utils.BytesToString(deltaApplyInfo.NcaIdNew).ToLower()}.nca";
var length = new FileInfo(Path.Combine(folderPath, matching)).Length;
var hexLen = Math.Ceiling(Math.Log(length, 16.0));
if (deltaApplyInfo.FragmentCount > 1)
{
matching += $":{offset.ToString($"X{hexLen}")}:{0.ToString($"X{hexLen}")}";
}
var lowerNcaID = Utils.BytesToString(cnmtExtended.DeltaContents[DeltaContentID].NcaId)
.ToLower();
var ncaFileName = Path.Combine(folderPath, $"{lowerNcaID}.nca");
if (!File.Exists(ncaFileName))
{
Out.Warn($"[WARN] File: {ncaFileName} not found!\r\n");
break;
}
Out.Log($"{ncaFileName}\r\n");
var ncaStorage = new StreamStorage(new FileStream(ncaFileName, FileMode.Open, FileAccess.Read),
false);
var DecryptedHeader = new byte[0xC00];
ncaStorage.Read(DecryptedHeader, 0, 0xC00, 0);
var Header = new NcaHeader(new BinaryReader(new MemoryStream(DecryptedHeader)), keyset);
var fragmentTrimmed = false;
for (var sector = 0; sector < 4; ++sector)
{
var section = NcaParseSection.ParseSection(Header, sector);
if (section == null || section.Header.Type != SectionType.Pfs0)
{
continue;
}
IStorage sectionStorage = ncaStorage.Slice(section.Offset, section.Size, false);
IStorage pfs0Storage = sectionStorage.Slice(section.Header.Sha256Info.DataOffset,
section.Header.Sha256Info.DataSize, false);
var Pfs0Header = new PartitionFileSystemHeader(new BinaryReader(pfs0Storage.AsStream()));
var FileDict = Pfs0Header.Files.ToDictionary(x => x.Name, x => x);
var path = PathTools.Normalize("fragment").TrimStart('/');
if (FileDict.TryGetValue(path, out var fragmentFile))
{
if (Pfs0Header.NumFiles != 1)
{
throw new InvalidDataException(
"A fragment Pfs0 container should contain exactly 1 file");
}
if (fragmentTrimmed)
{
throw new InvalidDataException(
"Multiple fragments in NCA found!");
}
IStorage fragmentStorage = pfs0Storage.Slice(
Pfs0Header.HeaderSize + fragmentFile.Offset,
fragmentFile.Size, false);
var buffer = new byte[0xC00];
fragmentStorage.Read(buffer, 0, buffer.Length, 0);
var writerPath = Path.Combine(folderPath, $"{lowerNcaID}.tca");
var writer = File.Open(writerPath, FileMode.Create);
var offsetBefore = section.Offset + section.Header.Sha256Info.DataOffset +
Pfs0Header.HeaderSize +
fragmentFile.Offset;
var offsetAfter = offsetBefore + fragmentFile.Size;
IStorage ncaStorageBeforeFragment = ncaStorage.Slice(0, offsetBefore, false);
IStorage ncaStorageAfterFragment =
ncaStorage.Slice(offsetAfter, ncaStorage.GetSize() - offsetAfter, false);
ncaStorageBeforeFragment.CopyToStream(writer);
offset += SaveDeltaHeader.Save(fragmentStorage, writer, matching);
ncaStorageAfterFragment.CopyToStream(writer);
writer.Position = 0x200;
writer.WriteByte(0x54);
writer.Dispose();
fragmentTrimmed = true;
}
}
ncaStorage.Dispose();
File.Delete(ncaFileName);
++DeltaContentID;
}
Out.Log("----------\r\n");
}
}
//static string path = "Databases/";
public static void Main6()
{
Random rnd = new Random();
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
Console.WriteLine("Start demo of table and Universal Index with PagedStorage and id as string");
// Тип основной таблицы
PType tp_person = new PTypeRecord(
new NamedType("id", new PType(PTypeEnumeration.sstring)),
new NamedType("name", new PType(PTypeEnumeration.sstring)),
new NamedType("age", new PType(PTypeEnumeration.real)));
// ======================== Страничное хранилище возьмем из описания PagedStreamStore =========================
//PagedStreamStore ps_store = new PagedStreamStore(path + "storage.bin", 4); // заказали 4 стрима, конкретные будут: ps_store[i]
StreamStorage ps_store = new StreamStorage(dbpath + "storage6uindex.bin", 4);
// База данных такая же, как и в программе 4, только идентификатор определен как строка, придется воспользоваться полуключем
TableView tab_person;
IndexHalfkeyImmutable <string> index_id_arr;
IndexDynamic <string, IndexHalfkeyImmutable <string> > index_id;
// Подключение к таблице
tab_person = new TableView(ps_store[0], tp_person);
// Подключение к индексу по уникальному идентификатору (нулевое поле)
Func <object, string> person_id_keyproducer = v => (string)((object[])((object[])v)[1])[0];
index_id_arr = new IndexHalfkeyImmutable <string>(ps_store[1])
{
Table = tab_person,
KeyProducer = person_id_keyproducer,
HalfProducer = s => Hashfunctions.HashRot13(s),
Scale = null
};
index_id_arr.Scale = new ScaleCell(ps_store[2])
{
IndexCell = index_id_arr.IndexCell
};
index_id = new IndexDynamic <string, IndexHalfkeyImmutable <string> >(true)
{
Table = tab_person,
IndexArray = index_id_arr,
KeyProducer = person_id_keyproducer
};
tab_person.RegisterIndex(index_id);
int nelements = 1000000;
bool toload = false; // Загружать или нет новую базу данных
if (toload)
{
sw.Restart();
// Очистим ячейки последовательности и индекса
tab_person.Clear();
IEnumerable <object> flow = Enumerable.Range(0, nelements)
.Select(i =>
{
int id = nelements - i;
string name = "=" + id.ToString() + "=";
double age = rnd.NextDouble() * 100.0;
return(new object[] { id.ToString(), name, age });
});
tab_person.Fill(flow);
// Теперь надо отсортировать индексный массив по ключу
tab_person.BuildIndexes();
sw.Stop();
Console.WriteLine("Load ok. duration for {0} elements: {1} ms", nelements, sw.ElapsedMilliseconds);
}
else
{
// Загрузка кеша ускоряет обработку
sw.Restart();
ps_store.LoadCache();
sw.Stop();
Console.WriteLine("Cache load duration={0}", sw.ElapsedMilliseconds);
}
// Проверим работу
int search_key = nelements * 2 / 3;
var ob = index_id.GetAllByKey(search_key.ToString())
.Select(ent => ((object[])ent.Get())[1])
.FirstOrDefault();
if (ob == null)
{
throw new Exception("Didn't find person " + search_key);
}
Console.WriteLine("Person {0} has name {1}", search_key, ((object[])ob)[1]);
// Засечем скорость выборок
int nprobes = 10000;
sw.Restart();
for (int i = 0; i < nprobes; i++)
{
search_key = rnd.Next(nelements) + 1;
ob = index_id.GetAllByKey(search_key.ToString())
.Select(ent => ((object[])ent.Get())[1])
.FirstOrDefault();
if (ob == null)
{
throw new Exception("Didn't find person " + search_key);
}
string nam = (string)((object[])ob)[1];
}
sw.Stop();
Console.WriteLine("Duration for {0} search in {1} elements: {2} ms", nprobes, nelements, sw.ElapsedMilliseconds);
}
public void Given_MongoDB_Is_Running_ItemStorage_Can_Successfully_Connect()
{
var storage = new StreamStorage(ConnectionString, DatabaseName);
Assert.IsNotNull(storage);
}
public void Given_Existing_10_Items_GetLatest_Returns_Limited_Number_Of_Items()
{
var limit = 1;
var items = BuildItems(count: 10).OrderBy(i => i.Published);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(null, null, null, limit).ToList();
Assert.AreEqual(limit, gotItems.Count);
}
public void ItemStorage_Can_Save_5_New_And_5_Modified_Items()
{
this.Items.InsertBatch(BuildItems(count: 5));
var existingItems = this.Items.AsQueryable().ToList();
foreach (var item in existingItems)
{
item.Tags.Add("test");
}
var newItems = BuildItems(count: 5);
var items = existingItems.Union(newItems).ToList();
var storage = new StreamStorage(ConnectionString, DatabaseName);
storage.Save(items);
var savedItems = this.Items.AsQueryable().ToList();
Assert.That(items.Count(), Is.EqualTo(savedItems.Count));
}
public static void Process(Context ctx)
{
using (var file = new StreamStorage(new FileStream(ctx.Options.InFile, FileMode.Open, FileAccess.Read), false))
{
var nca = new Nca(ctx.Keyset, file, false);
nca.ValidateMasterHashes();
nca.ParseNpdm();
if (ctx.Options.BaseNca != null)
{
var baseFile = new StreamStorage(new FileStream(ctx.Options.BaseNca, FileMode.Open, FileAccess.Read), false);
var baseNca = new Nca(ctx.Keyset, baseFile, false);
nca.SetBaseNca(baseNca);
}
for (int i = 0; i < 3; i++)
{
if (ctx.Options.SectionOut[i] != null)
{
nca.ExportSection(i, ctx.Options.SectionOut[i], ctx.Options.Raw, ctx.Options.IntegrityLevel, ctx.Logger);
}
if (ctx.Options.SectionOutDir[i] != null)
{
nca.ExtractSection(i, ctx.Options.SectionOutDir[i], ctx.Options.IntegrityLevel, ctx.Logger);
}
if (ctx.Options.Validate && nca.Sections[i] != null)
{
nca.VerifySection(i, ctx.Logger);
}
}
if (ctx.Options.ListRomFs && nca.Sections[1] != null)
{
var romfs = new Romfs(nca.OpenSection(1, false, ctx.Options.IntegrityLevel, true));
foreach (RomfsFile romfsFile in romfs.Files)
{
ctx.Logger.LogMessage(romfsFile.FullPath);
}
}
if (ctx.Options.RomfsOutDir != null || ctx.Options.RomfsOut != null)
{
NcaSection section = nca.Sections.FirstOrDefault(x => x?.Type == SectionType.Romfs || x?.Type == SectionType.Bktr);
if (section == null)
{
ctx.Logger.LogMessage("NCA has no RomFS section");
return;
}
if (section.Type == SectionType.Bktr && ctx.Options.BaseNca == null)
{
ctx.Logger.LogMessage("Cannot save BKTR section without base RomFS");
return;
}
if (ctx.Options.RomfsOut != null)
{
nca.ExportSection(section.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Options.IntegrityLevel, ctx.Logger);
}
if (ctx.Options.RomfsOutDir != null)
{
var romfs = new Romfs(nca.OpenSection(section.SectionNum, false, ctx.Options.IntegrityLevel, true));
romfs.Extract(ctx.Options.RomfsOutDir, ctx.Logger);
}
}
if (ctx.Options.ExefsOutDir != null || ctx.Options.ExefsOut != null)
{
if (nca.Header.ContentType != ContentType.Program)
{
ctx.Logger.LogMessage("NCA's content type is not \"Program\"");
return;
}
NcaSection section = nca.Sections[(int)ProgramPartitionType.Code];
if (section == null)
{
ctx.Logger.LogMessage("Could not find an ExeFS section");
return;
}
if (ctx.Options.ExefsOut != null)
{
nca.ExportSection(section.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Options.IntegrityLevel, ctx.Logger);
}
if (ctx.Options.ExefsOutDir != null)
{
nca.ExtractSection(section.SectionNum, ctx.Options.ExefsOutDir, ctx.Options.IntegrityLevel, ctx.Logger);
}
}
if (ctx.Options.PlaintextOut != null)
{
nca.OpenDecryptedNca().WriteAllBytes(ctx.Options.PlaintextOut, ctx.Logger);
}
ctx.Logger.LogMessage(nca.Print());
}
}
public void ItemStorage_Can_Save_10_New_Items()
{
var items = BuildItems(count: 10);
var storage = new StreamStorage(ConnectionString, DatabaseName);
storage.Save(items);
var savedItems = this.Items.AsQueryable().ToList();
Assert.That(items.Count, Is.EqualTo(savedItems.Count));
}
public void ItemStorage_Can_Save_5_Modified_Items_And_Retrieve_Them_Back()
{
this.Items.InsertBatch(BuildItems(count: 5));
var items = this.Items.AsQueryable().ToList();
foreach (var item in items)
{
item.Tags.Add("test");
}
var storage = new StreamStorage(ConnectionString, DatabaseName);
storage.Save(items);
var savedItems = this.Items.AsQueryable().ToList();
foreach (var item in savedItems)
{
Assert.IsTrue(item.Tags.Contains("test"));
}
}
public void Given_Non_Valid_Id_Get_Returns_Null()
{
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItem = storage.Get("id");
Assert.IsNull(gotItem);
}
public void Given_Existing_Item_Get_Returns_It_By_Id()
{
var item = BuildItems(count: 1).Single();
this.Items.Insert(item);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItem = storage.Get(item.Id);
Assert.IsNotNull(gotItem);
Assert.AreEqual(item.Url, gotItem.Url);
}
public void Given_Existing_10_Items_Top_Returns_Most_Recent_Item()
{
var items = BuildItems(count: 10).OrderBy(i => i.Published);
var recentItem = items.Last();
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItem = storage.Top();
Assert.AreEqual(recentItem.Url, gotItem.Url);
}
public void Given_Existing_10_Items_GetLatest_Returns_Them_Sorted_By_Date_Descending()
{
var items = BuildItems(count: 10);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(null, null, null, null).ToList();
Assert.That(gotItems, Is.Ordered.Descending.By("Published"));
}
public void Given_Existing_10_Items_GetLatest_Returns_Rss_Items_Only()
{
var items = BuildItems(count: 10).OrderBy(i => i.Published);
var numberOfRssItems = items.Count(i => i.ItemType == ItemType.Rss);
this.Items.InsertBatch(items);
var storage = new StreamStorage(ConnectionString, DatabaseName);
var gotItems = storage.GetLatest(ItemType.Rss, null, null, null).ToList();
Assert.AreEqual(numberOfRssItems, gotItems.Count);
}
// Проверка скорости ввода данных
public static void Main9()
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
Console.WriteLine("Start Program9");
int mbytes = 800;
int nelements = mbytes / 8 * 1024 * 1024;
// ======== Проверяем ввод данных через IO =======
//FileStream fs = new FileStream(path + "filestream.bin", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, 10000000);
FileStream fs = new FileStream(dbpath + "filestream.bin", FileMode.OpenOrCreate, FileAccess.ReadWrite);
byte[] buffer = new byte[2048 * 32];
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = (byte)(i & 255);
}
sw.Restart();
for (int i = 0; i < 16 * mbytes; i++)
{
fs.Write(buffer, 0, buffer.Length);
}
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds} ms. for writing {mbytes} mbytes");
// ======== Проверяем ввод данных через BinaryWriter =======
fs.Position = 0L;
BinaryWriter bw = new BinaryWriter(fs);
long val = 29877777298392742L;
sw.Restart();
for (int i = 0; i < mbytes / 8 * 1024 * 1024; i++)
{
bw.Write(val);
}
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds} ms. for writing {mbytes / 8 * 1024 * 1024} long numbers");
// ======== Проверяем ввод данных через ячейку =======
PaCell cell = new PaCell(new PTypeSequence(new PType(PTypeEnumeration.longinteger)), dbpath + "sequ_long.pac", false);
cell.Clear();
cell.Fill(new object[0]);
sw.Restart();
for (int i = 0; i < nelements; i++)
{
cell.Root.AppendElement((long)(nelements - i));
}
cell.Flush();
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds} ms. for writing {mbytes / 8 * 1024 * 1024} long elements");
// ======== Проверяем ввод данных через ячейку =======
PType tp_rec = new PTypeRecord(
new NamedType("f1", new PType(PTypeEnumeration.integer)),
new NamedType("f2", new PType(PTypeEnumeration.sstring)),
new NamedType("f3", new PType(PTypeEnumeration.real)));
PaCell cell2 = new PaCell(new PTypeSequence(tp_rec), dbpath + "sequ_rec.pac", false);
cell2.Clear();
cell2.Fill(new object[0]);
sw.Restart();
int ne = nelements / 3;
for (int i = 0; i < ne; i++)
{
int id = nelements - i;
cell2.Root.AppendElement(new object[] { id, "=" + id, 5.5 });
}
cell2.Flush();
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds} ms. for writing {ne} record elements");
// ======== Проверяем ввод данных через ячейку =======
PType tp_rec3 = new PTypeRecord(
new NamedType("f1", new PType(PTypeEnumeration.integer)),
new NamedType("f2", new PType(PTypeEnumeration.sstring)),
new NamedType("f3", new PType(PTypeEnumeration.real)));
Stream stream = new FileStream(dbpath + "fstream.bin", FileMode.OpenOrCreate, FileAccess.ReadWrite); //new MemoryStream();
BinaryWriter bw3 = new BinaryWriter(stream);
sw.Restart();
int ne3 = nelements / 3;
for (int i = 0; i < ne3; i++)
{
int id = nelements - i;
PaCell.SetPO(tp_rec3, bw3, new object[] { id, "=" + id, 5.5 });
}
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds} ms. for writing {ne3} record elements to memory stream");
// ======== Проверяем ввод данных через ячейку и страничное хранилище потоков =======
PType tp_rec4 = new PTypeRecord(
new NamedType("f1", new PType(PTypeEnumeration.integer)),
new NamedType("f2", new PType(PTypeEnumeration.sstring)),
new NamedType("f3", new PType(PTypeEnumeration.real)));
//PagedStreamStore ps_store = new PagedStreamStore(path + "storage.bin", 4); // заказали 4 стрима, конкретные будут: ps_store[i]
StreamStorage ps_store = new StreamStorage(dbpath + "storage9.bin", 4);
ps_store.DeactivateCache();
PaCell cell4 = new PaCell(new PTypeSequence(tp_rec), ps_store[0], false);
cell4.Clear();
cell4.Fill(new object[0]);
sw.Restart();
int ne4 = nelements / 3;
for (int i = 0; i < ne4; i++)
{
int id = nelements - i;
cell4.Root.AppendElement(new object[] { id, "=" + id, 5.5 });
}
cell4.Flush();
sw.Stop();
Console.WriteLine($"{sw.ElapsedMilliseconds} ms. for storing {ne} record elements");
}
public static void Process(string folderPath, IFileSystem newBaseFolderFs, Keyset keyset, Output Out)
{
var dirDecrypted = new DirectoryInfo(folderPath);
foreach (var inFile in dirDecrypted.GetFiles("*.tca"))
{
Out.Log($"{inFile}\r\n");
var ncaStorage = new StreamStorage(new FileStream(inFile.FullName, FileMode.Open, FileAccess.Read),
false);
var DecryptedHeader = new byte[0xC00];
ncaStorage.Read(DecryptedHeader, 0, 0xC00, 0);
var Header = new NcaHeader(new BinaryReader(new MemoryStream(DecryptedHeader)), keyset);
var fragmentTrimmed = false;
for (var i = 0; i < 4; ++i)
{
var section = NcaParseSection.ParseSection(Header, i);
if (section == null || section.Header.Type != SectionType.Pfs0)
{
continue;
}
if (fragmentTrimmed)
{
Out.Warn(
"Multiple fragments in NCA found! Skip trimming this fragment.\r\n");
continue;
}
IStorage sectionStorage = ncaStorage.Slice(section.Offset, section.Size, false);
IStorage pfs0Storage = sectionStorage.Slice(section.Header.Sha256Info.DataOffset,
section.Header.Sha256Info.DataSize, false);
var Pfs0Header = new PartitionFileSystemHeader(new BinaryReader(pfs0Storage.AsStream()));
var FileDict = Pfs0Header.Files.ToDictionary(x => x.Name, x => x);
var path = PathTools.Normalize(FragmentFileName).TrimStart('/');
if (Pfs0Header.NumFiles == 1 && FileDict.TryGetValue(path, out var fragmentFile))
{
var inFileNameNoExtension = Path.GetFileNameWithoutExtension(inFile.Name);
var writer = File.Open($"{folderPath}/{inFileNameNoExtension}.nca", FileMode.Create);
var offsetBefore = section.Offset + section.Header.Sha256Info.DataOffset +
Pfs0Header.HeaderSize +
fragmentFile.Offset;
IStorage ncaStorageBeforeFragment = ncaStorage.Slice(0, offsetBefore, false);
IStorage fragmentStorageOverflow = ncaStorage.Slice(offsetBefore,
ncaStorage.GetSize() - offsetBefore, false);
ncaStorageBeforeFragment.CopyToStream(writer);
var TDV0len = RecreateDelta.Recreate(fragmentStorageOverflow, writer, newBaseFolderFs);
var offsetAfter = offsetBefore + TDV0len;
IStorage fragmentStorageAfter = ncaStorage.Slice(offsetAfter,
ncaStorage.GetSize() - offsetAfter, false);
fragmentStorageAfter.CopyToStream(writer);
writer.Position = 0x200;
writer.WriteByte(0x4E);
writer.Dispose();
fragmentTrimmed = true;
}
}
ncaStorage.Dispose();
File.Delete(inFile.FullName);
}
}
