private static void ReadDat(FileInfo[] fis, string subPath, uint dirId, bool extraDir)
{
foreach (FileInfo f in fis)
{
_datsProcessed++;
_bgw.ReportProgress(_datsProcessed);
uint?datId = FindDat(subPath, f.Name, f.LastWriteTime, extraDir);
if (datId != null)
{
SetDatFound((uint)datId);
continue;
}
_bgw.ReportProgress(0, new bgwText("Dat : " + subPath + @"\" + f.Name));
if (DatRead.ReadDat(f.FullName, ReadError, out DatHeader dh))
//if (DatReader.DatReader.ReadDat(f.FullName, _bgw, out RvDat rvDat))
{
RvDat rvDat = ExternalDatConverter.ConvertFromExternalDat(f.FullName, dh);
uint nextDirId = dirId;
if (extraDir)
{
string extraDirName = VarFix.CleanFileName(rvDat.GetExtraDirName()); // read this from dat.
nextDirId = RvDir.FindOrInsertIntoDir(dirId, extraDirName, Path.Combine(subPath, extraDirName) + "\\");
}
rvDat.DirId = nextDirId;
rvDat.ExtraDir = extraDir;
rvDat.Path = subPath;
rvDat.DatTimeStamp = f.LastWriteTime;
DatSetRemoveUnneededDirs(rvDat);
DatSetCheckParentSets(rvDat);
DatSetRenameAndRemoveDups(rvDat);
if ((rvDat.MergeType ?? "").ToLower() == "full")
{
DatSetMergeSets(rvDat);
}
DatSetCheckCollect(rvDat);
DBSqlite.db.Commit();
DBSqlite.db.Begin();
rvDat.DbWrite();
DBSqlite.db.Commit();
DBSqlite.db.Begin();
}
if (_bgw.CancellationPending)
{
return;
}
}
}
public static void test()
{
string datFullName = @"D:\baddat.xml";
_file = new StreamWriter(@"D:\timeout.txt");
DatRead dr = new DatRead
{
ErrorReport = ReadError
};
dr.ReadDat(datFullName, out DatHeader dh);
DatClean.CleanFilenames(dh.BaseDir);
//DatClean.DirectoryExpand(dh.BaseDir);
//DatClean.RemoveNonCHD(dh.BaseDir);
DatClean.RemoveCHD(dh.BaseDir);
DatClean.RemoveNoDumps(dh.BaseDir);
SetMergeType(MergeType.Split, dh);
//if (datRule.SingleArchive)
// DatClean.MakeDatSingleLevel(dh);
SetCompressionMethod(FileType.Zip, dh);
DatClean.DirectoryExpand(dh.BaseDir);
_file.Close();
}
public static RvFile ReadInDatFile(RvDat datFile, ThreadWorker thWrk, out string extraDirName)
{
try
{
_thWrk = thWrk;
extraDirName = null;
string datRootFullName = datFile.GetData(RvDat.DatData.DatRootFullName);
string fullPath = RvFile.GetDatPhysicalPath(datRootFullName);
Debug.WriteLine("Reading Dat " + fullPath);
DatRead.ReadDat(fullPath, ReadError, out DatHeader dh);
if (dh == null)
{
return(null);
}
string dirNameRule = Path.GetDirectoryName(datRootFullName) + Path.DirSeparatorChar;
if (
!datFile.MultiDatOverride && dh.Dir != "noautodir" &&
(datFile.MultiDatsInDirectory || !string.IsNullOrEmpty(dh.RootDir))
)
{
// if we are auto adding extra directories then create a new directory.
extraDirName = "";
if (string.IsNullOrEmpty(extraDirName) && datFile.UseDescriptionAsDirName && !string.IsNullOrWhiteSpace(dh.Description))
{
extraDirName = dh.Description;
}
if (string.IsNullOrEmpty(extraDirName))
{
extraDirName = dh.RootDir;
}
if (string.IsNullOrEmpty(extraDirName))
{
extraDirName = dh.Name;
}
if (string.IsNullOrEmpty(extraDirName))
{
extraDirName = Path.GetFileNameWithoutExtension(fullPath);
}
dirNameRule += VarFix.CleanFileName(extraDirName) + Path.DirSeparatorChar;
}
ReportError.LogOut($"DatRule {dirNameRule}");
DatRule datRule = FindDatRule(dirNameRule);
DatClean.CleanFilenames(dh.BaseDir);
switch (datRule.Filter)
{
case FilterType.CHDsOnly:
DatClean.RemoveNonCHD(dh.BaseDir);
break;
case FilterType.RomsOnly:
DatClean.RemoveCHD(dh.BaseDir);
break;
}
DatClean.RemoveNoDumps(dh.BaseDir);
SetMergeType(datRule, dh);
if (datRule.SingleArchive)
{
DatClean.MakeDatSingleLevel(dh, datRule.UseDescriptionAsDirName, datRule.SubDirType, isFile(datRule, dh));
}
DatClean.RemoveUnNeededDirectories(dh.BaseDir);
SetCompressionMethod(datRule, dh); // This sorts the files into the required dir order for the set compression type. (And also sets '\' characters to '/' in zip files.)
DatClean.DirectoryExpand(dh.BaseDir); // this works because we only expand files, so the order inside the zip / 7z does not matter.
DatClean.RemoveEmptyDirectories(dh.BaseDir);
DatClean.CleanFilenamesFixDupes(dh.BaseDir); // you may get repeat filenames inside Zip's / 7Z's and they may not be sorted to find them by now.
RvFile newDir = ExternalDatConverter.ConvertFromExternalDat(dh, datFile);
return(newDir);
}
catch (Exception e)
{
string datRootFullName = datFile?.GetData(RvDat.DatData.DatRootFullName);
Console.WriteLine(e);
throw new Exception("Error is DAT " + datRootFullName + " " + e.Message);
}
}
private static void ProcVer(string ver)
{
WriteLine();
WriteLine();
_lastTime = null;
Sw.Reset();
Sw.Start();
DatRead dr = new DatRead
{
ErrorReport = ReadError
};
DatXMLWriter dxw = new DatXMLWriter();
WriteLine(ver, "Reading BINDat Set");
dr.ReadDat(@"TestDATs\MAME " + ver + " ROMS (from bin).xml", out DatHeader dh);
dh.Name += " (split)";
dh.Description += " (split)";
WriteLine(ver, "Dat read");
DatClean.RemoveCHD(dh.BaseDir);
WriteLine(ver, "CHD removed");
DatClean.RemoveNoDumps(dh.BaseDir);
WriteLine(ver, "Removed No Dumps");
DatClean.DatSetMakeSplitSet(dh.BaseDir);
WriteLine(ver, "Made Split Set");
DatClean.RemoveNotCollected(dh.BaseDir);
WriteLine(ver, "Removed Not Collected");
DatClean.RemoveDupes(dh.BaseDir);
WriteLine(ver, "Removed Dupes");
DatClean.RemoveEmptySets(dh.BaseDir);
WriteLine(ver, "Removed Empty Sets");
DatSetTorrentZip.SetTorrentZip(dh.BaseDir);
WriteLine(ver, "Set TorrentZip");
dxw.WriteDat(@"TestDATs\out\MAME " + ver + " ROMS (split-fromBin).xml", dh);
WriteLine(ver, "Reading Dat Set");
dr.ReadDat(@"TestDATs\MAME " + ver + " ROMs (split).xml", out dh);
DatSetTorrentZip.SetTorrentZip(dh.BaseDir);
dxw.WriteDat(@"TestDATs\out\MAME " + ver + " ROMs (split-sorted).xml", dh);
WriteLine(ver, "Done Set 1");
WriteLine(ver, "Reading BINDat Set");
dr.ReadDat(@"TestDATs\MAME " + ver + " ROMS (from bin).xml", out dh);
dh.Name += " (merged)";
dh.Description += " (merged)";
WriteLine(ver, "Dat read");
DatClean.RemoveCHD(dh.BaseDir);
WriteLine(ver, "CHD removed");
DatClean.RemoveNoDumps(dh.BaseDir);
WriteLine(ver, "Removed No Dumps");
DatClean.DatSetMakeMergeSet(dh.BaseDir);
WriteLine(ver, "Made Merge Set");
DatClean.RemoveDupes(dh.BaseDir);
WriteLine(ver, "Removed Dupes");
DatClean.RemoveEmptySets(dh.BaseDir);
WriteLine(ver, "Removed Empty Sets");
DatSetTorrentZip.SetTorrentZip(dh.BaseDir);
WriteLine(ver, "Set TorrentZip");
dxw.WriteDat(@"TestDATs\out\MAME " + ver + " ROMS (merged-fromBin).xml", dh);
WriteLine(ver, "Reading Dat Set");
dr.ReadDat(@"TestDATs\MAME " + ver + " ROMs (merged).xml", out dh);
DatSetTorrentZip.SetTorrentZip(dh.BaseDir);
dxw.WriteDat(@"TestDATs\out\MAME " + ver + " ROMs (merged-sorted).xml", dh);
WriteLine(ver, "Done Set 2");
WriteLine(ver, "Reading BINDat Set");
dr.ReadDat(@"TestDATs\MAME " + ver + " ROMS (from bin).xml", out dh);
dh.Name += " (non-merged)";
dh.Description += " (non-merged)";
WriteLine(ver, "Dat read");
DatClean.RemoveCHD(dh.BaseDir);
WriteLine(ver, "CHD removed");
DatClean.RemoveNoDumps(dh.BaseDir);
WriteLine(ver, "Removed No Dumps");
DatClean.DatSetMakeNonMergeSet(dh.BaseDir);
WriteLine(ver, "Made Merge Set");
DatClean.RemoveDupes(dh.BaseDir);
WriteLine(ver, "Removed Dupes");
DatClean.RemoveEmptySets(dh.BaseDir);
WriteLine(ver, "Removed Empty Sets");
DatSetTorrentZip.SetTorrentZip(dh.BaseDir);
WriteLine(ver, "Set TorrentZip");
dxw.WriteDat(@"TestDATs\out\MAME " + ver + " ROMS (non-merged-fromBin).xml", dh);
WriteLine(ver, "Reading Dat Set");
dr.ReadDat(@"TestDATs\MAME " + ver + " ROMs (non-merged).xml", out dh);
DatSetTorrentZip.SetTorrentZip(dh.BaseDir);
dxw.WriteDat(@"TestDATs\out\MAME " + ver + " ROMs (non-merged-sorted).xml", dh);
WriteLine(ver, "Done Set 3");
}
public static RvFile ReadInDatFile(RvDat datFile, ThreadWorker thWrk)
{
try
{
_thWrk = thWrk;
string datRootFullName = datFile.GetData(RvDat.DatData.DatRootFullName);
DatRead dr = new DatRead
{
ErrorReport = ReadError
};
string fullPath = RvFile.GetDatPhysicalPath(datRootFullName);
Debug.WriteLine("Reading Dat " + fullPath);
dr.ReadDat(fullPath, out DatHeader dh);
if (dh == null)
{
return(null);
}
string extraPath = !string.IsNullOrEmpty(dh.RootDir) ?
dh.RootDir : dh.Name;
string dirName = Path.GetDirectoryName(datRootFullName) + System.IO.Path.DirectorySeparatorChar + extraPath + System.IO.Path.DirectorySeparatorChar;
DatRule datRule = FindDatRule(dirName);
DatClean.CleanFilenames(dh.BaseDir);
switch (datRule.Filter)
{
case FilterType.CHDsOnly:
DatClean.RemoveNonCHD(dh.BaseDir);
break;
case FilterType.RomsOnly:
DatClean.RemoveCHD(dh.BaseDir);
break;
}
DatClean.RemoveNoDumps(dh.BaseDir);
SetMergeType(datRule, dh);
if (datRule.SingleArchive)
{
DatClean.MakeDatSingleLevel(dh);
}
DatClean.RemoveUnNeededDirectories(dh.BaseDir);
SetCompressionMethod(datRule, dh); // This sorts the files into the required dir order for the set compression type.
DatClean.DirectoryExpand(dh.BaseDir); // this works because we only expand files, so the order inside the zip / 7z does not matter.
DatClean.RemoveEmptyDirectories(dh.BaseDir);
DatClean.CleanFilenamesFixDupes(dh.BaseDir); // you may get repeat filenames inside Zip's / 7Z's and they may not be sorted to find them by now.
RvFile newDir = ExternalDatConverter.ConvertFromExternalDat(dh, datRootFullName, datFile.TimeStamp);
return(newDir);
}
catch (Exception e)
{
string datRootFullName = datFile?.GetData(RvDat.DatData.DatRootFullName);
Console.WriteLine(e);
throw new Exception("Error is DAT " + datRootFullName + " " + e.Message);
}
}
private static void readerDir(string dirName)
{
DirectoryInfo di = new DirectoryInfo(dirName);
FileInfo[] aFI = di.GetFiles();
foreach (FileInfo f in aFI)
{
string ext = Path.GetExtension(f.Name).ToLower();
if (ext == ".ini" || ext == ".txt")
{
continue;
}
if (ext != ".dat" && ext != ".xml")
{
continue;
}
DatRead dr = new DatRead
{
ErrorReport = ReadError
};
DatHeader dh;
dr.ReadDat(f.FullName, out dh);
if (dh != null)
{
DatTypeTester r = new DatTypeTester();
r.ProcessDat(dh);
string header = dh.Header ?? "";
string compression = dh.Compression ?? "";
string mergeType = dh.MergeType ?? "";
string noDump = dh.NoDump ?? "";
string dir = dh.Dir ?? "";
string outtxt = header + "," + compression + "," + mergeType + "," + noDump + "," + dir + "," + dh.IsSuperDat + "," + dh.NotZipped;
if (outtxt != ",,,,,False,False" || r.Found())
{
Console.WriteLine(outtxt + "," + r.toString() + "," + f.FullName);
Debug.WriteLine(outtxt + "," + r.toString() + "," + f.FullName);
}
/*
* if (!string.IsNullOrWhiteSpace(dh.Header)) Console.WriteLine("Header : " + dh.Header);
* if (!string.IsNullOrWhiteSpace(dh.Compression)) Console.WriteLine("Compression : " + dh.Compression);
* if (!string.IsNullOrWhiteSpace(dh.MergeType)) Console.WriteLine("MergeType : " + dh.MergeType);
* if (!string.IsNullOrWhiteSpace(dh.NoDump)) Console.WriteLine("NoDump : " + dh.NoDump);
* if (!string.IsNullOrWhiteSpace(dh.Dir)) Console.WriteLine("Dir : " + dh.Dir);
* if (dh.IsSuperDat) Console.WriteLine("IsSuperDat = true");
* if (dh.NotZipped) Console.WriteLine("NotZipped = true");
*/
/*
* string outpath = @"D:\out\" + f.FullName.Substring(3);
*
* if (outpath.Length > 260)
* {
* // Console.WriteLine("Long filename " + outpath.Length);
* // Console.WriteLine(outpath);
* }
*
* string outDir = Path.GetDirectoryName(outpath);
* DatXMLWriter dxw = new DatXMLWriter();
*
* if (!Directory.Exists(outDir))
* Directory.CreateDirectory(outDir);
* dxw.WriteDat(outpath, dh);
*/
}
}
DirectoryInfo[] adi = di.GetDirectories();
foreach (DirectoryInfo d in adi)
{
readerDir(d.FullName);
}
}
