public void get_column_index_or_null_returns_column_index_if_present()
{
var header = new HeaderRecord(new string[] { "Name", "Age", "Gender" });
Assert.Equal(0, header.GetColumnIndexOrNull("Name"));
Assert.Equal(1, header.GetColumnIndexOrNull("Age"));
Assert.Equal(2, header.GetColumnIndexOrNull("Gender"));
}
public void indexer_returns_column_index()
{
var header = new HeaderRecord(new string[] { "Name", "Age", "Gender" });
Assert.Equal(0, header["Name"]);
Assert.Equal(1, header["Age"]);
Assert.Equal(2, header["Gender"]);
}
public void public_constructor_assigns_given_header_record()
{
var header = new HeaderRecord();
var data = new DataRecord(header);
Assert.NotNull(data.HeaderRecord);
Assert.Same(header, data.HeaderRecord);
}
public void indexer_throws_if_column_is_not_found()
{
var header = new HeaderRecord(new string[] { "Name", "Age", "Gender" });
var i = 0;
var ex = Assert.Throws<ArgumentException>(() => i = header["foobar"]);
Assert.Equal("No column named 'foobar' was found in the header record.", ex.Message);
}
public void get_column_index_or_null_returns_null_if_column_is_not_found()
{
var header = new HeaderRecord(new string[] { "Name", "Age", "Gender" });
Assert.Null(header.GetColumnIndexOrNull("Foo"));
Assert.Null(header.GetColumnIndexOrNull("name"));
Assert.Null(header.GetColumnIndexOrNull("Age "));
Assert.Null(header.GetColumnIndexOrNull("GENDER"));
}
public void public_constructor_assigns_columns_as_values()
{
var header = new HeaderRecord("Name", "Age", "Gender");
Assert.Equal(3, header.Count);
Assert.Equal("Name", header[0]);
Assert.Equal("Age", header[1]);
Assert.Equal("Gender", header[2]);
}
public DiskStream(Stream fileStream, Ownership ownsStream, HeaderRecord fileHeader)
{
_fileStream = fileStream;
_fileHeader = fileHeader;
_ownsStream = ownsStream;
ReadBlockTable();
}
public void header_record_creation()
{
var repeatCount = 1000000;
var creationCount = 0L;
for (var i = 0; i < repeatCount; ++i)
{
var headerRecord = new HeaderRecord();
// ensure optimization doesn't remove the object creation
creationCount += headerRecord.Count;
++creationCount;
}
Assert.Equal(repeatCount, creationCount);
}
private static bool IsHeaderMatch(ConstructFile constructFile, HeaderRecord fileHeader)
{
var titles = constructFile.GetColumnTitles();
for (int i = 0; i < fileHeader.Count; ++i)
{
if (i < titles.Length)
{
if (titles[i] != fileHeader[i])
{
return(false);
}
}
else
{
return(false);
}
}
return(true);
}
public DefineBitsLossless2Tag(HeaderRecord header, FlashReader input)
: base(header)
{
Id = input.ReadUInt16();
_format = input.ReadByte();
ushort width = input.ReadUInt16();
ushort height = input.ReadUInt16();
_argbMap = new Color[width, height];
if (_format == 3)
{
_colorTableSize = input.ReadByte();
}
int partialLength = (7 + (_format == 3 ? 1 : 0));
_compressedData = input.ReadBytes(header.Length - partialLength);
}
public DefineBitsJPEG3(HeaderRecord header, FlashReader input) : base(header)
{
Id = input.ReadUInt16();
var alphaDataOffset = input.ReadInt32();
Data = input.ReadBytes(alphaDataOffset);
Format = GetFormat(Data);
if (Format == ImageFormat.JPEG)
{
var partialLength = 2 + 4 + alphaDataOffset;
AlphaData = input.ReadBytes(Header.Length - partialLength);
}
else
{
// Minimum Compressed Empty Data Length
AlphaData = input.ReadBytes(8);
}
}
private void ParserTableStr(string content)
{
StringReader rdr = new StringReader(content);
using (var reader = new CsvReader(rdr))
{
HeaderRecord header = reader.ReadHeaderRecord();
while (reader.HasMoreRecords)
{
DataRecord data = reader.ReadDataRecord();
if (data[0].StartsWith("#"))
{
continue;
}
BuffInfoRecord record = new BuffInfoRecord(data);
Records.Add(record.Id, record);
}
}
}
public async Task WriteAsync(IList <CalendarEvent> events)
{
var headerRecord = new HeaderRecord("Subject", "Start Date", "Start Time", "End Date", "End Time", "Location");
var dataRecords = events.Select(o => new DataRecord(
null,
o.Title,
o.Start.ToString("M/d/yyyy"),
o.Start.ToString("H:mm:ss"),
o.End.ToString("M/d/yyyy"),
o.End.ToString("H:mm:ss"),
o.Location ?? string.Empty
));
var records = new RecordBase[] { headerRecord }.Union(dataRecords).ToArray();
using (var streamWriter = new StreamWriter(OutputFileName))
using (var writer = new CsvWriter(streamWriter))
{
writer.ForceDelimit = true;
await writer.WriteRecordsAsync(records, 0, records.Length);
}
}
// ---------------------------------------------------------------------------
private int GetFrameCount(HeaderRecord Header)
{
int result;
// Get frame count
if (4 == GetStreamVersion(Header))
{
result = Header.IntegerArray[1] >> 16;
}
else
if ((5 <= GetStreamVersion(Header)) && (GetStreamVersion(Header) <= 71))
{
result = Header.IntegerArray[1];
}
else
{
result = 0;
}
return(result);
}
private static Int32 getFrameCount(HeaderRecord Header)
{
Int32 result;
// Get frame count
if (40 == getStreamVersion(Header))
{
result = Header.IntegerArray[1] >> 16;
}
else
if ((50 <= getStreamVersion(Header)) && (getStreamVersion(Header) <= 71))
{
result = Header.IntegerArray[1];
}
else
{
result = 0;
}
return(result);
}
protected virtual TagItem ReadTag(HeaderRecord header, FlashReader input)
{
switch (header.Kind)
{
case TagKind.DefineBinaryData: return(new DefineBinaryDataTag(header, input));
case TagKind.DefineBitsJPEG3: return(new DefineBitsJPEG3(header, input));
case TagKind.DefineBitsLossless2: return(new DefineBitsLossless2Tag(header, input));
case TagKind.DefineFontName: return(new DefineFontNameTag(header, input));
case TagKind.DefineSound: return(new DefineSoundTag(header, input));
case TagKind.DoABC: return(new DoABCTag(header, input));
case TagKind.End: return(new EndTag(header));
case TagKind.ExportAssets: return(new ExportAssetsTag(header, input));
case TagKind.FileAttributes: return(new FileAttributesTag(header, input));
case TagKind.FrameLabel: return(new FrameLabelTag(header, input));
case TagKind.Metadata: return(new MetadataTag(header, input));
case TagKind.ProductInfo: return(new ProductInfoTag(header, input));
case TagKind.ScriptLimits: return(new ScriptLimitsTag(header, input));
case TagKind.SetBackgroundColor: return(new SetBackgroundColorTag(header, input));
case TagKind.ShowFrame: return(new ShowFrameTag(header));
case TagKind.SymbolClass: return(new SymbolClassTag(header, input));
default:
case TagKind.Unknown: return(new UnknownTag(header, input));
}
}
/// <summary>
/// Asynchronously writes the items in <paramref name="this"/> to <paramref name="csvWriter"/>.
/// </summary>
/// <remarks>
/// <para>
/// This overload provides maximum flexibility in how items are written CSV.
/// </para>
/// </remarks>
/// <typeparam name="T">
/// The type of the items to be written to <paramref name="csvWriter"/>.
/// </typeparam>
/// <param name="this">
/// The items to write.
/// </param>
/// <param name="csvWriter">
/// The <see cref="CsvWriter"/>.
/// </param>
/// <param name="header">
/// If non-<see langword="null"/>, this will be written to <paramref name="csvWriter"/> before any data records are written.
/// </param>
/// <param name="objectToRecordConverter">
/// Converts an item in <paramref name="this"/> to a CSV record.
/// </param>
/// <returns>
/// The number of items written.
/// </returns>
public async static Task <int> WriteCsvAsync <T>(this IEnumerable <T> @this, CsvWriter csvWriter, IEnumerable <string> header, Func <T, IEnumerable <string> > objectToRecordConverter)
{
@this.AssertNotNull("@this");
csvWriter.AssertNotNull("csvWriter");
objectToRecordConverter.AssertNotNull("objectToRecordConverter");
HeaderRecord headerRecord = null;
if (header != null)
{
headerRecord = new HeaderRecord(header);
await csvWriter.WriteRecordAsync(headerRecord).ConfigureAwait(false);
}
var num = 0;
var buffer = new DataRecord[16];
var bufferOffset = 0;
foreach (var item in @this)
{
var record = new DataRecord(headerRecord, objectToRecordConverter(item));
buffer[bufferOffset++] = record;
if (bufferOffset == buffer.Length)
{
// buffer full
await csvWriter.WriteRecordsAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
bufferOffset = 0;
}
++num;
}
// write any outstanding data in buffer
await csvWriter.WriteRecordsAsync(buffer, 0, bufferOffset).ConfigureAwait(false);
return(num);
}
public MainPage()
{
InitializeComponent();
ObservableCollection <TmiImage> tmiImageList = new ObservableCollection <TmiImage>();
try
{
StreamResourceInfo sri = null;
try
{
Uri csvUri = new Uri("TmiPivotMetadata.csv", UriKind.Relative);
sri = Application.GetResourceStream(csvUri);
}
catch (Exception exp2)
{
System.Diagnostics.Debug.WriteLine("### Exception while getting resource stream: {0}", exp2.ToString());
}
using (CsvReader reader = new CsvReader(sri.Stream))
{
HeaderRecord head = reader.ReadHeaderRecord();
while (reader.HasMoreRecords)
{
DataRecord rec = reader.ReadDataRecord();
tmiImageList.Add(createFromRecord(rec));
}
reader.Close();
}
}
catch (Exception exp)
{
System.Diagnostics.Debug.WriteLine("### Exception while reading csv file: {0}", exp.ToString());
}
MyPV.ItemsSource = tmiImageList;
}
private static void InitBoolean(ColumnMappingAttribute attribute, HeaderRecord record)
{
var importP = attribute.ImportParam as string;
if (importP != null)
{
int index = SearchHeadIndex(attribute, record);
if (index != -1)
{
attribute.getObject = r => string.Equals(r[index], importP, StringComparison.InvariantCultureIgnoreCase);
}
else
{
attribute.getObject = r => false;
}
}
else
{
ConverToInternal <bool>(attribute, record);
}
}
private static Byte getProfileID(HeaderRecord Header)
{
var result = MPP_PROFILE_UNKNOWN;
// Get MPEGplus profile (exists for stream version 7 only)
if ((70 == getStreamVersion(Header)) || (71 == getStreamVersion(Header)))
{
// ((and $F0) shr 4) is needed because samplerate is stored in the same byte!
switch (((Header.ByteArray[10] & 0xF0) >> 4))
{
case 1: result = MPP_PROFILE_EXPERIMENTAL; break;
case 5: result = MPP_PROFILE_QUALITY0; break;
case 6: result = MPP_PROFILE_QUALITY1; break;
case 7: result = MPP_PROFILE_TELEPHONE; break;
case 8: result = MPP_PROFILE_THUMB; break;
case 9: result = MPP_PROFILE_RADIO; break;
case 10: result = MPP_PROFILE_STANDARD; break;
case 11: result = MPP_PROFILE_XTREME; break;
case 12: result = MPP_PROFILE_INSANE; break;
case 13: result = MPP_PROFILE_BRAINDEAD; break;
case 14: result = MPP_PROFILE_QUALITY9; break;
case 15: result = MPP_PROFILE_QUALITY10; break;
}
}
return(result);
}
private static void InitSingleArray(ColumnMappingAttribute attribute, HeaderRecord record)
{
int index = SearchHeadIndex(attribute, record);
if (index != -1)
{
attribute.getObject = r =>
{
var str = r[index];
var strs = str.Split('_');
var count = strs.Length;
float[] values = new float[count];
float id;
for (int i = 0; i < count; i++)
{
if (string.IsNullOrEmpty(strs[i]))
{
values[i] = 0f;
continue;
}
if (float.TryParse(strs[i], out id))
{
values[i] = id;
}
else
{
Debug.LogErrorFormat("InitSingleArray float.TryParse(strs[i],out id Failed,strs:{0}:{1}:Count is {2}", str, attribute.TargetField.Name, count);
}
}
return(values);
}
}
;
else
{
attribute.getObject = r => string.Empty;
}
}
// ---------------------------------------------------------------------------
// No explicit destructors with C#
// ---------------------------------------------------------------------------
public bool ReadFromFile(String FileName)
{
HeaderRecord Header = new HeaderRecord();
bool result;
// Reset data and load header from file to variable
FResetData();
Array.Clear(Header.ByteArray, 0, Header.ByteArray.Length);
Array.Clear(Header.IntegerArray, 0, Header.IntegerArray.Length);
Header.FileSize = 0;
Header.ID3v2Size = 0;
// At first try to load ID3v2 tag data, then header
if (FID3v2.ReadFromFile(FileName))
{
Header.ID3v2Size = FID3v2.Size;
}
result = ReadHeader(FileName, ref Header);
// Process data if loaded and file valid
if ((result) && (Header.FileSize > 0) && (GetStreamVersion(Header) > 0))
{
FValid = true;
// Fill properties with header data
FSampleRate = GetSampleRate(Header);
FChannelModeID = GetChannelModeID(Header);
FFileSize = Header.FileSize;
FFrameCount = GetFrameCount(Header);
FBitRate = GetBitRate(Header);
FStreamVersion = GetStreamVersion(Header);
FProfileID = GetProfileID(Header);
FEncoder = GetEncoder(Header);
FID3v1.ReadFromFile(FileName);
FAPEtag.ReadFromFile(FileName);
}
return(result);
}
/****************************************************************/
private void btnOpenData_Click(object sender, EventArgs e)
{
OpenFileDialog openFileDialog1 = new OpenFileDialog();
//openFileDialog1.InitialDirectory = "c:\\";
openFileDialog1.Filter = "csv files (*.csv)|*.csv|All files (*.*)|*.*";
openFileDialog1.FilterIndex = 2;
openFileDialog1.RestoreDirectory = true;
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
try
{
using (var streamReader = new StreamReader(openFileDialog1.FileName))
using (var csvReader = new CsvReader(streamReader))
{
HeaderRecord header = csvReader.ReadHeaderRecord();
PointPairList obDataPoints = new PointPairList();
uint uiRowIndex = 0U;
while (csvReader.HasMoreRecords)
{
DataRecord record = csvReader.ReadDataRecord();
var data = record[1];
obDataPoints.Add(Convert.ToDouble(uiRowIndex), Convert.ToDouble(data));
}
FillGraphData(obDataPoints);
}
}
catch (Exception ex)
{
MessageBox.Show("Error: Could not read file from disk. Original error: " + ex.Message);
}
}
}
private void ParserTableContent(string content)
{
using (var rdr = new StringReader(content))
using (var reader = new CsvReader(rdr))
{
HeaderRecord header = reader.ReadHeaderRecord();
const string tableName = "SkillStepTable";
int lastId = -1;
while (reader.HasMoreRecords)
{
DataRecord data = reader.ReadDataRecord();
var r = new SkillStepTableRecord();
if (data[0].StartsWith("#"))
{
continue;
}
Records.Add(data[0], r);
}
}
}
public void Parse_WhenCalled_ParsesFileds()
{
// arrange
var input = @"H|\^&|MessageControlID|AccessPassword|SenderID|SenderStreetAddress|Reserved0|SenderTelephoneNumber|SenderCharacteristiucs|ReceiverID|Comment|P|VersionNUmber|19700101010203";
//act
var actual = HeaderRecord.Parse(input);
// assert
Assert.Equal("MessageControlID", actual.MessageControlID);
Assert.Equal("AccessPassword", actual.AccessPassword);
Assert.Equal("SenderID", actual.SenderID);
Assert.Equal("SenderStreetAddress", actual.SenderStreetAddress);
Assert.Equal("Reserved0", actual.Reserved0);
Assert.Equal("SenderTelephoneNumber", actual.SenderTelephoneNumber);
Assert.Equal("SenderCharacteristiucs", actual.SenderCharacteristics);
Assert.Equal("ReceiverID", actual.ReceiverID);
Assert.Equal("Comment", actual.Comment);
Assert.Equal(ProcessingType.Production, actual.ProcessingID);
Assert.Equal("VersionNUmber", actual.VersionNumber);
Assert.NotNull(actual.MessageDateTime);
Assert.Equal(new DateTime(1970, 01, 01, 01, 02, 03), actual.MessageDateTime.Value.DateTime);
}
public void returns_expected_result()
{
var recordParser = BuildParser();
var recordToParse = "HDTPS.UDFROC1.PD1512303012152116DFROC1EDFROC1DUA301215291216 ";
var expectedResult = new HeaderRecord
{
RecordIdentity = ScheduleRecordType.HD,
MainFrameIdentity = "TPS.UDFROC1.PD151230",
DateOfExtract = new DateTime(2015, 12, 30),
TimeOfExtract = "2116",
CurrentFileRef = "DFROC1E",
LastFileRef = "DFROC1D",
ExtractUpdateType = ExtractUpdateType.U,
CifSoftwareVersion = "A",
UserExtractStartDate = new DateTime(2015, 12, 30),
UserExtractEndDate = new DateTime(2016, 12, 29),
MainFrameUser = "******",
MainFrameExtractDate = new DateTime(2015, 12, 30)
};
var result = recordParser.ParseRecord(recordToParse);
Assert.AreEqual(expectedResult, result);
}
// DataContext creates plugins
internal Plugin(DataContext context, string fileName, PluginMode mode)
{
this.context = context;
this.fileName = fileName;
if (mode == PluginMode.Create)
{
header = context.CreateHeader();
header.NextFormId = 0x800;
}
else if (mode == PluginMode.Open)
{
// Read only the header mainly to fetch the list of masters
Stream stream = context.DataFileProvider.GetDataFile(FileMode.Open, fileName).Open();
using (var reader = context.CreateReader(stream))
{
header = reader.ReadHeader();
}
}
else
{
throw new ArgumentException("Illegal plugin mode specified");
}
}
public Boolean Read(BinaryReader source, SizeInfo sizeInfo, MetaDataIO.ReadTagParams readTagParams)
{
var Header = new HeaderRecord();
Boolean result;
Byte version;
this.sizeInfo = sizeInfo;
resetData();
// Load header from file to variable
result = readHeader(source, ref Header);
version = getStreamVersion(Header);
// Process data if loaded and file valid
if (result && (sizeInfo.FileSize > 0) && (version > 0))
{
FStreamVersion = version;
if (version < 80)
{
// Fill properties with SV7 header data
sampleRate = getSampleRate(Header);
channelModeID = getChannelModeID(Header);
frameCount = getFrameCount(Header);
bitrate = getBitRate(Header);
profileID = getProfileID(Header);
encoder = getEncoder(Header);
duration = getDuration();
}
else
{
// VS8 data already read
}
}
return(result);
}
public void returns_expected_result()
{
var recordParser = BuildParser();
var recordToParse = "HDTPS.UDFROC1.PD1512303012152116DFROC1EDFROC1DUA301215291216 ";
var expectedResult = new HeaderRecord
{
RecordIdentity = ScheduleRecordType.HD,
MainFrameIdentity = "TPS.UDFROC1.PD151230",
DateOfExtract = new DateTime(2015, 12, 30),
TimeOfExtract = "2116",
CurrentFileRef = "DFROC1E",
LastFileRef = "DFROC1D",
ExtractUpdateType = ExtractUpdateType.U,
CifSoftwareVersion = "A",
UserExtractStartDate = new DateTime(2015, 12, 30),
UserExtractEndDate = new DateTime(2016, 12, 29),
MainFrameUser = "******",
MainFrameExtractDate = new DateTime(2015, 12, 30)
};
var result = recordParser.ParseRecord(recordToParse);
Assert.AreEqual(expectedResult, result);
}
// ---------------------------------------------------------------------------
private int GetSampleRate(HeaderRecord Header)
{
int[] mpp_samplerates = new int[4] { 44100, 48000, 37800, 32000 };
/* get samplerate from header
note: this is the same byte where profile is stored
*/
return mpp_samplerates[Header.ByteArray[10] & 3];
}
private static void InitGameObject(ColumnMappingAttribute attribute, HeaderRecord record)
{
LoadObjectFromResource(attribute, record);
}
private static void InitInt32(ColumnMappingAttribute attribute, HeaderRecord record)
{
ConverToInternal <int>(attribute, record);
}
// ---------------------------------------------------------------------------
private ushort GetBitRate(HeaderRecord Header)
{
// Try to get bit rate
if ( (6 >= GetStreamVersion(Header)) /*|| (5 == GetStreamVersion(Header))*/ )
{
return (ushort)((Header.IntegerArray[0] >> 23)& 0x01FF);
}
else
{
return 0;
}
}
public void Load()
{
Log.Info("Indexing forms in plugin {0}", fileName);
Stream stream = context.DataFileProvider.GetDataFile(FileMode.Open, fileName).Open();
using (var reader = context.CreateReader(stream))
{
// Header needs to be read again
header = reader.ReadHeader();
// Prepare reference mapper
reader.ReferenceMapper = new PluginReferenceMapper(this);
int before = context.Forms.Count();
int overriding = 0;
int added = 0;
using (var progress = Display.StartProgress("Indexing forms"))
{
if (context.AsyncFormIndexing)
{
// Retrieve plugin number so that loaded forms can be linked to it
byte pluginNumber = context.Plugins.GetPluginNumber(this);
foreach (var record in reader.FindRecordsAsync())
{
Form form = new Form()
{
PluginNumber = pluginNumber,
FilePosition = record.FilePosition,
FormKind = (FormKind)record.Signature,
FormId = reader.ReferenceMapper.LocalToContext(record.FormId),
ParentFormId = reader.ReferenceMapper.LocalToContext(record.ParentRecordFormId)
};
// Index loaded form
context.Forms.Add(form);
added++;
if (form.IsOverriding)
{
overriding++;
}
progress.Update(reader.TotalRecordsFound, header.NumRecords, "{0} ({1})", fileName, form.FormKind);
}
}
else
{
FindRecordsListener listener = new FindRecordsListener(this, reader, progress);
reader.FindRecords(listener);
added = listener.Added;
overriding = listener.Overriding;
}
}
Log.Info("Found {0} forms ({1} overrides)", added, overriding);
//if (header.NumRecords != reader.TotalRecordsFound)
//{
// Log.Warning("Number of records specified in header {0} does not match number of records found {1}", header.NumRecords, reader.TotalRecordsFound);
//}
}
}
public FrameLabelTag(HeaderRecord header, FlashReader input)
: base(header)
{
Name = input.ReadNullString();
}
public bool notEquals(HeaderRecord value)
{
return !this.isEqual(value);
}
public void header_record_can_be_set()
{
using (var reader = CsvReader.FromCsvString(string.Empty))
{
var headerRecord = new HeaderRecord();
reader.HeaderRecord = headerRecord;
Assert.Same(headerRecord, reader.HeaderRecord);
}
}
// ---------------------------------------------------------------------------
private byte GetProfileID(HeaderRecord Header)
{
byte result = MPP_PROFILE_UNKNOWN;
// Get MPEGplus profile (exists for stream version 7 only)
if ( (7 == GetStreamVersion(Header)) || (71 == GetStreamVersion(Header)) )
// ((and $F0) shr 4) is needed because samplerate is stored in the same byte!
switch( ((Header.ByteArray[10] & 0xF0) >> 4) )
{
case 1: result = MPP_PROFILE_EXPERIMENTAL; break;
case 5: result = MPP_PROFILE_QUALITY0; break;
case 6: result = MPP_PROFILE_QUALITY1; break;
case 7: result = MPP_PROFILE_TELEPHONE; break;
case 8: result = MPP_PROFILE_THUMB; break;
case 9: result = MPP_PROFILE_RADIO; break;
case 10: result = MPP_PROFILE_STANDARD; break;
case 11: result = MPP_PROFILE_XTREME; break;
case 12: result = MPP_PROFILE_INSANE; break;
case 13: result = MPP_PROFILE_BRAINDEAD; break;
case 14: result = MPP_PROFILE_QUALITY9; break;
case 15: result = MPP_PROFILE_QUALITY10; break;
}
return result;
}
// ---------------------------------------------------------------------------
// No explicit destructors with C#
// ---------------------------------------------------------------------------
public bool ReadFromFile(String FileName)
{
HeaderRecord Header = new HeaderRecord();
bool result;
// Reset data and load header from file to variable
FResetData();
Array.Clear(Header.ByteArray,0,Header.ByteArray.Length);
Array.Clear(Header.IntegerArray,0,Header.IntegerArray.Length);
Header.FileSize = 0;
Header.ID3v2Size = 0;
// At first try to load ID3v2 tag data, then header
if ( FID3v2.ReadFromFile(FileName) ) Header.ID3v2Size = FID3v2.Size;
result = ReadHeader(FileName, ref Header);
// Process data if loaded and file valid
if ((result) && (Header.FileSize > 0) && (GetStreamVersion(Header) > 0) )
{
FValid = true;
// Fill properties with header data
FSampleRate = GetSampleRate(Header);
FChannelModeID = GetChannelModeID(Header);
FFileSize = Header.FileSize;
FFrameCount = GetFrameCount(Header);
FBitRate = GetBitRate(Header);
FStreamVersion = GetStreamVersion(Header);
FProfileID = GetProfileID(Header);
FEncoder = GetEncoder(Header);
FID3v1.ReadFromFile(FileName);
FAPEtag.ReadFromFile(FileName);
}
return result;
}
public async Task parse_records_async_uses_the_specified_header_record()
{
var headerRecord = new HeaderRecord(true, "Name", "Age");
var csv = @"Kent,33
Belinda,34
Tempany,8
Xak,0";
var parser = this.CreateParserFromString(csv);
var records = new DataRecord[4];
Assert.Equal(4, await parser.ParseRecordsAsync(headerRecord, records, 0, records.Length));
Assert.Same(headerRecord, records[0].HeaderRecord);
Assert.Same(headerRecord, records[1].HeaderRecord);
Assert.Same(headerRecord, records[2].HeaderRecord);
Assert.Same(headerRecord, records[3].HeaderRecord);
}
public HeaderRecord(HeaderRecord value)
{
/// Initiliaze the protected variables
m_parent = null;
m_MessageIDHeader = 0xf011;
/// Copy the values
m_MessageIDHeader = value.m_MessageIDHeader;
}
public async Task <int> ParseRecordsAsync(HeaderRecord headerRecord, DataRecord[] buffer, int offset, int count)
{
// see performance notes in SkipRecord
var ch = char.MinValue;
var recordsParsed = 0;
var delimited = false;
for (var i = offset; i < offset + count; ++i)
{
while (true)
{
if (!this.IsBufferEmpty)
{
ch = this.buffer[this.bufferIndex++];
if (!this.IsPossiblySpecialCharacter(ch))
{
// if it's definitely not a special character, then we can just append it and continue on with the loop
this.valueBuilder.NotifyPreviousCharIncluded(delimited);
continue;
}
if (!delimited)
{
if (ch == this.valueSeparator)
{
this.values.Add(this.valueBuilder.GetValueAndClear());
}
else if (ch == this.valueDelimiter)
{
this.valueBuilder.NotifyPreviousCharExcluded();
delimited = true;
// since we're in a delimited area, the only special character is the value delimiter
this.activeSpecialCharacterMask = this.valueDelimiter.Value;
}
else if (ch == Constants.CR)
{
// we need to look at the next character, so make sure it is available
if (this.IsBufferEmpty && !(await this.FillBufferAsync().ConfigureAwait(false)))
{
// undelimited CR indicates the end of a record, so add the existing value and then exit
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
// we deal with CRLF right here by checking if the next character is LF, in which case we just discard it
if (this.buffer[this.bufferIndex] == Constants.LF)
{
++this.bufferIndex;
}
// undelimited CR or CRLF both indicate the end of a record, so add the existing value and then exit
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
else if (ch == Constants.LF)
{
// undelimited LF indicates the end of a record, so add the existing value and then exit
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
else
{
// it wasn't a special character after all, so just append it
this.valueBuilder.NotifyPreviousCharIncluded(false);
}
}
else if (ch == this.valueDelimiter)
{
// we need to look at the next character, so make sure it is available
if (this.IsBufferEmpty && !(await this.FillBufferAsync().ConfigureAwait(false)))
{
// out of data
delimited = false;
this.activeSpecialCharacterMask = this.specialCharacterMask;
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
if (this.buffer[this.bufferIndex] == this.valueDelimiter)
{
// delimiter is escaped, so append it to the value and discard the escape character
this.valueBuilder.NotifyPreviousCharExcluded();
++this.bufferIndex;
this.valueBuilder.NotifyPreviousCharIncluded(true);
}
else
{
// delimiter isn't escaped, so we are no longer in a delimited area
this.valueBuilder.NotifyPreviousCharExcluded();
this.activeSpecialCharacterMask = this.specialCharacterMask;
delimited = false;
}
}
else
{
// it wasn't a special character after all, so just append it
this.valueBuilder.NotifyPreviousCharIncluded(true);
}
}
else if (!(await this.FillBufferAsync().ConfigureAwait(false)))
{
if (this.valueBuilder.HasValue)
{
// a value is outstanding, so add it
this.values.Add(this.valueBuilder.GetValueAndClear());
}
if (ch == this.valueSeparator)
{
// special case: last character is a separator, which means there should be an empty value after it. eg. "foo," results in ["foo", ""]
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, string.Empty);
break;
}
else
{
var record = this.values.GetDataRecordAndClear(headerRecord);
if (record != null)
{
buffer[i] = record;
++recordsParsed;
}
}
// data exhausted - we're done, even though we may not have filled the records array
return(recordsParsed);
}
}
++recordsParsed;
}
return(recordsParsed);
}
public bool isEqual(HeaderRecord value)
{
if (this.getMessageIDHeader() != value.getMessageIDHeader())
{
return false;
}
return true;
}
private void ParseMPQHeader(BinaryReader br)
{
Header = new HeaderRecord
{
HeaderSize = br.ReadUInt32(),
ArchiveSize = br.ReadUInt32(),
FormatVersion = br.ReadUInt16(),
BlockSize = (byte)br.ReadInt16(),
HashTablePos = br.ReadUInt32(),
BlockTablePos = br.ReadUInt32(),
HashTableSize = br.ReadUInt32(),
BlockTableSize = br.ReadUInt32()
};
if (FormatVersion != eMPQFormatVersion.Format1)
{
throw new OpenDiablo2Exception($"Unsupported MPQ format version of {Header.FormatVersion} detected for '{Path}'!");
}
if (br.BaseStream.Position != Header.HeaderSize)
{
throw new OpenDiablo2Exception($"Invalid header size detected for '{Path}'. Expected to be at offset {Header.HeaderSize} but we are at offset {br.BaseStream.Position} instead!");
}
br.BaseStream.Seek(Header.BlockTablePos, SeekOrigin.Begin);
// Process the block table
var bData = br.ReadBytes((int)(16 * Header.BlockTableSize));
DecryptBlock(bData, HashString("(block table)", MPQ_HASH_FILE_KEY));
using (var ms = new MemoryStream(bData))
using (var dr = new BinaryReader(new MemoryStream(bData)))
for (var index = 0; index < Header.BlockTableSize; index++)
{
blockTable.Add(new BlockRecord
{
BlockOffset = dr.ReadUInt32(),
BlockSize = dr.ReadUInt32(),
FileSize = dr.ReadUInt32(),
Flags = dr.ReadUInt32()
});
}
// Process the hash table
br.BaseStream.Seek(Header.HashTablePos, SeekOrigin.Begin);
bData = br.ReadBytes((int)(16 * Header.HashTableSize));
DecryptBlock(bData, HashString("(hash table)", MPQ_HASH_FILE_KEY));
using (var ms = new MemoryStream(bData))
using (var dr = new BinaryReader(new MemoryStream(bData)))
for (var index = 0; index < Header.HashTableSize; index++)
{
hashTable.Add(new HashRecord
{
FilePathHashA = dr.ReadUInt32(),
FilePathHashB = dr.ReadUInt32(),
Language = dr.ReadUInt16(),
Platform = dr.ReadUInt16(),
FileBlockIndex = dr.ReadUInt32()
});
}
}
public ReportAddition.Header.HeaderRecord setHeaderRecord(HeaderRecord value)
{
m_MessageIDHeader = value.m_MessageIDHeader;
return this;
}
// convert this value list to a data record, placing the extra value at the end of the record's values, then clear ready to construct the next list of values
// the extra parameter saves the client code having to add the value and then call GetDataRecordAndClear, which is more expensive than just doing it in one step
public DataRecord GetDataRecordAndClear(HeaderRecord headerRecord, string extra)
{
var result = new string[this.valueEndIndex + 1];
Array.Copy(this.values, 0, result, 0, this.valueEndIndex);
result[this.valueEndIndex] = extra;
// clear
this.valueEndIndex = 0;
// cast is to select the internal constructor, which is faster
return new DataRecord(headerRecord, (IList<string>)result);
}
public void LoadForms(bool lazyLoading, Func <Form, bool> predicate)
{
if (context.AsyncFromLoadingMaxWorkers < 1)
{
throw new InvalidOperationException("AsyncFromLoadingMaxWorkers cannot be less than 1");
}
Stream stream = context.DataFileProvider.GetDataFile(FileMode.Open, fileName).Open();
// Uncomment this to preload a whole file into memory
// The loading time is marginaly shorter (around 4%)
// but only when another background worker is employed
//Log.Fine("Preloading plugin file {0}.", fileName);
//var data = new byte[stream.Length];
//stream.Read(data, 0, (int)stream.Length);
//stream = new ReadOnlyMemoryStream(data);
using (var reader = context.CreateReader(stream))
{
// Header needs to be read again
header = reader.ReadHeader();
// Prepare reference mapper
reader.ReferenceMapper = new PluginReferenceMapper(this);
// Prepare string locator (if plugin strings are localized)
if (reader.PluginFlags.HasFlag(PluginFlags.Localized))
{
reader.StringLocator = new PluginStringLocator(this);
}
int formsToLoadCount = 0;
// Apply predicate to the enumeration, if any
// this will cause the total number of forms to load be unknown
var formsToLoadEnumeration = context.Forms.OfPlugin(this);
if (predicate != null)
{
// TODO: Fix the list to make the total count known
formsToLoadEnumeration = formsToLoadEnumeration.Where(predicate);
formsToLoadCount = formsToLoadEnumeration.Count();
}
else
{
// Get count if enumeration is a list or a collection
formsToLoadCount = formsToLoadEnumeration.Count();
}
if (formsToLoadEnumeration.Any())
{
var formsToLoad = formsToLoadEnumeration;
int formsToLoadCountAtLeast = 0;
// Determine number of background jobs, if any
int jobs = 0;
if (context.AsyncFormLoading)
{
// Materialize part of the list
// to determine the count of background jobs at least
if (formsToLoadCount == 0)
{
int take = context.AsyncFormLoadingWorkerThreshold * context.AsyncFromLoadingMaxWorkers;
formsToLoadCountAtLeast = formsToLoad.Take(take).Count();
if (formsToLoadCountAtLeast < take)
{
formsToLoadCount = formsToLoadCountAtLeast;
}
}
// Determine number of jobs
if (formsToLoadCount > 0)
{
// Use known number of forms to load
jobs = Math.Min(formsToLoadCount / context.AsyncFormLoadingWorkerThreshold + 1, context.AsyncFromLoadingMaxWorkers);
}
else
{
// Use minimal determined number of forms to load
jobs = Math.Min(formsToLoadCountAtLeast / context.AsyncFormLoadingWorkerThreshold + 1, context.AsyncFromLoadingMaxWorkers);
}
}
// Indicate that total number of forms to load will be determined during iteration
bool formsToLoadIsUnknown = formsToLoadCount == 0;
if (formsToLoadIsUnknown)
{
Log.Fine("Total number of forms to load is not unknown");
if (context.AsyncFormLoading)
{
Log.Fine("Using {0} background jobs to load more than {1} forms", jobs, formsToLoadCountAtLeast);
}
}
else if (context.AsyncFormLoading)
{
Log.Fine("Using {0} background jobs to load {1} forms", jobs, formsToLoadCount);
}
using (var loader = new FormLoader(this, reader, lazyLoading, jobs))
{
using (var progress = Display.StartProgress("Loading forms"))
{
foreach (Form form in formsToLoad)
{
// Do not load froms which have been injected during indexing
if (!form.IsInjected && !form.IsHardcoded)
{
loader.LoadForm(form);
}
// Unknown total number of forms to load
// has to be determined during iteration
if (formsToLoadIsUnknown)
{
formsToLoadCount++;
}
var lastFormLoaded = loader.LastFormLoaded;
progress.Update(loader.Loaded, Math.Max(formsToLoadCount, formsToLoadCountAtLeast),
"{0} {1}", fileName, lastFormLoaded == null ? string.Empty : lastFormLoaded.ToString());
}
if (context.AsyncFormLoading)
{
// Tell loader no more forms will be loaded
loader.Complete();
// Show progress while loader is still busy
while (loader.IsBusy)
{
System.Threading.Thread.Sleep(30);
var lastFormLoaded = loader.LastFormLoaded;
progress.Update(loader.Loaded, Math.Max(formsToLoadCount, formsToLoadCountAtLeast),
"{0} {1}", fileName, lastFormLoaded == null ? string.Empty : lastFormLoaded.ToString());
}
// Wait for loader to finish completely
Log.Fine("Waiting for background jobs to finish");
loader.WaitForCompleted();
}
}
Log.Info("Loaded {0} forms from {1} ({2} supported, {3} unsupported)", loader.Loaded, fileName, loader.Supported, loader.Unsupported);
if (loader.Skipped > 0)
{
Log.Info("Skipped {0} forms that had been already loaded", loader.Skipped);
}
}
}
}
}
// ---------------------------------------------------------------------------
private byte GetChannelModeID(HeaderRecord Header)
{
byte result;
if ((7 == GetStreamVersion(Header)) || (71 == GetStreamVersion(Header)))
// Get channel mode for stream version 7
if ((Header.ByteArray[11] % 128) < 64) result = MPP_CM_STEREO;
else result = MPP_CM_JOINT_STEREO;
else
// Get channel mode for stream version 4-6
if (0 == (Header.ByteArray[2] % 128)) result = MPP_CM_STEREO;
else result = MPP_CM_JOINT_STEREO;
return result;
}
public ImageTag(HeaderRecord header)
: base(header)
{
}
public Header(Header value)
{
/// Initiliaze the protected variables
m_HeaderRecord = new HeaderRecord();
m_HeaderRecord.setParent(this);
/// Copy the values
m_HeaderRecord = value.getHeaderRecord();
m_HeaderRecord.setParent(this);
}
// ---------------------------------------------------------------------------
private int GetFrameCount(HeaderRecord Header)
{
int result;
// Get frame count
if (4 == GetStreamVersion(Header) ) result = Header.IntegerArray[1] >> 16;
else
if ((5 <= GetStreamVersion(Header)) && (GetStreamVersion(Header) <= 71) )
result = Header.IntegerArray[1];
else result = 0;
return result;
}
public async Task read_data_records_async_assigns_any_header_record()
{
var csv = @"first
second
third
fourth
fifth
sixth
seventh
eighth
ninth
tenth";
using (var reader = CsvReader.FromCsvString(csv))
{
var headerRecord = new HeaderRecord("Number");
reader.HeaderRecord = headerRecord;
var buffer = new DataRecord[10];
Assert.Equal(10, await reader.ReadDataRecordsAsync(buffer, 0, buffer.Length));
for (var i = 0; i < 10; ++i)
{
Assert.Same(headerRecord, buffer[i].HeaderRecord);
}
}
}
private static void InitSingle(ColumnMappingAttribute attribute, HeaderRecord record)
{
ConverToInternal <float>(attribute, record);
}
public void setHeaderRecord(HeaderRecord value)
{
m_HeaderRecord = value;
setParentPresenceVector();
}
public ReportAddition.Header setHeader(Header value)
{
m_HeaderRecord = value.getHeaderRecord();
m_HeaderRecord.setParent(this);
return this;
}
public async Task read_data_record_async_assigns_any_header_record()
{
var csv = "Kent,33";
using (var reader = CsvReader.FromCsvString(csv))
{
var headerRecord = new HeaderRecord("Name", "Age");
reader.HeaderRecord = headerRecord;
var record = await reader.ReadDataRecordAsync();
Assert.NotNull(record);
Assert.Same(headerRecord, record.HeaderRecord);
}
}
public int ParseRecords(HeaderRecord headerRecord, DataRecord[] buffer, int offset, int count)
{
// see performance notes in SkipRecord
var ch = char.MinValue;
var recordsParsed = 0;
var delimited = false;
for (var i = offset; i < offset + count; ++i)
{
while (true)
{
if (!this.IsBufferEmpty)
{
ch = this.buffer[this.bufferIndex++];
if (!this.IsPossiblySpecialCharacter(ch))
{
// if it's definitely not a special character, then we can just append it and continue on with the loop
this.valueBuilder.NotifyPreviousCharIncluded(delimited);
continue;
}
if (!delimited)
{
if (ch == this.valueSeparator)
{
this.values.Add(this.valueBuilder.GetValueAndClear());
}
else if (ch == this.valueDelimiter)
{
this.valueBuilder.NotifyPreviousCharExcluded();
delimited = true;
// since we're in a delimited area, the only special character is the value delimiter
this.activeSpecialCharacterMask = this.valueDelimiter.Value;
}
else if (ch == Constants.CR)
{
// we need to look at the next character, so make sure it is available
if (this.IsBufferEmpty && !this.FillBuffer())
{
// undelimited CR indicates the end of a record, so add the existing value and then exit
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
// we deal with CRLF right here by checking if the next character is LF, in which case we just discard it
if (this.buffer[this.bufferIndex] == Constants.LF)
{
++this.bufferIndex;
}
// undelimited CR or CRLF both indicate the end of a record, so add the existing value and then exit
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
else if (ch == Constants.LF)
{
// undelimited LF indicates the end of a record, so add the existing value and then exit
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
else
{
// it wasn't a special character after all, so just append it
this.valueBuilder.NotifyPreviousCharIncluded(false);
}
}
else if (ch == this.valueDelimiter)
{
// we need to look at the next character, so make sure it is available
if (this.IsBufferEmpty && !this.FillBuffer())
{
// out of data
delimited = false;
this.activeSpecialCharacterMask = this.specialCharacterMask;
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, this.valueBuilder.GetValueAndClear());
break;
}
if (this.buffer[this.bufferIndex] == this.valueDelimiter)
{
// delimiter is escaped, so append it to the value and discard the escape character
this.valueBuilder.NotifyPreviousCharExcluded();
++this.bufferIndex;
this.valueBuilder.NotifyPreviousCharIncluded(true);
}
else
{
// delimiter isn't escaped, so we are no longer in a delimited area
this.valueBuilder.NotifyPreviousCharExcluded();
delimited = false;
this.activeSpecialCharacterMask = this.specialCharacterMask;
}
}
else
{
// it wasn't a special character after all, so just append it
this.valueBuilder.NotifyPreviousCharIncluded(true);
}
}
else if (!this.FillBuffer())
{
if (this.valueBuilder.HasValue)
{
// a value is outstanding, so add it
this.values.Add(this.valueBuilder.GetValueAndClear());
}
if (ch == this.valueSeparator)
{
// special case: last character is a separator, which means there should be an empty value after it. eg. "foo," results in ["foo", ""]
buffer[i] = this.values.GetDataRecordAndClear(headerRecord, string.Empty);
++recordsParsed;
}
else
{
var record = this.values.GetDataRecordAndClear(headerRecord);
if (record != null)
{
buffer[i] = record;
++recordsParsed;
}
}
// data exhausted - we're done, even though we may not have filled the records array
return recordsParsed;
}
}
++recordsParsed;
}
return recordsParsed;
}
public static void ReadContentFile(ConstructFile construct, ref ContentFile contentFile)
{
string path = TableGlobalConfig.Instance.ResTablePath + "\\" + construct.Path + construct.Name + ".csv";
if (!string.IsNullOrEmpty(construct.OldName))
{
path = TableGlobalConfig.Instance.ResTablePath + "\\" + construct.Path + construct.OldName + ".csv";
}
if (!File.Exists(path))
{
//CreateNewFile(construct);
TableContentManager.Instance.AddContentFile(contentFile);
return;
}
string[] lines = File.ReadAllLines(path, Encoding.Default);
if (lines.Length == 0)
{
return;
}
contentFile.ContentRow.Clear();
lines[0] = lines[0].Replace("\r\n", "\n");
StringReader rdr = new StringReader(string.Join("\n", lines));
using (var reader = new CsvReader(rdr))
{
HeaderRecord header = reader.ReadHeaderRecord();
//string curGourpName = "";
//string curClassName = "";
var columnTitles = construct.GetColumnTitles();
while (reader.HasMoreRecords)
{
DataRecord data = reader.ReadDataRecord();
//if (data[0].StartsWith("###"))
//{
// curGourpName = data[0].TrimStart('#');
// continue;
//}
//if (data[0].StartsWith("##"))
//{
// curClassName = data[0].TrimStart('#');
// continue;
//}
ContentRow clumn = ReadRow(construct, data, contentFile);
contentFile.AddContentRow(clumn);
}
if (IsHeaderMatch(construct, header))
{
contentFile.WriteFlag = false;
}
else
{
contentFile.WriteFlag = true;
}
contentFile.IsInit = false;
TableContentManager.Instance.AddContentFile(contentFile);
}
}
public Header()
{
m_HeaderRecord = new HeaderRecord();
m_HeaderRecord.setParent(this);
}
// ---------------------------------------------------------------------------
private String GetEncoder(HeaderRecord Header)
{
int EncoderID;
String result = "";
EncoderID = Header.ByteArray[10+2+15];
if (0 == EncoderID)
{
//FEncoder := 'Buschmann 1.7.0...9, Klemm 0.90...1.05';
}
else
{
switch ( EncoderID % 10 )
{
case 0: result = "Release "+(EncoderID / 100)+"."+ ( (EncoderID / 10) % 10 ); break;
case 2 : result = "Beta "+(EncoderID / 100)+"."+ (EncoderID % 100); break; // Not exactly...
case 4: goto case 2;
case 6: goto case 2;
case 8: goto case 2;
default: result = "--Alpha-- "+(EncoderID / 100)+"."+(EncoderID % 100); break;
}
}
return result;
}
