private void ProcessVariables(IDictionary <string, string> variableLongNames, IDictionary <string, int> veryLongStrings, List <VariableRecord> variableRecords, List <ValueLabel> valueLabels)
{
var namesList = new SortedSet <byte[]>(new ByteArrayComparer());
foreach (var variable in _variables)
{
int dictionaryIndex = variableRecords.Count + 1;
var records = VariableRecord.GetNeededVaraibles(variable, _options.HeaderEncoding, namesList, ref _longNameCounter, veryLongStrings);
variableRecords.AddRange(records);
// Check if a longNameVariableRecord is needed
if (records[0].Name != variable.Name)
{
variableLongNames.Add(records[0].Name, variable.Name);
}
// TODO Avoid repeating the same valueLabels on the file
// Add ValueLabels if necesary
if (variable.ValueLabels != null && variable.ValueLabels.Any())
{
var valueLabel = new ValueLabel(variable.ValueLabels.ToDictionary(p => BitConverter.GetBytes(p.Key), p => p.Value));
valueLabel.VariableIndex.Add(dictionaryIndex);
valueLabels.Add(valueLabel);
}
}
}
private void ProcessVariables(IDictionary <string, string> variableLongNames, IDictionary <string, int> veryLongStrings, List <VariableRecord> variableRecords, List <ValueLabel> valueLabels)
{
int longNameCounter = 0;
var namesList = new SortedSet <byte[]>(new ByteArrayComparer());
var segmentsNamesList = new SortedList <byte[], int>(new ByteArrayComparer());
foreach (var variable in _variables)
{
int dictionaryIndex = variableRecords.Count + 1;
var records = VariableRecord.GetNeededVariables(variable, _options.HeaderEncoding, namesList, ref longNameCounter, veryLongStrings, segmentsNamesList);
variableRecords.AddRange(records);
// Even if the variable name is the same, it still needs a long record indicator otherwise SPSS doesn't know how to handle it.
variableLongNames.Add(records[0].Name, variable.Name);
// TODO Avoid repeating the same valueLabels on the file
// Add ValueLabels if necesary
if (variable.ValueLabels != null && variable.ValueLabels.Any())
{
var valueLabel = new ValueLabel(variable.ValueLabels.ToDictionary(p => BitConverter.GetBytes(p.Key), p => p.Value));
valueLabel.VariableIndex.Add(dictionaryIndex);
valueLabels.Add(valueLabel);
}
}
}
public VariableHandler(ConfigNode node, RasterPropMonitorComputer ourComp)
{
handlerName = node.GetValue("name");
foreach (string variableRecord in node.GetValues("variable")) {
var record = new VariableRecord();
string[] tokens = variableRecord.Split(',');
if (tokens.Length >= 2) {
double defaultDouble;
if (double.TryParse(tokens[1], out defaultDouble)) {
record.defaultValue = defaultDouble;
} else {
if (tokens[1].Trim() == "fallback") {
record.fallback = true;
} else {
record.defaultString = tokens[1];
}
}
}
if (tokens.Length >= 3) {
record.cacheable = bool.Parse(tokens[2]);
}
handledVariables.Add(tokens[0], record);
}
active = InstantiateHandler(node, ourComp, out handlerFunction);
}
public VariableHandler(ConfigNode node, RasterPropMonitorComputer ourComp)
{
handlerName = node.GetValue("name");
foreach (string variableRecord in node.GetValues("variable"))
{
var record = new VariableRecord();
string[] tokens = variableRecord.Split(',');
if (tokens.Length >= 2)
{
double defaultDouble;
if (double.TryParse(tokens[1], out defaultDouble))
{
record.defaultValue = defaultDouble;
}
else
{
if (tokens[1].Trim() == "fallback")
{
record.fallback = true;
}
else
{
record.defaultString = tokens[1];
}
}
}
if (tokens.Length >= 3)
{
record.cacheable = bool.Parse(tokens[2]);
}
handledVariables.Add(tokens[0], record);
}
active = InstantiateHandler(node, ourComp, out handlerFunction);
}
public void WriteString(string s, int width)
{
_recordWriter.StartString();
// Get the char array (excluding trailing spaces)
var chars = (s ?? string.Empty).TrimEnd(' ').ToCharArray();
// Actual byte length of this variable
var length = VariableRecord.GetLongStringBytesCount(width);
var charIndex = 0;
int writtenBytes = 0;
var writtenUpToIndex = 0;
while (chars.Length > charIndex && writtenBytes < length)
{
var charsToRead = Math.Min(chars.Length - charIndex, CharsBufferSize);
int charsRead, bytesRead;
bool completed;
_encoder.Convert(chars, charIndex, charsToRead, _stringBytesBuffer, 0, _stringBytesBuffer.Length,
false, out charsRead, out bytesRead, out completed);
// Move index forward for next read
charIndex += charsRead;
var bytesLeftToWrite = bytesRead;
writtenBytes = WriteBytes(bytesLeftToWrite, writtenBytes, length, ref writtenUpToIndex);
}
// Clean encoder internal buffer
_encoder.Reset();
_recordWriter.EndStringVariable(writtenBytes, length);
}
private void ParseVariableRecords()
{
// Dataset.Variables collection will contain fewer Variables, because all string extension VariableRecords will be removed
int variableIndex = 0;
for (int recordIndex = 0; recordIndex < this.SavFile.VariableRecords.Count; recordIndex++)
{
VariableRecord vr = this.SavFile.VariableRecords[recordIndex];
if (vr.Type != VariableType.StringExtention) // for all numeric variables and string variables
{
Variable v = new Variable(this);
v.VariableIndex = variableIndex + 1;
v.ReadFromVariableRecord(vr, this.SavFile.VariableRecords.IndexOf(vr));
this.Variables.Add(v);
this.RecordIndexByVariableIndex.Add(variableIndex, recordIndex);
this.VariableIndexByShortName.Add(vr.Name.Trim(), variableIndex);
variableIndex += 1;
}
else // for string extensions variable records increments the VariableRecordLength field for the last added variable
{
this.Variables[variableIndex - 1].VariableRecordLength += 1;
}
this.VariableIndexByRecordIndex.Add(recordIndex, variableIndex - 1);
}
}
public void ParseMetaData()
{
var meta = new MetaData();
reader = new BinaryReader(this.Stream, Encoding.ASCII);
var variableRecords = new List <VariableRecord>();
var valueLabelRecords = new List <ValueLabelRecord>();
var infoRecords = new InfoRecords();
RecordType nextRecordType = (RecordType)reader.ReadInt32();
while (nextRecordType != RecordType.End)
{
switch (nextRecordType)
{
case RecordType.HeaderRecord:
meta.HeaderRecord = HeaderRecord.ParseNextRecord(reader);
break;
case RecordType.VariableRecord:
variableRecords.Add(VariableRecord.ParseNextRecord(reader));
break;
case RecordType.ValueLabelRecord:
valueLabelRecords.Add(ValueLabelRecord.ParseNextRecord(reader));
break;
case RecordType.DocumentRecord:
meta.DocumentRecord = DocumentRecord.ParseNextRecord(reader);
break;
case RecordType.InfoRecord:
infoRecords.AllRecords.Add(InfoRecord.ParseNextRecord(reader));
break;
default:
throw new UnexpectedFileFormatException();
}
nextRecordType = (RecordType)reader.ReadInt32();
}
meta.VariableRecords = new Collection <VariableRecord>(variableRecords);
meta.ValueLabelRecords = new Collection <ValueLabelRecord>(valueLabelRecords);
// Interpret known inforecords:
infoRecords.ReadKnownRecords(meta.VariableCount);
meta.InfoRecords = infoRecords;
this.SysmisValue = meta.InfoRecords.MachineFloatingPointInfoRecord.SystemMissingValue;
// Filler Record
reader.ReadInt32();
this.dataStartPosition = this.Stream.Position;
this.MetaData = meta;
SetDataRecordStream();
this.MetaDataParsed = true;
}
static public void Test_T(VectorVariableRecord vlrs)
{
Debug.Assert(vlrs.Count == 1);
VariableRecord vlr = vlrs[0];
Test(vlrs[0]);
}
static public void Test(VariableRecord vlr)
{
// BUG: padding bool doesn't work?
Debug.Assert(vlr.GetUserId(false) == "laszip encoded");
Debug.Assert(vlr.GetRecordId() == 22204);
Debug.Assert(vlr.GetRecordLength() == 52);
Debug.Assert(vlr.GetDescription(true) == "encoded for sequential access");
// BUG: need to check the data contents
}
static public void Test(VariableRecord vlr)
{
// BUG: padding bool doesn't work?
Debug.Assert(vlr.GetUserId(false) == "laszip encoded");
Debug.Assert(vlr.GetRecordId() == 22204);
Debug.Assert(vlr.GetRecordLength() == 52);
Debug.Assert(vlr.GetDescription(true) == "encoded for sequential access");
// BUG: need to check the data contents
}
private static string GetLongName(MetaData metaData, VariableRecord variableRecord)
{
string longName;
// Look for the right (long) name if there is one
if (metaData.LongVariableNames != null
&& metaData.LongVariableNames.Dictionary.TryGetValue(variableRecord.Name, out longName))
{
return longName;
}
// If not, just return the short name
return variableRecord.Name;
}
/// <summary>
/// Fills the variables collection with just the actual variables (no string continuation records or very long
/// strings extra segments)
/// </summary>
private void GetVariablesFromRecords()
{
_variables = new Collection <Variable>();
// Get the longs strings dictionary
var veryLongStrings = MetaData.VeryLongStringsDictionary;
// Ammount of variables to jump (to skip variable continuation records and additional very long string segments)
int delta;
// Index of variable with out string continuation records but INCLUDING very long string record variables (segments)
// This will be used for things like finding the VariableDisplayInfoRecord
int segmentIndex = 0;
// Index of variable with out string continuation records AND very long string record variables (segments)
int variableIndex = 0;
// Dictionary index is the VariableRecord index that contains the header info for this variable
for (int dictionaryIndex = 0; dictionaryIndex < MetaData.VariableRecords.Count; dictionaryIndex += delta)
{
var record = MetaData.VariableRecords[dictionaryIndex];
// If it's a string continuation record (SCR). This ones should have been skiped.
if (record.Type < 0)
{
throw new SpssFileFormatException("String continuation record out of place. Dictonary index " + dictionaryIndex);
}
// Actual byte lenght for variable
int length;
// Ammount of segments (for VeryLongString variables, 1 for all other vars).
int segments;
if (veryLongStrings.ContainsKey(record.Name))
{
// Variable is a VeryLongString variable
// Take actual length from VLSR dictionary
length = veryLongStrings[record.Name];
// Calculate the ammount of segments and the amount of VariableRecords to skip (SCR)
segments = VariableRecord.GetLongStringSegmentsCount(length);
delta = VariableRecord.GetLongStringContinuationRecordsCount(length);
}
else
{
// Variable is NOT a VeryLongString variable
// numeric type is 0 so lenght is 1, > 0 for the lenght of strings
length = record.Type == 0 ? 1 : record.Type;
// This ones have only one segment
segments = 1;
// Skip the string continuation records if any, or just move next.
delta = VariableRecord.GetStringContinuationRecordsCount(length);
}
_variables.Add(GetVariable(variableIndex++, dictionaryIndex, MetaData, length, segmentIndex));
// Increment the segment count by how many segments this var had.
segmentIndex += segments;
}
}
public MultivariableRecord(int numberOfVariables, IList <IDoubleStatistic> statisticsToCollect)
{
Debug.Assert(statisticsToCollect != null);
Debug.Assert(numberOfVariables > 0);
this.NumberOfVariables = numberOfVariables;
VariableRecords = new VariableRecord[numberOfVariables];
for (int i = 0; i < numberOfVariables; ++i)
{
VariableRecords[i] = new VariableRecord(statisticsToCollect);
}
}
public void ReadFromVariableRecord(VariableRecord vr, int variableRecordIndex)
{
this.DataType = (vr.Type == VariableType.Numeric) ? DataType.Numeric : DataType.String;
this.StringWidth = (vr.Type == VariableType.Numeric) ? 0 : (int)(vr.Type);
this.VariableRecordIndex = variableRecordIndex;
this.VariableRecordLength = 1;
this.HasVariableLabel = (vr.HasVariableLabel == 1) ? true : false;
this.MissingValuesType = this.GetMissingValueTypeFromMissingValueCount(vr.MissingValueCount);
this.MissingValues = vr.MissingValues;
this.PrintFormat = vr.PrintFormat;
this.WriteFormat = vr.WriteFormat;
this.ShortName = vr.Name.Trim();
this.Label = this.HasVariableLabel ? vr.Label.Trim() : vr.Label;
}
public object ValueToObject(byte[] value, VariableRecord variable)
{
if (variable.Type == 0)
{
var doubleValue = BitConverter.ToDouble(value, 0);
if (doubleValue == this.SysmisValue)
{
return(null);
}
else
{
return(doubleValue);
}
}
else
{
return(Encoding.ASCII.GetString(value));
}
}
private void ReadDictionary()
{
RecordType currentRecordType = (RecordType)this.BinaryReader.ReadInt32();
while (currentRecordType != RecordType.DictionaryTerminationRecord)
{
switch (currentRecordType)
{
case RecordType.FileHeaderRecord:
this.SystemFile.FileHeaderRecord.ReadFromStream(this.BinaryReader);
break;
case RecordType.VariableRecord:
VariableRecord v = new VariableRecord(this.SystemFile);
v.ReadFromStream(this.BinaryReader);
this.SystemFile.VariableRecords.Add(v);
break;
case RecordType.ValueLabelRecord:
ValueLabelsRecord vl = new ValueLabelsRecord(this.SystemFile);
vl.ReadFromStream(this.BinaryReader);
this.SystemFile.ValueLabelsRecords.Add(vl);
break;
case RecordType.DocumentRecord:
this.SystemFile.DocumentRecord.ReadFromStream(this.BinaryReader);
break;
case RecordType.InfoRecord:
InfoRecord ir = new InfoRecord(this.SystemFile);
ir.ReadFromStream(this.BinaryReader);
this.SystemFile.InfoRecords.Add(ir);
break;
default:
break;
}
currentRecordType = (RecordType)this.BinaryReader.ReadInt32();
this.ParseInfoRecords();
}
}
public VariableHandler(ConfigNode node, Part ourPart)
{
handlerName = node.GetValue("name");
foreach (string variableRecord in node.GetValues("variable"))
{
var record = new VariableRecord();
string[] tokens = variableRecord.Split(',');
if (tokens.Length >= 2)
{
double defaultDouble;
if (double.TryParse(tokens[1], NumberStyles.Any, CultureInfo.InvariantCulture, out defaultDouble))
{
record.defaultValue = defaultDouble;
}
else
{
if (tokens[1].Trim() == "fallback")
{
record.fallback = true;
}
else
{
record.defaultValue = tokens[1];
//record.defaultString = tokens[1];
}
}
}
if (tokens.Length >= 3)
{
record.cacheable = bool.Parse(tokens[2]);
}
handledVariables.Add(tokens[0], record);
}
active = InstantiateHandler(node, ourPart, out handlerFunction);
}
public VariableHandler(ConfigNode node, Part ourPart)
{
handlerName = node.GetValue("name");
foreach (string variableRecord in node.GetValues("variable"))
{
var record = new VariableRecord();
string[] tokens = variableRecord.Split(',');
if (tokens.Length >= 2)
{
double defaultDouble;
if (double.TryParse(tokens[1], NumberStyles.Any, CultureInfo.InvariantCulture, out defaultDouble))
{
record.defaultValue = defaultDouble;
}
else
{
if (tokens[1].Trim() == "fallback")
{
record.fallback = true;
}
else
{
record.defaultValue = tokens[1];
//record.defaultString = tokens[1];
}
}
}
if (tokens.Length >= 3)
{
record.cacheable = bool.Parse(tokens[2]);
}
handledVariables.Add(tokens[0], record);
}
active = InstantiateHandler(node, ourPart, out handlerFunction);
}
/// <summary>
/// Parse SimpleExpression
/// </summary>
/// <param name="simpleExpressionRecord"></param>
private void SimpleExpression(ref VariableRecord simpleExpressionRecord)
{
string signRecord = string.Empty;
switch(lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_INTEGER_LIT:
case Tags.MP_NOT:
case Tags.MP_IDENTIFIER:
UsedRules.WriteLine("78");
OptionalSign(ref signRecord);
Term(ref simpleExpressionRecord);
analyzer.GenerateNegation(signRecord);
TermTail(ref simpleExpressionRecord);
break;
default:
Error("Expecting Simple Expression found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse Term
/// </summary>
/// <param name="termRecord"></param>
private void Term(ref VariableRecord termRecord)
{
switch(lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_IDENTIFIER:
case Tags.MP_INTEGER_LIT:
case Tags.MP_NOT:
UsedRules.WriteLine("87");
Factor(ref termRecord);
FactorTail(ref termRecord);
break;
default:
Error("Expecting Term found " + lookAheadToken.tag);
break;
}
}
/// <summary>
/// Parse TermTail
/// </summary>
/// <param name="termTailRecord"></param>
private void TermTail(ref VariableRecord termTailRecord)
{
string addOpRecord = null;
VariableRecord termRecord = new VariableRecord();
VariableRecord resultRecord = new VariableRecord();
VariableType tempType = VariableType.Void;
switch(lookAheadToken.tag)
{
//AddingOperator Term TermTail
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_OR:
UsedRules.WriteLine("79");
AddingOperator(ref addOpRecord);
Term(ref termRecord);
analyzer.GenerateArithmetic(termTailRecord.variableType, addOpRecord,
termRecord.variableType,ref tempType);
resultRecord.variableType = tempType;
TermTail(ref resultRecord);
break;
case Tags.MP_SCOLON:
case Tags.MP_RPAREN:
case Tags.MP_END:
case Tags.MP_COMMA:
case Tags.MP_THEN:
case Tags.MP_ELSE:
case Tags.MP_UNTIL:
case Tags.MP_DO:
case Tags.MP_TO:
case Tags.MP_DOWNTO:
case Tags.MP_EQUAL:
case Tags.MP_LTHAN:
case Tags.MP_GTHAN:
case Tags.MP_LEQUAL:
case Tags.MP_GEQUAL:
case Tags.MP_NEQUAL:
UsedRules.WriteLine("80");
//lambda
break;
default:
Error("Expecting TermTail but found " + lookAheadToken.lexeme);
break;
}
}
public static VariableRecord[] GetVariableRecords(INetCDFReader reader, float minLat,
float maxLat, float minLon, float maxLon)
{
float[][][] lats = reader.Read3DFloatArray("XLAT");
float[][][] lons = reader.Read3DFloatArray("XLONG");
float[][][] rainc = reader.Read3DFloatArray("RAINC");
float[][][] rainnc = reader.Read3DFloatArray("RAINNC");
float[][][] tc2 = reader.Read3DFloatArray("T2");
float[][][] snowh = reader.Read3DFloatArray("SNOWH");
float[][][] surfacePressure = reader.Read3DFloatArray("PSFC");
float[][][] surfaceSkinTemp = reader.Read3DFloatArray("TSK");
float[][][] u10 = reader.Read3DFloatArray("U10");
float[][][] v10 = reader.Read3DFloatArray("V10");
float[][][] cloudFraction = reader.Read3DFloatArray("CLDFRA");
DateTime[] dates = reader.ReadDateArray("Times");
//tc2 = tc2-273.16 ; T2 in C
//tf2 = 1.8*tc2+32. ; Turn temperature into Fahrenheit
List <VariableRecord> ret = new List <VariableRecord>();
for (int t = 0; t < dates.Length; t++)
{
DateTime date = dates[t];
for (int y = 0; y < lats.Length; y++)
{
for (int x = 0; x < lons.Length; x++)
{
float lat = lats[t][y][x];
float lon = lons[t][y][x];
if (lat > minLat && lat < maxLat &&
lon < maxLon && lon > minLon)
{
float tempInK = tc2[t][y][x];
float tempInC = tempInK - 273.16f;
float tempInF = 1.8f * tempInC + 32;
float skinTempInK = surfaceSkinTemp[t][y][x];
float skinTempInC = skinTempInK - 273.16f;
float skinTempInF = 1.8f * skinTempInC + 32;
VariableRecord rec = new VariableRecord
{
DateTime = date,
Lat = lat,
Lon = lon,
PrecipInMM = rainc[t][y][x] + rainnc[t][y][x],
TempInF = tempInF,
SnowDepthInM = snowh[t][y][x],
SurfacePressure = surfacePressure[t][y][x],
SurfaceSkinTemperature = skinTempInF,
UWind = u10[t][y][x],
VWind = v10[t][y][x],
//CloudFraction = cloudFraction[t][y][x],
};
ret.Add(rec);
}
}
}
}
return(ret.ToArray());
}
/// <summary>
/// Parse OptionalRelationalPart
/// </summary>
/// <param name="expressionRecord"></param>
private void OptionalRelationalPart(ref VariableRecord expressionRecord)
{
VariableRecord simpleExpressionRecord = new VariableRecord();
VariableType resultRecord = VariableType.Void;
string relationalOpRecord = string.Empty;
switch (lookAheadToken.tag)
{
case Tags.MP_EQUAL:
case Tags.MP_LTHAN:
case Tags.MP_GTHAN:
case Tags.MP_LEQUAL:
case Tags.MP_GEQUAL:
case Tags.MP_NEQUAL://RelationalOperator SimpleExpression
UsedRules.WriteLine("70");
RelationalOperator(ref relationalOpRecord);
SimpleExpression(ref simpleExpressionRecord);
analyzer.GenerateArithmetic(expressionRecord.variableType, relationalOpRecord,
simpleExpressionRecord.variableType, ref resultRecord);
break;
case Tags.MP_SCOLON:
case Tags.MP_RPAREN:
case Tags.MP_END:
case Tags.MP_COMMA:
case Tags.MP_THEN:
case Tags.MP_ELSE:
case Tags.MP_UNTIL:
case Tags.MP_DO:
case Tags.MP_TO:
case Tags.MP_DOWNTO://lambda
UsedRules.WriteLine("71");
break;
default:
Error("Expecting OptionalRelationalPart but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse ActualParameter
/// </summary>
private void ActualParameter(ref List<Parameter> parameterList)
{
VariableRecord parameterRecord = new VariableRecord();
switch (lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_INTEGER_LIT:
case Tags.MP_NOT:
case Tags.MP_IDENTIFIER:
UsedRules.WriteLine("68");
analyzer.processParameter(ref parameterRecord, ref parameterList);
OrdinalExpression(ref parameterRecord);
break;
default:
Error("Expecting ActualParameter but found " + lookAheadToken.lexeme);
break;
}
}
private static byte[] MoveNext(byte[][] record, IList <VariableRecord> variableRecords, out VariableRecord variableRecord, ref int variableIndex)
{
variableRecord = variableRecords[variableIndex];
return(record[variableIndex++]);
}
/// <summary>
/// Parse AssignmentStatement
/// </summary>
private void AssignmentStatement()
{
IdentifierRecord idRecord = new IdentifierRecord();
VariableRecord expressionRecord = new VariableRecord();
string idRecName = null;
switch (lookAheadToken.tag)
{
case Tags.MP_IDENTIFIER: // Identifier ":=" Expression
UsedRules.WriteLine("51");
Identifier(ref idRecName);
idRecord.lexeme = idRecName;
analyzer.ProcessId(ref idRecord);
Match((int)Tags.MP_ASSIGN);
analyzer.GenerateObjectScope(idRecord, StoreMode.Store);
Expression(ref expressionRecord);
analyzer.GenerateAssign(idRecord, expressionRecord);
break;
default:
Error("Expecting AssignmentStatement but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse FactorTail
/// </summary>
/// <param name="factorTailRecord"></param>
private void FactorTail(ref VariableRecord factorTailRecord)
{
string mulOpRecord = null;
VariableRecord factorRecord = new VariableRecord(),
resultRecord = new VariableRecord();
VariableType tempType = VariableType.Void;
switch (lookAheadToken.tag)
{
case Tags.MP_TIMES:
case Tags.MP_DIV:
case Tags.MP_MOD:
case Tags.MP_AND:
UsedRules.WriteLine("88");
MultiplyingOperator(ref mulOpRecord);
Factor(ref factorTailRecord);
analyzer.GenerateArithmetic(factorTailRecord.variableType, mulOpRecord,
factorRecord.variableType,ref tempType);
resultRecord.variableType = tempType;
FactorTail(ref factorTailRecord);
break;
case Tags.MP_SCOLON://lambda case!
case Tags.MP_RPAREN:
case Tags.MP_END:
case Tags.MP_COMMA:
case Tags.MP_THEN:
case Tags.MP_ELSE:
case Tags.MP_UNTIL:
case Tags.MP_DO:
case Tags.MP_TO:
case Tags.MP_DOWNTO:
case Tags.MP_EQUAL:
case Tags.MP_LTHAN:
case Tags.MP_GTHAN:
case Tags.MP_LEQUAL:
case Tags.MP_GEQUAL:
case Tags.MP_NEQUAL:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_OR:
UsedRules.WriteLine("89");
break;
default:
Error("Expecting FactorTail but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse ForStatement
/// </summary>
private void ForStatement()
{
IdentifierRecord controlVariableRecord = new IdentifierRecord();
VariableRecord finalValueRecord = new VariableRecord();
VariableRecord controlStatementRecord = new VariableRecord();
string controlLabelRecord = string.Empty;
string loopLabel = string.Empty;
ControlRecord stepValueRecord = new ControlRecord();
switch(lookAheadToken.tag)
{
case Tags.MP_FOR:
UsedRules.WriteLine("57");
Match((int)Tags.MP_FOR);
ControlVariable(ref controlVariableRecord);
Match((int)Tags.MP_ASSIGN);
InitialValue(ref controlVariableRecord);
analyzer.GenerateLabel(ref controlLabelRecord);
analyzer.GenerateIdPush(controlVariableRecord,ref controlStatementRecord);
StepValue(ref stepValueRecord);
FinalValue(ref finalValueRecord);
analyzer.GenerateBranch(ref loopLabel, stepValueRecord.branchType);
Match((int)Tags.MP_DO);
Statement();
analyzer.GenerateIncrement(ref controlVariableRecord, stepValueRecord.addingOperator);
analyzer.GenerateBranch(ref controlLabelRecord, BranchType.br);
analyzer.GenerateLabel(ref loopLabel);
break;
default:
Error("Expecting ForStatement found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse OrdinalExpression
/// </summary>
/// <param name="expressionRecord"></param>
private void OrdinalExpression(ref VariableRecord expressionRecord)
{
switch (lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_IDENTIFIER:
case Tags.MP_NOT:
case Tags.MP_INTEGER_LIT:
UsedRules.WriteLine("99");
Expression(ref expressionRecord);
break;
default:
Error("Expecting OridinalExpression but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse Expression
/// </summary>
/// <param name="expressionRecord"></param>
private void Expression(ref VariableRecord expressionRecord)
{
switch(lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_INTEGER_LIT:
case Tags.MP_NOT:
case Tags.MP_IDENTIFIER:
UsedRules.WriteLine("69");
SimpleExpression(ref expressionRecord);
OptionalRelationalPart(ref expressionRecord);
break;
default:
Error("Expecting Expression Args found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Convert a row of raw data to proper objects. (strings or doubles)
/// </summary>
/// <param name="record">The complete row data, as an array of byte[8] (the block of a single VariableRecord)</param>
/// <returns>The enumeration of objects for this row</returns>
public IEnumerable <object> RecordToObjects(byte[][] record)
{
// Decoder for strings
Decoder dec = MetaData.DataEncoding.GetDecoder();
// Buffer to write the decoded chars to
var charBuffer = new char[MetaData.DataEncoding.GetMaxCharCount(256)];
// String builder to get the full string result (mainly for VLS)
StringBuilder sBuilder = new StringBuilder();
// The dictionary with the lengths for each VLS variable
var veryLongStrings = MetaData.VeryLongStringsDictionary;
// All raw variable records and it's count (i's also the count of 8 bytes blocks in the row)
var variableRecords = MetaData.VariableRecords;
var variableCount = variableRecords.Count;
// Read the values, guided by it's VariableRecord
for (int variableIndex = 0; variableIndex < variableCount;)
{
// Variable record that correspond to the current 8 bytes block
VariableRecord variableRecord;
// Currrent 8 bytes block
byte[] element = MoveNext(record, variableRecords, out variableRecord, ref variableIndex);
// Numeric variable (also date, etc)
if (variableRecord.Type == 0)
{
// Convert value to double and check Sysmiss
yield return(ParseDoubleValue(element));
}
// String variable
else if (variableRecord.Type > 0)
{
// Count of segments of up to 255 bytes. 1 for not VLS
int segments = 1;
// If VLS, calculate total count of segments according to the value for this var in the VLS dictionary
if (veryLongStrings.ContainsKey(variableRecord.Name))
{
segments = VariableRecord.GetLongStringSegmentsCount(veryLongStrings[variableRecord.Name]);
}
// Ok, so let's start reading all the string...
do
{
// The index of the char buffer. How many chars have been read
var bufferIndex = 0;
// The length of this segment, in bytes
var length = variableRecord.Type;
// Count of bytes read of the string
int bytesRead = element.Length;
// The index for the 8 byte block inside the VLS segment
int inSegmentIndex = 1;
// Start reading the segment
// Decode the characters into the charBuffer array and increment the index
bufferIndex += dec.GetChars(element, 0, element.Length, charBuffer, bufferIndex, false);
// While we haven't read all bytes, is still a SCR and not the end of the row
while (bytesRead < length && variableRecords[variableIndex].Type == -1 && variableIndex < variableCount)
{
// Read next block
element = MoveNext(record, variableRecords, out variableRecord, ref variableIndex);
// When we get to the 32nd segment, we have to ignote the 8th byte, as it is the
// number 256 and segments are only 255. If this byte is not skiped, spaces will
// appear where they shouldn't be.
int lengthRead = element.Length;
if (++inSegmentIndex == 32)
{
// Substract one from the read length to ignore the last bye
lengthRead--;
// Reset the counter for a new segment
inSegmentIndex = 0;
}
// Decode the characters into the charBuffer array and increment the index
bufferIndex += dec.GetChars(element, 0, lengthRead, charBuffer, bufferIndex, false);
bytesRead += element.Length;
}
// If the type of variable changed before the end of the string length or before the end of the file
// there must be something wrong
if (length > 8 && variableRecord.Type != -1)
{
throw new SpssFileFormatException("Long string terminated early. " +
"There must be missing some of the needed string continuation record. Dictionary index " +
variableIndex);
}
// Flush the buffer of the decoder
bufferIndex += dec.GetChars(element, 0, 0, charBuffer, bufferIndex, true);
// take the segment's string we were building (the buffer up to the writen index)
sBuilder.Append(charBuffer, 0, bufferIndex);
// If there afe more records, move next and continue
if (--segments > 0)
{
element = MoveNext(record, variableRecords, out variableRecord, ref variableIndex);
}
else
{
// if all segments are processed, exit the loop
break;
}
} while (true);
// Clear decoder for future use
dec.Reset();
// Get full string and clear the string builder
var s = sBuilder.ToString();
sBuilder.Clear();
// Finally, return the string
yield return(s);
}
// String Continuation record (either we read something wrong or the file is not very well formed)
else if (variableRecord.Type == -1)
{
throw new SpssFileFormatException("Unexpected string continuation record. To start reading the record must be either string or numeric (dates, etc). " +
"Dictionary index " + variableIndex);
}
// I don't know any more VariableRecord's types
else
{
throw new SpssFileFormatException("Unrecognized variable type: " + variableRecord.Type);
}
}
}
/// <summary>
/// Generates code for pushing an Identifier
/// </summary>
/// <param name="idRecord"></param>
/// <param name="factorRecord"></param>
internal void GenerateIdPush(IdentifierRecord idRecord, ref VariableRecord factorRecord)
{
GenerateObjectScope(idRecord,StoreMode.Load);
if (factorRecord.ioMode == IOMode.InOut)
{
cilOutput.Write(" ldflda\t");
}
else
{
cilOutput.Write(" ldfld\t");
}
if (idRecord.symbol.symbolType == SymbolType.FunctionSymbol)
{
cilOutput.WriteLine("class Program/" + idRecord.lexeme + "Delegate " +
idRecord.symbolTable.cilScope + "/c__" + idRecord.symbolTable.name
+ "::d__" + idRecord.lexeme + Environment.NewLine);
}
else
{
GenerateFieldLocation(idRecord);
}
if (idRecord.symbol is ParameterSymbol)
{
if ((idRecord.symbol as ParameterSymbol).parameter.mode == IOMode.InOut)
{
cilOutput.WriteLine("ldind.i4");
}
}
factorRecord.variableType = idRecord.symbol.variableType;
}
/// <summary>
/// Parse Factor
/// </summary>
/// <param name="factorRecord"></param>
private void Factor(ref VariableRecord factorRecord)
{
IdentifierRecord idRecord = new IdentifierRecord();
LiteralRecord litRecord = new LiteralRecord();
VariableType tempType = VariableType.Void;
List<Parameter> parameterList = new List<Parameter>();
MethodRecord methodRecord = new MethodRecord();
methodRecord.parameterList = new List<Parameter>();
string idRecName = null;
switch (lookAheadToken.tag)
{
case Tags.MP_INTEGER_LIT:
UsedRules.WriteLine("94");
litRecord.lexeme = lookAheadToken.lexeme;
litRecord.type = VariableType.Integer;
Match((int)Tags.MP_INTEGER_LIT);
analyzer.GenerateLitPush(litRecord, ref tempType);
factorRecord.variableType = tempType;
break;
case Tags.MP_NOT:
UsedRules.WriteLine("95");
Match((int)Tags.MP_NOT);
Factor(ref factorRecord);
break;
case Tags.MP_LPAREN:
UsedRules.WriteLine("96");
Match((int)Tags.MP_LPAREN);
Expression(ref factorRecord);
Match((int)Tags.MP_RPAREN);
break;
case Tags.MP_IDENTIFIER:
UsedRules.WriteLine("97");
Identifier(ref idRecName);
idRecord.lexeme = idRecName;
analyzer.ProcessId(ref idRecord);
analyzer.GenerateIdPush(idRecord, ref factorRecord);
analyzer.processParameters(idRecord,ref methodRecord,ref parameterList);
OptionalActualParameterList(parameterList);
analyzer.ProcessMethod(idRecord,ref methodRecord);
analyzer.GenerateCallMethod(methodRecord);
break;
default:
Error("Expecting Factor but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Process parameter
/// </summary>
/// <param name="parameterRecord"></param>
/// <param name="parameterList"></param>
internal void processParameter(ref VariableRecord parameterRecord, ref List<Parameter> parameterList)
{
parameterRecord.ioMode = parameterList.ElementAt(0).mode;
parameterRecord.variableType = parameterList.ElementAt(0).variableType;
parameterList.RemoveAt(0);
}
/// <summary>
/// Parse WriteParameter
/// </summary>
private void WriteParameter()
{
VariableRecord ordinalExpressionRecord = new VariableRecord();
switch(lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_INTEGER_LIT:
case Tags.MP_NOT:
case Tags.MP_IDENTIFIER:
UsedRules.WriteLine("50");
OrdinalExpression(ref ordinalExpressionRecord);
break;
default:
Error("Expecting WriteParameter but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Parse InitialValue
/// </summary>
/// <param name="initialValueRecord"></param>
private void InitialValue(ref IdentifierRecord initialValueRecord)
{
VariableRecord ordinalExpressionRecord = new VariableRecord();
switch (lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_INTEGER_LIT:
case Tags.MP_NOT:
case Tags.MP_IDENTIFIER:
UsedRules.WriteLine("59");
analyzer.GenerateObjectScope(initialValueRecord, StoreMode.Store);
OrdinalExpression(ref ordinalExpressionRecord);
analyzer.GenerateAssign(initialValueRecord, ordinalExpressionRecord);
break;
default:
Error("Expecting InitialValue but found " + lookAheadToken.lexeme);
break;
}
}
/// <summary>
/// Generates code for assignment statements
/// </summary>
/// <param name="idRecord"></param>
/// <param name="expressionRecord"></param>
internal void GenerateAssign(IdentifierRecord idRecord, VariableRecord expressionRecord)
{
switch (idRecord.symbol.symbolType)
{
case SymbolType.VariableSymbol:
cilOutput.WriteLine(" stfld\t" +
Enumerations.GetDescription<VariableType>(idRecord.symbol.variableType) + " " +
idRecord.symbolTable.cilScope + "/c__" + idRecord.symbolTable.name
+ "::" + idRecord.lexeme + Environment.NewLine);
break;
case SymbolType.ParameterSymbol:
if ((idRecord.symbol as ParameterSymbol).parameter.mode == IOMode.InOut)
{
cilOutput.WriteLine(" stind.i4");
}
else
{
cilOutput.WriteLine(" stfld\t" +
Enumerations.GetDescription<VariableType>(idRecord.symbol.variableType) + " " +
idRecord.symbolTable.cilScope + "/c__" + idRecord.symbolTable.name
+ "::" + idRecord.lexeme + Environment.NewLine);
}
break;
case SymbolType.FunctionSymbol:
cilOutput.WriteLine(" stloc.1" + Environment.NewLine);
break;
}
}
/// <summary>
/// Parse BooleanExpression
/// </summary>
private void BooleanExpression()
{
switch (lookAheadToken.tag)
{
case Tags.MP_LPAREN:
case Tags.MP_PLUS:
case Tags.MP_MINUS:
case Tags.MP_INTEGER:
case Tags.MP_NOT:
case Tags.MP_IDENTIFIER:
VariableRecord expressionRecord = new VariableRecord();
UsedRules.WriteLine("98");
Expression(ref expressionRecord);
break;
default:
Error("Expecting BooleanExpression but found " + lookAheadToken.lexeme);
break;
}
}
