private static MetadataRecord ReadRecord(XmlNode recordElement)
{
MetadataRecord record = new MetadataRecord();
for (int recordIndex = 0; recordIndex < recordElement.ChildNodes.Count; recordIndex++)
{
XmlNode element = recordElement.ChildNodes[recordIndex];
switch (element.Name)
{
case "Operation":
record.Operation = element.InnerText;
break;
case "Arguments":
record.Arguments = ReadArguments(element);
break;
case "User":
record.User = element.InnerText;
break;
case "Time":
string dateTimeString = element.InnerText;
record.Time = DateTime.Parse(dateTimeString);
break;
case "ComputerName":
record.ComputerName = element.InnerText;
break;
default:
throw new Exception("Unexpected element found \"" + element.Name + "\"");
}
}
return record;
}
public static void RemoveColumns(string inputFileName, string outputFileName, List<string> variableNames)
{
if (!FileFunctions.FileExists(inputFileName))
{
throw new Exception("cannot find file \"" + inputFileName + "\"");
}
if (variableNames.Count == 0)
{
throw new Exception("Variables missing");
}
if (Path.GetFullPath(inputFileName) == Path.GetFullPath(outputFileName))
{
throw new Exception("Output cannot be the same as input");
}
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
StataScriptFunctions.LoadFileCommand(lines, inputFileName);
string line = "drop " + StataScriptFunctions.VariableList(variableNames);
lines.Add(line);
line = StataScriptFunctions.SaveFileCommand(outputFileName);
lines.Add(line);
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(outputFileName))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(inputFileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(inputFileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", inputFileName));
record.Arguments.Add(new Tuple<string, string>("OutputFileName", outputFileName));
record.Arguments.Add(new Tuple<string, string>("VariableNames", string.Join(" ", variableNames)));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
record.ComputerName = Environment.MachineName;
record.Operation = "RemoveColumns";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(outputFileName, metadata);
}
public static void Replace(string inputFileName, string outputFileName, string condition, string value)
{
//SwishFunctions.MessageTextBox(""
//+ "inputFileName: " + inputFileName + Environment.NewLine
//+ "outputFileName: " + outputFileName + Environment.NewLine
//+ "condition: " + condition + Environment.NewLine
//+ "value: " + value + Environment.NewLine, false);
if (!FileFunctions.FileExists(inputFileName))
{
throw new Exception("cannot find file \"" + inputFileName + "\"");
}
if (string.IsNullOrWhiteSpace(condition))
{
throw new Exception("condition missing");
}
if (string.IsNullOrWhiteSpace(value))
{
throw new Exception("value missing");
}
if (Path.GetFullPath(inputFileName) == Path.GetFullPath(outputFileName))
{
throw new Exception("Output cannot be the same as input");
}
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
StataScriptFunctions.LoadFileCommand(lines, inputFileName);
//replace oldvar =exp [if] [in] [, nopromote]
lines.Add("replace " + value + " if " + condition);
string line = StataScriptFunctions.SaveFileCommand(outputFileName);
lines.Add(line);
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(outputFileName))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(inputFileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(inputFileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", inputFileName));
record.Arguments.Add(new Tuple<string, string>("OutputFileName", outputFileName));
record.Arguments.Add(new Tuple<string, string>("Condition", condition));
record.Arguments.Add(new Tuple<string, string>("Value", value));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
record.ComputerName = Environment.MachineName;
record.Operation = "Replace";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(outputFileName, metadata);
}
public static string Append(string input1FileName, string input2FileName, string outputFileName)
{
if (!FileFunctions.FileExists(input1FileName))
{
throw new Exception("cannot find file \"" + input1FileName + "\"");
}
if (!FileFunctions.FileExists(input2FileName))
{
throw new Exception("cannot find file \"" + input2FileName + "\"");
}
if (
Path.GetFullPath(input1FileName) == Path.GetFullPath(outputFileName)
|| Path.GetFullPath(input2FileName) == Path.GetFullPath(outputFileName)
)
{
throw new Exception("Output cannot be the same as input");
}
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
string intermediateFileName = StataScriptFunctions.ConvertToStataFormat(lines, input2FileName);
StataScriptFunctions.LoadFileCommand(lines, input1FileName);
lines.Add("append using \"" + intermediateFileName + "\"");
string line = StataScriptFunctions.SaveFileCommand(outputFileName);
lines.Add(line);
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(outputFileName))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
/// delete all the files not needed
File.Delete(intermediateFileName);
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(input1FileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(input1FileName);
metadata.AddRange(inputMetadata);
}
if (MetadataFunctions.Exists(input2FileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(input2FileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", input1FileName));
record.Arguments.Add(new Tuple<string, string>("InputFileName", input2FileName));
record.Arguments.Add(new Tuple<string, string>("OutputFileName", outputFileName));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
record.ComputerName = Environment.MachineName;
record.Operation = "Append";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(outputFileName, metadata);
return outputFileName;
}
public static void Merge(string input1FileName, string input2FileName, List<string> variableNames, string outputFileName, bool keepMergeColumn, List<MergeRecordResult> keep)
{
//if (!FileFunctions.FileExists(input1FileName))
//{
// throw new Exception("cannot find file \"" + input1FileName + "\"");
//}
//if (!FileFunctions.FileExists(input2FileName))
//{
// throw new Exception("cannot find file \"" + input2FileName + "\"");
//}
if (
Path.GetFullPath(input1FileName) == Path.GetFullPath(outputFileName)
|| Path.GetFullPath(input2FileName) == Path.GetFullPath(outputFileName)
)
{
throw new Exception("Output cannot be the same as input");
}
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
if (input1FileName == input2FileName)
{
throw new Exception("Cannot merge the same tables");
}
if (variableNames.Count == 0)
{
throw new Exception("Variables missing");
}
string doOutputFileName = FileFunctions.TempoaryOutputFileName(Path.GetExtension(outputFileName));
if (FileFunctions.FileExists(doOutputFileName))
{
// Stata does not overwrite files
File.Delete(doOutputFileName);
}
string intermediateFileName = FileFunctions.TempoaryOutputFileName(".dta");
if (FileFunctions.FileExists(intermediateFileName))
{
File.Delete(intermediateFileName);
}
/// create the do file
// merge [varlist] using filename [filename ...] [, options]
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
StataScriptFunctions.LoadFileCommand(lines, input2FileName);
string line = StataScriptFunctions.SortCommand(variableNames);
lines.Add(line);
line = StataScriptFunctions.SaveFileCommand(intermediateFileName);
lines.Add(line);
lines.Add("clear");
StataScriptFunctions.LoadFileCommand(lines, input1FileName);
line = StataScriptFunctions.SortCommand(variableNames);
lines.Add(line);
//line = "merge 1:1 " + StataScriptFunctions.VariableList(variableNames) + " using \"" + intermediateFileName + "\", force ";
//if (keep != null && keep.Count > 0)
//{
// line += "keep(";
// if (keep.Contains(MergeRecordResult.Match))
// {
// line += "match ";
// }
// if (keep.Contains(MergeRecordResult.MatchConflict))
// {
// line += "match_conflict ";
// }
// if (keep.Contains(MergeRecordResult.MatchUpdate))
// {
// line += "match_update ";
// }
// if (keep.Contains(MergeRecordResult.Using))
// {
// line += "using ";
// }
// line += ") ";
//}
//lines.Add(line);
line = "merge " + StataScriptFunctions.VariableList(variableNames) + ", using \"" + intermediateFileName + "\"";
lines.Add(line);
if (!keepMergeColumn)
{
lines.Add("drop " + StataScriptFunctions.MergeColumnName);
}
line = StataScriptFunctions.SaveFileCommand(doOutputFileName);
lines.Add(line);
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(doOutputFileName))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
/// move the output file
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
File.Move(doOutputFileName, outputFileName);
/// delete all the files not needed
File.Delete(intermediateFileName);
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(input1FileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(input1FileName);
metadata.AddRange(inputMetadata);
}
if (MetadataFunctions.Exists(input2FileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(input2FileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", input1FileName));
record.Arguments.Add(new Tuple<string, string>("InputFileName", input2FileName));
record.Arguments.Add(new Tuple<string, string>("OutputFileName", outputFileName));
record.Arguments.Add(new Tuple<string, string>("VariableNames", string.Join(" ", variableNames)));
record.Arguments.Add(new Tuple<string, string>("KeepMergeColumn", keepMergeColumn.ToString()));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
if (keep != null)
{
string keepString = string.Join(" ", keep);
record.Arguments.Add(new Tuple<string, string>("Keep", keepString));
}
record.ComputerName = Environment.MachineName;
record.Operation = "Merge";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(outputFileName, metadata);
}
public static void Generate(string inputFileName, string outputFileName, string variableName, string type, string expression)
{
if (!FileFunctions.FileExists(inputFileName))
{
throw new Exception("cannot find file \"" + inputFileName + "\"");
}
if (string.IsNullOrWhiteSpace(variableName))
{
throw new Exception("Variable missing");
}
if (string.IsNullOrWhiteSpace(expression))
{
throw new Exception("Expression missing");
}
string extension = Path.GetExtension(outputFileName);
string intermaidateOutput = FileFunctions.TempoaryOutputFileName(extension);
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
StataScriptFunctions.LoadFileCommand(lines, inputFileName);
if (string.IsNullOrWhiteSpace(type))
{
lines.Add(" generate " + variableName + " =" + expression);
} else
{
lines.Add(" generate " + type + " " + variableName + " =" + expression);
}
string line = StataScriptFunctions.SaveFileCommand(intermaidateOutput);
lines.Add(line);
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(intermaidateOutput))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
File.Move(intermaidateOutput, outputFileName);
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(inputFileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(inputFileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", inputFileName));
record.Arguments.Add(new Tuple<string, string>("OutputFileName", outputFileName));
record.Arguments.Add(new Tuple<string, string>("Variable", variableName));
record.Arguments.Add(new Tuple<string, string>("Type", type));
record.Arguments.Add(new Tuple<string, string>("Expression", expression));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
record.ComputerName = Environment.MachineName;
record.Operation = "Generate";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(outputFileName, metadata);
}
public static double Collapse(string inputFileName, string variable, CollapseOpperation operation)
{
if (!FileFunctions.FileExists(inputFileName))
{
throw new Exception("cannot find file \"" + inputFileName + "\"");
}
string doOutputFileName = FileFunctions.TempoaryOutputFileName(".csv");
if (FileFunctions.FileExists(doOutputFileName))
{
// Stata does not overwrite files
File.Delete(doOutputFileName);
}
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
StataScriptFunctions.LoadFileCommand(lines, inputFileName);
// collapse (mean) mean=head4 (median) medium=head4, by(head6)
lines.Add("collapse " + "(" + StataScriptFunctions.Write(operation) + ") " + variable + "_" + StataScriptFunctions.Write(operation) + "=" + variable);
string line = StataScriptFunctions.SaveFileCommand(doOutputFileName);
lines.Add(line);
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(doOutputFileName))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
string[] resultLines = File.ReadAllLines(doOutputFileName);
if (resultLines.Length < 2)
{
throw new Exception("Unexpected format");
}
double result = double.Parse(resultLines[1]);
/// delete all the files not needed
File.Delete(doOutputFileName);
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(inputFileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(inputFileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", inputFileName));
record.Arguments.Add(new Tuple<string, string>("Variable", variable));
record.Arguments.Add(new Tuple<string, string>("Operation", operation.ToString()));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
record.ComputerName = Environment.MachineName;
record.Operation = "Collapse";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(inputFileName + "out", metadata);
return result;
}
public static void SaveFile(string inputFileName, string outputFileName)
{
if (!FileFunctions.FileExists(inputFileName))
{
throw new Exception("cannot find file \"" + inputFileName + "\"");
}
string inputFileExtension = Path.GetExtension(inputFileName);
string outputFileExtension = Path.GetExtension(outputFileName);
if (inputFileExtension.ToLower() == outputFileExtension.ToLower())
{
File.Copy(inputFileName, outputFileName);
return;
}
List<string> lines = new List<string>();
StataScriptFunctions.WriteHeadder(lines);
StataScriptFunctions.LoadFileCommand(lines, inputFileName);
string line = StataScriptFunctions.SaveFileCommand(outputFileName);
lines.Add(line);
if (FileFunctions.FileExists(outputFileName))
{
File.Delete(outputFileName);
}
StataScriptFunctions.WriteFooter(lines);
string log = StataFunctions.RunScript(lines, false);
if (!FileFunctions.FileExists(outputFileName))
{
throw new Exception("Output file was not created" + Environment.NewLine + log + Environment.NewLine + "Script lines: " + Environment.NewLine + string.Join(Environment.NewLine, lines) + Environment.NewLine);
}
List<MetadataRecord> metadata = new List<MetadataRecord>();
if (MetadataFunctions.Exists(inputFileName))
{
List<MetadataRecord> inputMetadata = MetadataFunctions.Load(inputFileName);
metadata.AddRange(inputMetadata);
}
MetadataRecord record = new MetadataRecord();
record.Arguments.Add(new Tuple<string, string>("InputFileName", inputFileName));
record.Arguments.Add(new Tuple<string, string>("OutputFileName", outputFileName));
if (RecordDoScript) { record.Arguments.Add(new Tuple<string, string>("DoScript", string.Join(Environment.NewLine, lines))); }
record.ComputerName = Environment.MachineName;
record.Operation = "Save";
record.Time = DateTime.Now;
record.User = Environment.UserName;
metadata.Add(record);
MetadataFunctions.Save(outputFileName, metadata);
}
