XDocument FetchQuotesFromYahoo(string symbols)
{
// see http://cliffngan.net/a/13 or
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(address + "?s=" + symbols + "&f=snl1p");
req.ContentType = "text/csv; charset=utf-8";
req.Accept = "text/csv";
req.UserAgent = "USER_AGENT=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;)";
req.Method = "GET";
req.Timeout = 10000;
req.UseDefaultCredentials = false;
_current = req;
WebResponse resp = req.GetResponse();
using (Stream stm = resp.GetResponseStream())
{
XDocument doc = null;
using (StreamReader sr = new StreamReader(stm, Encoding.UTF8))
{
using (XmlCsvReader reader = new XmlCsvReader(sr, new Uri(address), new NameTable()))
{
reader.ColumnNames = new string[] { "Symbol", "Name", "Price", "LastPrice" };
reader.RootName = "StockQuotes";
reader.RowName = "Quote";
doc = XDocument.Load(reader);
foreach (XElement e in new List <XElement>(doc.Root.Elements()))
{
e.Remove();
newQuotes.Root.Add(e);
}
}
}
_current = null;
XElement firstNode = doc.Root.FirstNode as XElement;
string text = firstNode != null ? firstNode.Value : null;
if (text == "exception")
{
AddError(text);
return(null);
}
else
{
return(doc);
}
}
}
public async Task <ActionResult> UploadQuestions()
{
this.CheckUser();
ViewBag.Message = "Upload";
if (!ViewBag.IsAdmin)
{
ViewBag.Message = "You do not have admin permission";
}
else
{
int count = 0;
StringWriter log = new StringWriter();
foreach (string file in Request.Files)
{
HttpPostedFileBase upload = Request.Files.Get(file);
string category = Path.GetFileNameWithoutExtension(upload.FileName);
if (upload != null)
{
var context = this.HttpContext.GetOwinContext();
var db = context.Get <ApplicationDbContext>();
List <QuestionModel> newQuestions = new List <QuestionModel>();
using (XmlCsvReader reader = new XmlCsvReader(upload.InputStream, Encoding.UTF8,
new Uri(file, UriKind.Relative), new NameTable()))
{
reader.FirstRowHasColumnNames = false;
reader.ColumnNames = new string[] { "Question", "Answer", "Wrong1", "Wrong2", "Wrong3" };
reader.Delimiter = ',';
XDocument doc = XDocument.Load(reader);
foreach (XElement row in doc.Root.Elements())
{
var table = db.Config;
QuestionModel model = new QuestionModel()
{
Id = Guid.NewGuid(),
Category = category,
Question = GetChildValue(row, "Question"),
Answer = GetChildValue(row, "Answer"),
Wrong1 = GetChildValue(row, "Wrong1"),
Wrong2 = GetChildValue(row, "Wrong2"),
Wrong3 = GetChildValue(row, "Wrong3")
};
if (model.Question == "Question" &&
model.Answer == "Answer")
{
// skip it, this is the row header
}
else
{
newQuestions.Add(model);
}
}
}
// remove the old questions in this category
var oldQuestions = (from q in db.Questions where q.Category == category select q);
int oldCount = oldQuestions.Count();
db.Questions.RemoveRange(oldQuestions);
// add the new questions.
db.Questions.AddRange(newQuestions);
count = newQuestions.Count;
// update the database!
await db.SaveChangesAsync();
if (oldCount > 0)
{
if (count == 0)
{
ViewBag.Message = "Deleted " + oldCount + " questions in the " + category + " category";
}
else
{
ViewBag.Message = "Replaced " + oldCount + " old questions with " + count + " new questions in the " + category + " category";
}
}
else
{
ViewBag.Message = "Added " + count + " questions in the " + category + " category";
}
}
}
}
return(View("Index"));
}
public async Task Load(string fileName, ProgressUtility progress)
{
flights.Clear();
// CSV doesn't have realtime clock, so go with the file date instead.
this.startTime = File.GetLastWriteTime(fileName);
// time (us)
int min = int.MaxValue;
int max = int.MinValue;
await Task.Run(() =>
{
using (Stream s = File.OpenRead(fileName))
{
XmlNameTable nametable = new NameTable();
using (XmlCsvReader reader = new XmlCsvReader(s, System.Text.Encoding.UTF8, new Uri(fileName), nametable))
{
progress.ShowProgress(0, s.Length, s.Position);
reader.FirstRowHasColumnNames = true;
data = XDocument.Load(reader);
this.schema = new LogItemSchema()
{
Name = "CsvDataLog", Type = "Root"
};
// create the schema
List <LogItemSchema> children = new List <Model.LogItemSchema>();
foreach (String name in reader.ColumnNames)
{
children.Add(new LogItemSchema()
{
Name = name, Parent = this.schema
});
}
this.schema.ChildItems = children;
progress.ShowProgress(0, s.Length, s.Position);
}
}
foreach (var e in data.Root.Elements())
{
int?i = GetTimeMicroseconds(e);
if (i.HasValue)
{
if (i.Value < min)
{
min = i.Value;
}
if (i > max)
{
max = i.Value;
}
}
}
});
// this log has no absolute UTC time, only ticks since board was booted, so we make up a start time.
DateTime end = this.startTime.AddMilliseconds((max - min) / 1000);
var flight = new Flight()
{
Log = this, StartTime = this.startTime, Duration = end - this.startTime
};
this.duration = end - this.startTime;
this.flights.Add(flight);
}
public async Task Load(string fileName, ProgressUtility progress)
{
flights.Clear();
// CSV doesn't have realtime clock, so go with the file date instead.
this.startTime = File.GetLastWriteTime(fileName);
// time (ms)
long min = long.MaxValue;
long max = long.MinValue;
await Task.Run(() =>
{
timeElementName = null;
using (Stream s = File.OpenRead(fileName))
{
Dictionary <string, LogItemSchema> map = new Dictionary <string, LogItemSchema>();
XmlNameTable nametable = new NameTable();
using (XmlCsvReader reader = new XmlCsvReader(s, System.Text.Encoding.UTF8, new Uri(fileName), nametable))
{
reader.FirstRowHasColumnNames = true;
reader.ColumnsAsAttributes = true;
while (reader.Read())
{
progress.ShowProgress(0, s.Length, s.Position);
if (this.schema == null)
{
// create the schema
this.schema = new LogItemSchema()
{
Name = "CsvLog", Type = "Root"
};
LogItemSchema row = null;
foreach (String name in reader.ColumnNames)
{
if (timeElementName == null && (name.ToLower().Contains("time") || name.ToLower().Contains("ticks")))
{
timeElementName = name;
}
if (name.Contains(":"))
{
// then we have sub-parts.
int pos = name.IndexOf(":");
string key = name.Substring(0, pos);
string field = name.Substring(pos + 1);
LogItemSchema group = null;
if (!map.ContainsKey(key))
{
group = new LogItemSchema()
{
Name = key, Type = key
};
this.schema.AddChild(group);
map[key] = group;
}
else
{
group = map[key];
}
var leaf = new LogItemSchema()
{
Name = field, Type = "Double"
};
group.AddChild(leaf);
map[name] = leaf;
}
else
{
if (row == null)
{
row = new LogItemSchema()
{
Name = "Other", Type = "Other"
};
this.schema.AddChild(row);
}
var leaf = new LogItemSchema()
{
Name = name, Type = "Double"
};
row.AddChild(leaf);
map[name] = leaf;
}
}
}
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "row")
{
// read a row
long time = GetTicks(reader);
min = Math.Min(min, time);
max = Math.Max(max, time);
LogEntry row = new Model.LogEntry()
{
Name = "Other", Timestamp = (ulong)time
};
log.Add(row);
Dictionary <string, LogEntry> groups = new Dictionary <string, LogEntry>();
if (reader.MoveToFirstAttribute())
{
do
{
string name = XmlConvert.DecodeName(reader.LocalName);
LogItemSchema itemSchema = map[name];
LogEntry e = row;
if (name.Contains(":"))
{
// then we have sub-parts.
int pos = name.IndexOf(":");
string key = name.Substring(0, pos);
string field = name.Substring(pos + 1);
if (!groups.ContainsKey(key))
{
e = new LogEntry()
{
Name = key, Timestamp = (ulong)time
};
groups[key] = e;
log.Add(e);
}
else
{
e = groups[key];
}
name = field;
}
string value = reader.Value;
double d = 0;
if (double.TryParse(value, out d))
{
e.SetField(name, d);
}
else
{
if (!string.IsNullOrEmpty(value))
{
// not a number.
itemSchema.Type = "String";
e.SetField(name, value);
}
}
}while (reader.MoveToNextAttribute());
reader.MoveToElement();
}
}
}
}
}
});
// this log has no absolute UTC time, only ticks since board was booted, so we make up a start time.
DateTime end = this.startTime.AddMilliseconds((max - min) / 1000);
var flight = new Flight()
{
Log = this, StartTime = this.startTime, Duration = end - this.startTime
};
this.duration = end - this.startTime;
this.flights.Add(flight);
}
void Munge(string filename)
{
filename = System.IO.Path.GetFullPath(filename);
XDocument doc = null;
using (StreamReader reader = new StreamReader(filename, Encoding.GetEncoding("Windows-1252")))
{
using (XmlCsvReader xmlReader = new XmlCsvReader(reader, new Uri(filename), new System.Xml.NameTable()))
{
xmlReader.FirstRowHasColumnNames = true;
doc = XDocument.Load(xmlReader);
foreach (var row in doc.Root.Elements())
{
List <string> wrong = new List <string>();
string answer = null;
string question = null;
XElement toRemove = null;
foreach (var item in row.Elements())
{
if (item.Name.LocalName == "Question")
{
question = item.Value;
}
else if (item.Name.LocalName == "Answer")
{
answer = item.Value;
}
else if (answer != null)
{
if (item.Value.Trim() == answer.Trim())
{
toRemove = item;
}
else
{
wrong.Add(item.Value);
}
}
}
if (toRemove == null)
{
Console.WriteLine("Did not find right answer for question: {0}", question);
}
else
{
toRemove.Remove();
}
if (wrong.Count != 3)
{
Console.WriteLine("Question '{0}' has {1} instead of 3 wrong answers", question, wrong.Count);
}
}
// ok, now save the updated csv file
}
}
string newName = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(filename), System.IO.Path.GetFileNameWithoutExtension(filename) + "_2.csv");
using (StreamWriter writer = new StreamWriter(newName, false, Encoding.UTF8))
{
writer.WriteLine("Question,Answer,Wrong1,Wrong2,Wrong3");
foreach (var row in doc.Root.Elements())
{
List <XElement> cols = new List <XElement>(row.Elements());
if (cols.Count == 5)
{
writer.WriteLine("{0},{1},{2},{3},{4}", cols[0].Value, cols[1].Value, cols[2].Value, cols[3].Value, cols[4].Value);
}
else
{
Console.WriteLine("Row has wrong number of columns, {0}", cols.Count);
}
}
}
}
