private void MainForm_DragDrop(object sender, DragEventArgs e)
{
try
{
string[] fileDrop = (string[])e.Data.GetData(DataFormats.FileDrop);
if (null != fileDrop)
{
string filename = fileDrop.GetValue(0).ToString();
UsageDataMessage currentMessage = FileImporter.ReadMessage(filename);
using (StringWriter w = new StringWriter())
{
using (XmlTextWriter xmlWriter = new XmlTextWriter(w))
{
xmlWriter.Formatting = Formatting.Indented;
DataContractSerializer serializer = new DataContractSerializer(typeof(UsageDataMessage));
serializer.WriteObject(xmlWriter, currentMessage);
}
FileContents.Text = w.ToString();
}
}
}
catch (Exception ex)
{
FileContents.Text = ex.ToString();
}
}
// list directory
// etl each file separately
// load file
// match all four types & insert (if necessary); match in memory (performance)
// match user & insert (if necessary); via sproc (Cache issue)
// load session and values into database
// delete file when done & no exceptions
public static void ImportMessagesFromDirectory(string directory, bool KeepFile)
{
using (var context = CollectorRepository.CreateContext())
{
CollectorRepository repo = new CollectorRepository();
repo.Context = context;
foreach (string filename in Directory.EnumerateFiles(directory, "*.xml.gz", SearchOption.TopDirectoryOnly))
{
UsageDataMessage message = null;
try
{
message = FileImporter.ReadMessage(filename);
}
catch (System.Exception ex)
{
Console.WriteLine("Failed to read file {0}, exception {1}", filename, ex.Message);
continue;
}
StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo);
processor.ProcessMessage();
}
}
}
public override bool Execute()
{
bool result = true;
if ((null == messagesToScrub) || (0 == messagesToScrub.Length))
{
return(true);
}
// Scrub files: fix schema, remove PII from files (applies to debug builds)
// Necessary only for older SharpDevelop-only message files, not official builds
foreach (ITaskItem msgFilename in messagesToScrub)
{
string filename = msgFilename.ItemSpec;
try
{
bool isModified;
UsageDataMessage message = ReadMessage(filename, out isModified);
if (null == message)
{
Log.LogWarning("Error reading {0}", filename);
continue;
}
bool hasShownMessageForThisFile = false;
foreach (UsageDataException exception in message.Sessions.SelectMany(s => s.Exceptions))
{
if (stacktraceRegex.IsMatch(exception.StackTrace))
{
// Old SharpDevelop versions could upload stack trace information including full paths, which might
// contain private data (e.g. C:\Users\USERNAME). We'll remove full paths from stack traces.
// Newer UDC versions do not transmit paths anymore.
exception.StackTrace = stacktraceRegex.Replace(exception.StackTrace, "");
isModified = true;
if (!hasShownMessageForThisFile)
{
Log.LogMessage("Removing potentially private information from " + filename);
hasShownMessageForThisFile = true;
}
}
}
if (isModified)
{
SaveMessage(filename, message);
}
}
catch (Exception ex)
{
Log.LogErrorFromException(ex);
continue;
}
}
return(result);
}
public static void ImportSingleMessage(string filename)
{
UsageDataMessage message = FileImporter.ReadMessage(filename);
using (var context = CollectorRepository.CreateContext())
{
CollectorRepository repo = new CollectorRepository();
repo.Context = context;
StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo);
processor.ProcessMessage();
}
}
private void SaveMessage(string filename, UsageDataMessage message)
{
DateTime fileDate = File.GetLastWriteTimeUtc(filename);
using (FileStream fs = File.Open(filename, FileMode.Create, FileAccess.Write, FileShare.None))
{
using (GZipStream zip = new GZipStream(fs, CompressionMode.Compress))
{
DataContractSerializer serializer = new DataContractSerializer(typeof(UsageDataMessage));
serializer.WriteObject(zip, message);
}
}
File.SetLastAccessTimeUtc(filename, fileDate);
}
public override bool Execute()
{
bool result = true;
if ((null == messagesToImport) || (0 == messagesToImport.Length))
{
// messagesFailedToImport is already newed as an empty list
return(true);
}
if (null == connectionString)
{
Log.LogError("Connection String for ADO.NET EF cannot be empty");
return(false);
}
// Import logic built for MSBuild
Stopwatch watch = Stopwatch.StartNew();
using (var context = CollectorRepository.CreateContext(connectionString.ItemSpec))
{
CollectorRepository repo = new CollectorRepository();
repo.Context = context;
foreach (ITaskItem msgFilename in messagesToImport)
{
UsageDataMessage message = null;
try
{
message = FileImporter.ReadMessage(msgFilename.ItemSpec);
}
catch (System.Exception ex)
{
Log.LogErrorFromException(ex);
messagesFailedToImport.Add(new TaskItem(msgFilename.ItemSpec));
continue;
}
if (ignoreSessionsTooFarFromFileDate)
{
// Acceptable sessions are between 1.5 months old and 3 days into the future.
// All other sessions indicate a horribly wrong system time on the user's machine and will be ignored.
DateTime fileDate = System.IO.File.GetLastWriteTimeUtc(msgFilename.ItemSpec);
message.Sessions.RemoveAll(s => s.StartTime <fileDate.AddDays(-45) || s.StartTime> fileDate.AddDays(3));
}
try
{
StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo);
processor.ProcessMessage();
}
catch (System.Exception ex)
{
Log.LogErrorFromException(ex);
messagesFailedToImport.Add(new TaskItem(msgFilename.ItemSpec));
}
}
}
watch.Stop();
Log.LogMessage("Imported " + (messagesToImport.Length - messagesFailedToImport.Count) + " messages in " + watch.Elapsed);
return(result);
}
public StoreMessageInSqlServer(UsageDataMessage msg, CollectorRepository repo)
{
message = msg;
repository = repo;
}
static UsageDataMessage FetchDataForUpload(SQLiteConnection connection, bool fetchIncompleteSessions)
{
UsageDataMessage message = new UsageDataMessage();
// Retrieve the User ID
using (SQLiteCommand cmd = connection.CreateCommand()) {
cmd.CommandText = "SELECT value FROM Properties WHERE name = 'userID';";
string userID = (string)cmd.ExecuteScalar();
message.UserID = new Guid(userID);
}
// Retrieve the list of sessions
using (SQLiteCommand cmd = connection.CreateCommand()) {
if (fetchIncompleteSessions)
{
cmd.CommandText = @"SELECT id, startTime, endTime FROM Sessions;";
}
else
{
// Fetch all sessions which are either closed or inactive for more than 24 hours
cmd.CommandText = @"SELECT id, startTime, endTime FROM Sessions
WHERE (endTime IS NOT NULL)
OR (ifnull((SELECT max(time) FROM FeatureUses WHERE FeatureUses.session = Sessions.id), Sessions.startTime)
< datetime('now','-1 day'));" ;
}
using (SQLiteDataReader reader = cmd.ExecuteReader()) {
while (reader.Read())
{
UsageDataSession session = new UsageDataSession();
session.SessionID = reader.GetInt64(0);
session.StartTime = reader.GetDateTime(1);
if (!reader.IsDBNull(2))
{
session.EndTime = reader.GetDateTime(2);
}
message.Sessions.Add(session);
}
}
}
string commaSeparatedSessionIDList = GetCommaSeparatedIDList(message.Sessions);
StringInterner stringInterning = new StringInterner();
// Retrieve the environment
using (SQLiteCommand cmd = connection.CreateCommand()) {
cmd.CommandText = "SELECT session, name, value FROM Environment WHERE session IN (" + commaSeparatedSessionIDList + ");";
using (SQLiteDataReader reader = cmd.ExecuteReader()) {
while (reader.Read())
{
long sessionID = reader.GetInt64(0);
UsageDataSession session = message.FindSession(sessionID);
session.EnvironmentProperties.Add(
new UsageDataEnvironmentProperty {
Name = stringInterning.Intern(reader.GetString(1)),
Value = stringInterning.Intern(reader.GetString(2))
});
}
}
}
// Retrieve the feature uses
using (SQLiteCommand cmd = connection.CreateCommand()) {
cmd.CommandText = "SELECT session, time, endTime, feature, activationMethod FROM FeatureUses WHERE session IN (" + commaSeparatedSessionIDList + ");";
using (SQLiteDataReader reader = cmd.ExecuteReader()) {
while (reader.Read())
{
long sessionID = reader.GetInt64(0);
UsageDataSession session = message.FindSession(sessionID);
UsageDataFeatureUse featureUse = new UsageDataFeatureUse();
featureUse.Time = reader.GetDateTime(1);
if (!reader.IsDBNull(2))
{
featureUse.EndTime = reader.GetDateTime(2);
}
featureUse.FeatureName = stringInterning.Intern(reader.GetString(3));
if (!reader.IsDBNull(4))
{
featureUse.ActivationMethod = stringInterning.Intern(reader.GetString(4));
}
session.FeatureUses.Add(featureUse);
}
}
}
// Retrieve the exceptions
using (SQLiteCommand cmd = connection.CreateCommand()) {
cmd.CommandText = "SELECT session, time, type, stackTrace FROM Exceptions WHERE session IN (" + commaSeparatedSessionIDList + ");";
using (SQLiteDataReader reader = cmd.ExecuteReader()) {
while (reader.Read())
{
long sessionID = reader.GetInt64(0);
UsageDataSession session = message.FindSession(sessionID);
UsageDataException exception = new UsageDataException();
exception.Time = reader.GetDateTime(1);
exception.ExceptionType = stringInterning.Intern(reader.GetString(2));
if (!reader.IsDBNull(3))
{
exception.StackTrace = stringInterning.Intern(reader.GetString(3));
}
session.Exceptions.Add(exception);
}
}
}
return(message);
}