예제 #1
0
        private void MainForm_DragDrop(object sender, DragEventArgs e)
        {
            try
            {
                string[] fileDrop = (string[])e.Data.GetData(DataFormats.FileDrop);

                if (null != fileDrop)
                {
                    string filename = fileDrop.GetValue(0).ToString();

                    UsageDataMessage currentMessage = FileImporter.ReadMessage(filename);

                    using (StringWriter w = new StringWriter())
                    {
                        using (XmlTextWriter xmlWriter = new XmlTextWriter(w))
                        {
                            xmlWriter.Formatting = Formatting.Indented;
                            DataContractSerializer serializer = new DataContractSerializer(typeof(UsageDataMessage));
                            serializer.WriteObject(xmlWriter, currentMessage);
                        }
                        FileContents.Text = w.ToString();
                    }
                }
            }
            catch (Exception ex)
            {
                FileContents.Text = ex.ToString();
            }
        }
        // list directory
        // etl each file separately
        // load file
        // match all four types & insert (if necessary); match in memory (performance)
        // match user & insert (if necessary); via sproc (Cache issue)
        // load session and values into database
        // delete file when done & no exceptions
        public static void ImportMessagesFromDirectory(string directory, bool KeepFile)
        {
            using (var context = CollectorRepository.CreateContext())
            {
                CollectorRepository repo = new CollectorRepository();
                repo.Context = context;

                foreach (string filename in Directory.EnumerateFiles(directory, "*.xml.gz", SearchOption.TopDirectoryOnly))
                {
                    UsageDataMessage message = null;
                    try
                    {
                        message = FileImporter.ReadMessage(filename);
                    }
                    catch (System.Exception ex)
                    {
                        Console.WriteLine("Failed to read file {0}, exception {1}", filename, ex.Message);
                        continue;
                    }

                    StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo);
                    processor.ProcessMessage();
                }
            }
        }
        public override bool Execute()
        {
            bool result = true;

            if ((null == messagesToScrub) || (0 == messagesToScrub.Length))
            {
                return(true);
            }

            // Scrub files: fix schema, remove PII from files (applies to debug builds)
            // Necessary only for older SharpDevelop-only message files, not official builds
            foreach (ITaskItem msgFilename in messagesToScrub)
            {
                string filename = msgFilename.ItemSpec;
                try
                {
                    bool             isModified;
                    UsageDataMessage message = ReadMessage(filename, out isModified);

                    if (null == message)
                    {
                        Log.LogWarning("Error reading {0}", filename);
                        continue;
                    }

                    bool hasShownMessageForThisFile = false;
                    foreach (UsageDataException exception in message.Sessions.SelectMany(s => s.Exceptions))
                    {
                        if (stacktraceRegex.IsMatch(exception.StackTrace))
                        {
                            // Old SharpDevelop versions could upload stack trace information including full paths, which might
                            // contain private data (e.g. C:\Users\USERNAME). We'll remove full paths from stack traces.
                            // Newer UDC versions do not transmit paths anymore.
                            exception.StackTrace = stacktraceRegex.Replace(exception.StackTrace, "");
                            isModified           = true;
                            if (!hasShownMessageForThisFile)
                            {
                                Log.LogMessage("Removing potentially private information from " + filename);
                                hasShownMessageForThisFile = true;
                            }
                        }
                    }

                    if (isModified)
                    {
                        SaveMessage(filename, message);
                    }
                }
                catch (Exception ex)
                {
                    Log.LogErrorFromException(ex);
                    continue;
                }
            }

            return(result);
        }
        public static void ImportSingleMessage(string filename)
        {
            UsageDataMessage message = FileImporter.ReadMessage(filename);

            using (var context = CollectorRepository.CreateContext())
            {
                CollectorRepository repo = new CollectorRepository();
                repo.Context = context;

                StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo);
                processor.ProcessMessage();
            }
        }
        private void SaveMessage(string filename, UsageDataMessage message)
        {
            DateTime fileDate = File.GetLastWriteTimeUtc(filename);

            using (FileStream fs = File.Open(filename, FileMode.Create, FileAccess.Write, FileShare.None))
            {
                using (GZipStream zip = new GZipStream(fs, CompressionMode.Compress))
                {
                    DataContractSerializer serializer = new DataContractSerializer(typeof(UsageDataMessage));
                    serializer.WriteObject(zip, message);
                }
            }
            File.SetLastAccessTimeUtc(filename, fileDate);
        }
        public override bool Execute()
        {
            bool result = true;

            if ((null == messagesToImport) || (0 == messagesToImport.Length))
            {
                // messagesFailedToImport is already newed as an empty list
                return(true);
            }

            if (null == connectionString)
            {
                Log.LogError("Connection String for ADO.NET EF cannot be empty");
                return(false);
            }

            // Import logic built for MSBuild
            Stopwatch watch = Stopwatch.StartNew();

            using (var context = CollectorRepository.CreateContext(connectionString.ItemSpec))
            {
                CollectorRepository repo = new CollectorRepository();
                repo.Context = context;

                foreach (ITaskItem msgFilename in messagesToImport)
                {
                    UsageDataMessage message = null;

                    try
                    {
                        message = FileImporter.ReadMessage(msgFilename.ItemSpec);
                    }
                    catch (System.Exception ex)
                    {
                        Log.LogErrorFromException(ex);
                        messagesFailedToImport.Add(new TaskItem(msgFilename.ItemSpec));

                        continue;
                    }

                    if (ignoreSessionsTooFarFromFileDate)
                    {
                        // Acceptable sessions are between 1.5 months old and 3 days into the future.
                        // All other sessions indicate a horribly wrong system time on the user's machine and will be ignored.
                        DateTime fileDate = System.IO.File.GetLastWriteTimeUtc(msgFilename.ItemSpec);
                        message.Sessions.RemoveAll(s => s.StartTime <fileDate.AddDays(-45) || s.StartTime> fileDate.AddDays(3));
                    }

                    try
                    {
                        StoreMessageInSqlServer processor = new StoreMessageInSqlServer(message, repo);
                        processor.ProcessMessage();
                    }
                    catch (System.Exception ex)
                    {
                        Log.LogErrorFromException(ex);
                        messagesFailedToImport.Add(new TaskItem(msgFilename.ItemSpec));
                    }
                }
            }
            watch.Stop();
            Log.LogMessage("Imported " + (messagesToImport.Length - messagesFailedToImport.Count) + " messages in " + watch.Elapsed);

            return(result);
        }
 public StoreMessageInSqlServer(UsageDataMessage msg, CollectorRepository repo)
 {
     message    = msg;
     repository = repo;
 }
예제 #8
0
        static UsageDataMessage FetchDataForUpload(SQLiteConnection connection, bool fetchIncompleteSessions)
        {
            UsageDataMessage message = new UsageDataMessage();

            // Retrieve the User ID
            using (SQLiteCommand cmd = connection.CreateCommand()) {
                cmd.CommandText = "SELECT value FROM Properties WHERE name = 'userID';";
                string userID = (string)cmd.ExecuteScalar();
                message.UserID = new Guid(userID);
            }

            // Retrieve the list of sessions
            using (SQLiteCommand cmd = connection.CreateCommand()) {
                if (fetchIncompleteSessions)
                {
                    cmd.CommandText = @"SELECT id, startTime, endTime FROM Sessions;";
                }
                else
                {
                    // Fetch all sessions which are either closed or inactive for more than 24 hours
                    cmd.CommandText = @"SELECT id, startTime, endTime FROM Sessions
						WHERE (endTime IS NOT NULL)
							OR (ifnull((SELECT max(time) FROM FeatureUses WHERE FeatureUses.session = Sessions.id), Sessions.startTime)
								< datetime('now','-1 day'));"                                ;
                }
                using (SQLiteDataReader reader = cmd.ExecuteReader()) {
                    while (reader.Read())
                    {
                        UsageDataSession session = new UsageDataSession();
                        session.SessionID = reader.GetInt64(0);
                        session.StartTime = reader.GetDateTime(1);
                        if (!reader.IsDBNull(2))
                        {
                            session.EndTime = reader.GetDateTime(2);
                        }
                        message.Sessions.Add(session);
                    }
                }
            }
            string commaSeparatedSessionIDList = GetCommaSeparatedIDList(message.Sessions);

            StringInterner stringInterning = new StringInterner();

            // Retrieve the environment
            using (SQLiteCommand cmd = connection.CreateCommand()) {
                cmd.CommandText = "SELECT session, name, value FROM Environment WHERE session IN (" + commaSeparatedSessionIDList + ");";
                using (SQLiteDataReader reader = cmd.ExecuteReader()) {
                    while (reader.Read())
                    {
                        long             sessionID = reader.GetInt64(0);
                        UsageDataSession session   = message.FindSession(sessionID);
                        session.EnvironmentProperties.Add(
                            new UsageDataEnvironmentProperty {
                            Name  = stringInterning.Intern(reader.GetString(1)),
                            Value = stringInterning.Intern(reader.GetString(2))
                        });
                    }
                }
            }

            // Retrieve the feature uses
            using (SQLiteCommand cmd = connection.CreateCommand()) {
                cmd.CommandText = "SELECT session, time, endTime, feature, activationMethod FROM FeatureUses WHERE session IN (" + commaSeparatedSessionIDList + ");";
                using (SQLiteDataReader reader = cmd.ExecuteReader()) {
                    while (reader.Read())
                    {
                        long                sessionID  = reader.GetInt64(0);
                        UsageDataSession    session    = message.FindSession(sessionID);
                        UsageDataFeatureUse featureUse = new UsageDataFeatureUse();
                        featureUse.Time = reader.GetDateTime(1);
                        if (!reader.IsDBNull(2))
                        {
                            featureUse.EndTime = reader.GetDateTime(2);
                        }
                        featureUse.FeatureName = stringInterning.Intern(reader.GetString(3));
                        if (!reader.IsDBNull(4))
                        {
                            featureUse.ActivationMethod = stringInterning.Intern(reader.GetString(4));
                        }
                        session.FeatureUses.Add(featureUse);
                    }
                }
            }

            // Retrieve the exceptions
            using (SQLiteCommand cmd = connection.CreateCommand()) {
                cmd.CommandText = "SELECT session, time, type, stackTrace FROM Exceptions WHERE session IN (" + commaSeparatedSessionIDList + ");";
                using (SQLiteDataReader reader = cmd.ExecuteReader()) {
                    while (reader.Read())
                    {
                        long               sessionID = reader.GetInt64(0);
                        UsageDataSession   session   = message.FindSession(sessionID);
                        UsageDataException exception = new UsageDataException();
                        exception.Time          = reader.GetDateTime(1);
                        exception.ExceptionType = stringInterning.Intern(reader.GetString(2));
                        if (!reader.IsDBNull(3))
                        {
                            exception.StackTrace = stringInterning.Intern(reader.GetString(3));
                        }
                        session.Exceptions.Add(exception);
                    }
                }
            }

            return(message);
        }