Example #1
0
 public static void AddFlushQue(DownloadDumpFilesInfo fileInfo, LinkedList <HttpTransaction> httpTransactions)
 {
     lock (queLock)
     {
         if (que.ContainsKey(fileInfo) == false)
         {
             que.Add(fileInfo, httpTransactions);
         }
     }
 }
Example #2
0
        public override bool IsDataExists(ProcessedDataPackage state)
        {
            DownloadDumpFilesInfo ddfi = new DownloadDumpFilesInfo(state);

            if (ddfi.Exist() == false)
            {
                return(false);
            }

            try{
                FileInfo fi = new FileInfo(ddfi.GetFullPath());
                return(fi.Exists && fi.Length > 0);
            }
            catch
            {
            }
            return(false);
        }
Example #3
0
        public static void FlushTracker(DownloadDumpFilesInfo fileInfo, String fromUrl)
        {
            lock (httpTransactions)
            {
                if (isTracking == false)
                {
                    return;
                }

                isTracking = false;

                if (httpTransactions.Count <= 0)
                {
                    return;
                }

                HttpTransaction t = null;

                while (httpTransactions.Count > 0)
                {
                    t = httpTransactions.First.Value;

                    if (t.URL != null && t.URL.Equals(fromUrl))
                    {
                        break;
                    }

                    httpTransactions.RemoveFirst();
                }
                foreach (HttpTransaction tt in httpTransactions)
                {
                    if (URLMasks.ContainsKey(tt.URL))
                    {
                        tt.URL = URLMasks[tt.URL];
                    }
                }
                if (httpTransactions.Count > 0)
                {
                    HttpFlushPipe.AddFlushQue(fileInfo, new LinkedList <HttpTransaction>(httpTransactions));
                }

                httpTransactions.Clear();
            }
        }
Example #4
0
        public override void SendData(byte[] buffer, int offset, int length)
        {
            if (this.PipesChain.ChainState.ContainsKey(HttpTracerPipe.STATE_KEY))
            {
                ((HttpTransaction)this.PipesChain.ChainState[HttpTracerPipe.STATE_KEY]).IsTrackable = false;
            }

            if (this.PipesChain.ChainState.ContainsKey("REQUEST_URI"))
            {
                String url = (String)this.PipesChain.ChainState["REQUEST_URI"];

                bool   isStart     = false;
                String originalUrl = "";

                Match m = parseQueryString.Match(url);

                if (m.Success)
                {
                    try
                    {
                        originalUrl = base64Decode(m.Groups[3].Value);
                        isStart     = m.Groups[1].Value == "START";

                        if (isStart)
                        {
                            if (m.Groups.Count >= 4 && this.Configuration is EngineSuProxyConfiguration)
                            {
                                HttpTracerPipe.AddURLMask(originalUrl, ((EngineSuProxyConfiguration)this.Configuration).URL);
                            }
                            HttpTracerPipe.StartTracking();
                        }
                        else if (this.Configuration is EngineSuProxyConfiguration)
                        {
                            DownloadDumpFilesInfo ddfi = new DownloadDumpFilesInfo((EngineSuProxyConfiguration)this.Configuration);
                            HttpTracerPipe.FlushTracker(ddfi, originalUrl);
                        }
                    }
                    catch
                    {
                    }
                }
            }
        }
Example #5
0
        public override DownloadData GetProcessedData(ProcessedDataPackage state)
        {
            DownloadDumpFilesInfo ddfi = new DownloadDumpFilesInfo(state);

            Stream file = ddfi.Open(FileAccess.Read);

            DownloadData processedData = new DownloadData();

            if (file == null)
            {
                return(processedData);
            }

            StreamReader source = new StreamReader(file);
            String       buffer = source.ReadToEnd();

            source.Close();
            source.Dispose();

            if (buffer == null)
            {
                return(null);
            }

            processedData.DownloadDataDumpFilename = ddfi.GetFilename();

            DownloadState downloadState = null;


            MatchCollection matchs = null;

            matchs = download_Dump_Pattern.Matches(buffer);

            Match   match = null;
            URLType type  = URLType.Unknown;

            processedData.TotalFiles = matchs.Count;

            for (int i = 0; i < matchs.Count; i++)
            {
                match = matchs[i];

                try
                {
                    downloadState                            = new DownloadState();
                    downloadState.FileGUID                   = match.Groups[1].ToString();
                    downloadState.ConnectionStartTime        = Math.Max(-1, long.Parse(match.Groups[2].ToString()));
                    downloadState.SendingRequestStartTime    = Math.Max(-1, long.Parse(match.Groups[3].ToString()));
                    downloadState.SendingRequestEndTime      = Math.Max(-1, long.Parse(match.Groups[4].ToString()));
                    downloadState.ReceivingResponseStartTime = Math.Max(-1, long.Parse(match.Groups[5].ToString()));
                    downloadState.ReceivingResponseEndTime   = Math.Max(-1, long.Parse(match.Groups[6].ToString()));
                    downloadState.ConnectionEndTime          = Math.Max(-1, long.Parse(match.Groups[7].ToString()));
                    downloadState.TotalSent                  = Math.Max(-1, int.Parse(match.Groups[8].ToString()));
                    downloadState.TotalReceived              = Math.Max(-1, int.Parse(match.Groups[9].ToString()));

                    processedData.TotalDataReceived += downloadState.TotalReceived;
                    processedData.TotalDataSent     += downloadState.TotalSent;

                    if (state != null)
                    {
                        if (downloadState.ConnectionStartTime > 0)
                        {
                            state.CollectionStartTime = Math.Min(state.CollectionStartTime, downloadState.ConnectionStartTime);
                        }
                        state.CollectionEndTime = Math.Max(state.CollectionEndTime, downloadState.ConnectionEndTime);
                    }

                    downloadState.URL = match.Groups[10].ToString();

                    processedData.AddLast(downloadState);

                    type = GetURLType(downloadState.URL);

                    downloadState.URLType = type;

                    if (type == URLType.CSS)
                    {
                        processedData.TotalCSS++;
                        processedData.TotalCSSWeight += downloadState.TotalReceived;
                    }
                    else if (type == URLType.Image)
                    {
                        processedData.TotalImages++;
                        processedData.TotalImagesWeight += downloadState.TotalReceived;
                    }
                    else if (type == URLType.JS)
                    {
                        processedData.TotalJS++;
                        processedData.TotalJSWeight += downloadState.TotalReceived;
                    }
                }
                catch
                {
                }
            }

            return(processedData);
        }