private Tuple <int> AggregeteResults(Tuple <string, int, int> tuple)
        {
            string tmpFile         = tuple.Item1;
            int    partIngestCount = tuple.Item2;
            int    index           = tuple.Item3;

            lock (_lockTaskWriter)
            {
                File.AppendAllText(workingFile, File.ReadAllText(tmpFile));
                File.Delete(tmpFile);
                //CosmosStream.UploadStream(serverFile, workingFile);
                ingestedCount += partIngestCount;
                ImageLogger.LogMessage(this.log, EventType.Information, "[ImageProcess] - Image Process in progress [{0}/{1}]", ingestedCount, listTask.Count);

                if (DateTime.Now - this.lastTime > this.reportInterval)
                {
                    this.WriteLog(
                        EventType.Information,
                        string.Format(
                            "[ImageProcess]: In Progress ... [{0}/{1}]",
                            ingestedCount,
                            listTask.Count));
                    lastTime = DateTime.Now;
                }
            }
            return(new Tuple <int>(index));
        }
        public void Start(int threadCount)
        {
            this.processFailureBuffer = this.failureBuffer;
            ingestedCount             = 0;
            this.lastTime             = DateTime.Now;

            ImageProcessHelper.SplitIntoBuckets(listTask, ingestBurstCount)
            .AsParallel()
            .WithDegreeOfParallelism(threadCount)
            .Select(DownloadImage)
            .Select(ScaleImage)
            .Select(IngestImage)
            .Select(AggregeteResults)
            .ToList();

            if (!Result)
            {
                ImageLogger.LogMessage(this.log, EventType.Warning, "Image process failure buffer exhausted!");
            }
            else
            {
                ImageLogger.LogMessage(this.log, EventType.Information, "[ImageProcess] - Image Process Done [{0}/{1}]",
                                       ingestedCount, listTask.Count);
            }
        }
Esempio n. 3
0
        public static bool ProcessImage(string imageUrlListFilePath, string imageDownloadFolder, string imageScaleFolder, string imageUrlMappingFile, string serverFile, TraceLog log, Action <EventType, string> WriteLog)
        {
            HashSet <string> folderFiles = new HashSet <string>();

            if (File.Exists(imageUrlMappingFile))
            {
                folderFiles = new HashSet <string>(File.ReadLines(imageUrlMappingFile).Select(l => GetHashFileName(l)));
            }

            // read Image URl list file to get process task
            ImageWorkingFile iwf     = new ImageWorkingFile(imageUrlListFilePath, log);
            List <Image>     listAll = iwf.Read();

            ImageLogger.LogMessage(log, EventType.Information, "[ImageProcess] - Total {0} images in list.", listAll.Count);

            // assign local path name to each myImage
            List <Image> listNew = AssignLocalPathName(listAll, imageDownloadFolder, imageScaleFolder, folderFiles);

            ImageLogger.LogMessage(log, EventType.Information, "[ImageProcess] - Total {0} new images need to process.", listNew.Count);
            WriteLog(
                EventType.Information,
                string.Format("[ImageProcess] - Total {0} new images need to process.", listNew.Count));

            int    failureBuffer    = Math.Max(listAll.Count * 3 / 100, 50); // 3% or 50 failure buffer
            int    threadCount      = 10;                                    // ConfigStore.Instance.OsClientConfig.IngestThreadCount;
            int    ingestBurstCount = 10;                                    // ConfigStore.Instance.OsClientConfig.IngestBurstCount;
            string proxy            = "bjsproxy";

            if (listNew.Count >= 1000000)
            {
                ImageLogger.LogMessage(log, EventType.Error, "[ImageProcess] - Too many images (1+ million) need to process, will casue image table full.");
                return(false);
            }
            else if (listNew.Count > 0)
            {
                System.Net.ServicePointManager.DefaultConnectionLimit = 5120;
                ImageProcessThread thread = new ImageProcessThread(log, WriteLog);
                thread.AssignTaskList(listNew, imageUrlMappingFile, serverFile, proxy, failureBuffer, ingestBurstCount);
                thread.Start(threadCount);

                if (!thread.Result)
                {
                    return(false);
                }
            }
            else
            {
                ImageLogger.LogMessage(log, EventType.Information, "[ImageProcess] - No new images to process.");
            }

            return(true);
        }
        private Tuple <List <Image>, int> ScaleImage(Tuple <List <Image>, int> tuple)
        {
            List <Image> list       = tuple.Item1;
            int          index      = tuple.Item2;
            List <Image> scaledList = new List <Image>();

            foreach (var image in list)
            {
                try
                {
                    lock (_failBufferLock)
                    {
                        if (!Result)
                        {
                            break;
                        }
                    }

                    bool bSuccess = image.ScaleDefault();
                    if (!bSuccess)
                    {
                        continue;
                    }

                    //image.ExtractAccentColor(log);
                    scaledList.Add(image);

                    lock (_failBufferLock)
                    {
                        // process succeed, reset failure buffer
                        this.processFailureBuffer = this.failureBuffer;
                    }
                }
                catch (Exception ex)
                {
                    ImageLogger.LogMessage(this.log, EventType.Error, ex.Message);
                    ImageLogger.LogMessage(this.log, EventType.Warning, "Exception when scale image {0}", image.SourceUrl);
                    lock (_failBufferLock)
                    {
                        this.processFailureBuffer -= 1;
                        if (this.processFailureBuffer < 0)
                        {
                            Result = false;
                        }
                    }
                    continue;
                }
            }

            return(new Tuple <List <Image>, int>(scaledList, index));
        }
Esempio n. 5
0
        public List <Image> Read()
        {
            List <Image> list = new List <Image>();

            if (!File.Exists(this.FileName))
            {
                return(list);
            }

            HashSet <string> unique = new HashSet <string>();

            using (StreamReader sr = new StreamReader(this.FileName, true))
            {
                while (!sr.EndOfStream)
                {
                    string line  = sr.ReadLine();
                    Image  image = null;

                    try
                    {
                        image = Image.FromString(line.Trim());
                    }
                    catch (Exception ex)
                    {
                        ImageLogger.LogMessage(this.log, EventType.Error, ex.Message);
                        ImageLogger.LogMessage(this.log, EventType.Warning, "Invalid image line: {0}", line.Trim());
                        continue;
                    }

                    if (string.IsNullOrEmpty(image.SourceUrl))
                    {
                        ImageLogger.LogMessage(this.log, EventType.Warning, "Invalid image line: {0}", line.Trim());
                        continue;
                    }
                    if (!unique.Contains(image.SourceUrl))  // dedup
                    {
                        list.Add(image);
                        unique.Add(image.SourceUrl);
                    }
                }
            }
            return(list);
        }
        private Tuple <string, int, int> IngestImage(Tuple <List <Image>, int> tuple)
        {
            List <Image> list              = tuple.Item1;
            int          index             = tuple.Item2;
            int          partIngestedCount = 0;

            string tmpFile = string.Format("{0}.part{1}", workingFile, index);

            using (StreamWriter sw = new StreamWriter(tmpFile, true))
            {
                string feed = string.Empty;
                List <IDataLoadResult> results = null;

                try
                {
                    using (DataLoader loader = new DataLoader(OSContext.Instance.Configuration))
                    {
                        for (int i = 0; i < list.Count; ++i)
                        {
                            Image  myImage   = list[i];
                            byte[] data      = ScaledImageToBinary(myImage);
                            string hashValue = HttpUrlHash.GetHashValueString(myImage.SourceUrl);
                            results            = ImageIngester.Ingest(hashValue, data, myImage, loader);
                            partIngestedCount += LogIngestResults(results, sw);
                        }

                        loader.Flush();
                        results            = loader.Receive(true);
                        partIngestedCount += LogIngestResults(results, sw);
                    }
                }
                catch (Exception ex)
                {
                    ImageLogger.LogMessage(this.log, EventType.Error, "[ImageIngest][{0}][T{1}]: {2}", feed, index, ex.Message);
                    ImageLogger.LogMessage(this.log, EventType.Error, ex.StackTrace);
                }
            }

            return(new Tuple <string, int, int>(tmpFile, partIngestedCount, index));
        }
        private int LogIngestResults(List <IDataLoadResult> results, StreamWriter sw)
        {
            int succeedCount = 0;

            foreach (IDataLoadResult result in results)
            {
                Image  myImage = result.Context as Image;
                string key, feed, scenario;
                scenario = ExtractImageContext(myImage, out feed, out key);

                if (result.IsSuccessful)
                {
                    sw.WriteLine(myImage.ToString());
                    ++succeedCount;
                }
                else
                {
                    ImageLogger.LogMessage(this.log, EventType.Warning, "[ImageIngest][{0}]: Ingest failed to {1} locations: {2}", scenario, result.FailedLocations.Count, String.Join(", ", result.FailedLocations));
                }
            }
            return(succeedCount);
        }
        private bool DownloadImage(Image myImage)
        {
            BaseCrawler crawler = null;

            string sourceUrl = myImage.SourceUrl.Trim();

            if (sourceUrl.StartsWith(@"https://cosmos", StringComparison.InvariantCultureIgnoreCase)) // cosmos path
            {
                crawler = new CosmosCrawler(FeedUriTypes.ShareFolder, sourceUrl,
                                            myImage.DownloadImageName, DateTime.MinValue);
            }
            else if (sourceUrl.StartsWith("http://") ||
                     sourceUrl.StartsWith("https://"))
            {
                ProxyType proxyType;
                if (!Enum.TryParse <ProxyType>(proxy, true, out proxyType))
                {
                    proxyType = ProxyType.NULL;
                }
                crawler = new HttpCrawler(FeedUriTypes.Http, sourceUrl, myImage.DownloadImageName, DateTime.MinValue, null, proxyType);
            }
            else // wrong image URI
            {
                ImageLogger.LogMessage(this.log, EventType.Warning, "Cannot identify this image's URI: {0}",
                                       sourceUrl);
                return(false);
            }

            if (crawler != null)
            {
                if (crawler.Crawl() != BuildResults.Crawler_Succeed)
                {
                    ImageLogger.LogMessage(this.log, EventType.Warning, "Exception when download image {0}", sourceUrl);
                    return(false);
                }
            }
            return(true);
        }