Ejemplo n.º 1
0
        /// <summary>
        /// Returns the image based on its label and image selection method.
        /// </summary>
        /// <param name="state">Specifies the query state.</param>
        /// <param name="labelSelectionMethod">Specifies the label selection method.</param>
        /// <param name="imageSelectionMethod">Specifies the image selection method.</param>
        /// <param name="log">Specifies the Log for status output.</param>
        /// <param name="nLabel">Optionally, specifies the label (default = null).</param>
        /// <param name="nDirectIdx">Optionally, specifies the image index to use when loading a specific index (default = -1).</param>
        /// <param name="bLoadDataCriteria">Optionally, specifies to load the data criteria data (default = false).</param>
        /// <param name="bLoadDebugData">Optionally, specifies to load the debug data (default = false).</param>
        /// <returns>The SimpleDatum containing the image is returned.</returns>
        public SimpleDatum GetImage(QueryState state, IMGDB_LABEL_SELECTION_METHOD labelSelectionMethod, IMGDB_IMAGE_SELECTION_METHOD imageSelectionMethod, Log log, int?nLabel = null, int nDirectIdx = -1, bool bLoadDataCriteria = false, bool bLoadDebugData = false)
        {
            if ((imageSelectionMethod & IMGDB_IMAGE_SELECTION_METHOD.BOOST) == IMGDB_IMAGE_SELECTION_METHOD.BOOST &&
                (labelSelectionMethod & IMGDB_LABEL_SELECTION_METHOD.RANDOM) == IMGDB_LABEL_SELECTION_METHOD.RANDOM)
            {
                labelSelectionMethod |= IMGDB_LABEL_SELECTION_METHOD.BOOST;
            }

            if (!nLabel.HasValue && (labelSelectionMethod & IMGDB_LABEL_SELECTION_METHOD.RANDOM) == IMGDB_LABEL_SELECTION_METHOD.RANDOM)
            {
                nLabel = state.GetNextLabel(labelSelectionMethod);
            }

            int?nIdx = state.GetNextImage(imageSelectionMethod, nLabel, nDirectIdx);

            if (!nIdx.HasValue || nIdx.Value < 0)
            {
                nIdx = state.GetNextImage(imageSelectionMethod, nLabel, nDirectIdx);
                if (!nIdx.HasValue || nIdx.Value < 0)
                {
                    string strBoosted = ((imageSelectionMethod & IMGDB_IMAGE_SELECTION_METHOD.BOOST) == IMGDB_IMAGE_SELECTION_METHOD.BOOST) ? "Boosted" : "";
                    string strLabel   = (nLabel.HasValue) ? " for label '" + nLabel.Value.ToString() + "'." : ".";
                    throw new Exception("Failed to find the image index! The data source '" + m_src.Name + "' has no " + strBoosted + " images" + strLabel + ". You may need to re-index the dataset.");
                }
            }

            SimpleDatum sd = m_masterList.GetImage(nIdx.Value, bLoadDataCriteria, bLoadDebugData, m_loadMethod);

            state.UpdateStats(sd);

            return(sd);
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Create a new QueryState on the dataset.
        /// </summary>
        /// <param name="bUseUniqueLabelIndexes">Optionally, specifies to use unique label indexes which is slightly slower, but ensures each label is hit per epoch (default = true).</param>
        /// <param name="bUseUniqueImageIndexes">Optionally, specifies to use unique image indexes which is slightly slower, but ensures each image is hit per epoch (default = true).</param>
        /// <param name="sort">Optionally, specifies an ordering for the query state (default = NONE).</param>
        /// <returns>The new query state is returned.</returns>
        public long CreateQueryState(bool bUseUniqueLabelIndexes = true, bool bUseUniqueImageIndexes = true, IMGDB_SORT sort = IMGDB_SORT.NONE)
        {
            QueryState qsTraining = m_TrainingImages.CreateQueryState(bUseUniqueLabelIndexes, bUseUniqueImageIndexes, sort);
            QueryState qsTesting  = m_TestingImages.CreateQueryState(bUseUniqueLabelIndexes, bUseUniqueImageIndexes, sort);

            return(m_queryStates.CreateNewState(qsTraining, qsTesting));
        }
Ejemplo n.º 3
0
        /// <summary>
        /// Initialize the ImageSet by creating the master list of images, starting its background image loading thread, and then creating the master index that maps the organization of the dataset.
        /// </summary>
        /// <param name="bSilentLoad">Specifies to load the data silently without status output.</param>
        /// <param name="bUseUniqueLabelIndexes">Optionally, specifies to use unique label indexes which is slightly slower, but ensures each label is hit per epoch equally (default = true).</param>
        /// <param name="bUseUniqueImageIndexes">Optionally, specifies to use unique image indexes which is slightly slower, but ensures each image is hit per epoch (default = true).</param>
        /// <param name="nMaxLoadCount">Optionally, specifies to automaticall start the image refresh which only applies when the number of images loaded into memory is less than the actual number of images (default = false).</param>
        /// <returns>Once initialized, the default query state for the image set is returned.  This method may be called multiple times and each time returns a new QueryState.</returns>
        public QueryState Initialize(bool bSilentLoad, bool bUseUniqueLabelIndexes = true, bool bUseUniqueImageIndexes = true, int nMaxLoadCount = 0)
        {
            if (m_masterList == null)
            {
                m_masterList = new MasterList(m_random, m_log, m_src, m_factory, m_rgAbort, nMaxLoadCount);

                if (OnCalculateImageMean != null)
                {
                    m_masterList.OnCalculateImageMean += OnCalculateImageMean;
                }

                if (m_loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL || m_loadMethod == IMAGEDB_LOAD_METHOD.LOAD_EXTERNAL || m_loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND)
                {
                    m_masterList.Load(bSilentLoad);
                }
            }

            if (m_masterIdx == null || m_masterIdx.LoadLimit != nMaxLoadCount)
            {
                m_masterIdx = new MasterIndexes(m_random, m_src, nMaxLoadCount);
            }

            QueryState state = new QueryState(m_masterIdx, bUseUniqueLabelIndexes, bUseUniqueImageIndexes);

            if (m_loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL)
            {
                m_masterList.WaitForLoadingToComplete(m_rgAbort);
            }

            return(state);
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Returns the array of images in the image set, possibly filtered with the filtering parameters.
        /// </summary>
        /// <param name="state">Specifies the query state to use.</param>
        /// <param name="nStartIdx">Specifies a starting index from which the query is to start within the set of images.</param>
        /// <param name="nQueryCount">Optionally, specifies a number of images to retrieve within the set (default = int.MaxValue).</param>
        /// <param name="strFilterVal">Optionally, specifies the filter value that the description must match (default = <i>null</i>, which ignores this parameter).</param>
        /// <param name="nBoostVal">Optionally, specifies the boost value that the boost must match (default = <i>null</i>, which ignores this parameter).</param>
        /// <param name="bBoostValIsExact">Optionally, specifies whether or the boost value (if specified) is to be used literally (exact = true), or as a minimum boost value.</param>
        /// <param name="bAttemptDirectLoad">Optionaly, specifies to directly load all images not already loaded.</param>
        /// <returns>The list of images is returned.</returns>
        /// <remarks>When using the 'nBoostValue' negative values are used to test the exact match of the boost value with the absolute value of the 'nBoostValue', ande
        /// positive values are used to test for boost values that are greater than or equal to the 'nBoostValue'.</remarks>
        public List <SimpleDatum> GetImages(QueryState state, int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int?nBoostVal = null, bool bBoostValIsExact = false, bool bAttemptDirectLoad = false)
        {
            List <int>         rgIdx = state.GetIndexes(nStartIdx, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact);
            List <SimpleDatum> rgSd;

            lock (m_syncObj)
            {
                rgSd = m_rgImages.Where(p => p != null && rgIdx.Contains(p.Index)).ToList();
            }

            if (bAttemptDirectLoad)
            {
                foreach (SimpleDatum sd in rgSd)
                {
                    if (sd != null)
                    {
                        rgIdx.Remove(sd.Index);
                    }
                }

                for (int i = 0; i < rgIdx.Count; i++)
                {
                    rgSd.Add(directLoadImage(rgIdx[i]));
                }

                rgSd = rgSd.OrderBy(p => p.Index).ToList();
            }

            return(rgSd);
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Returns the array of images in the image set, possibly filtered with the filtering parameters.
        /// </summary>
        /// <param name="state">Specifies the query state to use.</param>
        /// <param name="dtStart">Specifies a starting time from which the query is to start within the set of images.</param>
        /// <param name="nQueryCount">Optionally, specifies a number of images to retrieve within the set (default = int.MaxValue).</param>
        /// <param name="strFilterVal">Optionally, specifies the filter value that the description must match (default = <i>null</i>, which ignores this parameter).</param>
        /// <param name="nBoostVal">Optionally, specifies the boost value that the boost must match (default = <i>null</i>, which ignores this parameter).</param>
        /// <param name="bBoostValIsExact">Optionally, specifies whether or the boost value (if specified) is to be used literally (exact = true), or as a minimum boost value.</param>
        /// <returns>The list of images is returned.</returns>
        /// <remarks>When using the 'nBoostValue' negative values are used to test the exact match of the boost value with the absolute value of the 'nBoostValue', ande
        /// positive values are used to test for boost values that are greater than or equal to the 'nBoostValue'.</remarks>
        public List <SimpleDatum> GetImages(QueryState state, DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int?nBoostVal = null, bool bBoostValIsExact = false)
        {
            List <int> rgIdx = state.GetIndexes(dtStart, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact);

            lock (m_syncObj)
            {
                return(m_rgImages.Where(p => p != null && rgIdx.Contains(p.Index)).ToList());
            }
        }
Ejemplo n.º 6
0
        /// <summary>
        /// Create a new QueryState handle based on the QueryStates passed to this function.
        /// </summary>
        /// <param name="training">Specifies the training QueryState used on the training data source.</param>
        /// <param name="testing">Specifies the testing QueryState used on the testing data source.</param>
        /// <returns>The QueryState handle is returned.</returns>
        public long CreateNewState(QueryState training, QueryState testing)
        {
            long lHandle = 0;

            lock (m_objSync)
            {
                if (m_rgQueryStates.Count > 0)
                {
                    lHandle = m_rgQueryStates.Max(p => p.Key) + 1;
                }

                m_rgQueryStates.Add(lHandle, new Tuple <QueryState, QueryState>(training, testing));
            }

            return(lHandle);
        }
Ejemplo n.º 7
0
 /// <summary>
 /// Returns the array of images in the image set, possibly filtered with the filtering parameters.
 /// </summary>
 /// <param name="state">Specifies the query state to use.</param>
 /// <param name="dtStart">Specifies a starting time from which the query is to start within the set of images.</param>
 /// <param name="nQueryCount">Optionally, specifies a number of images to retrieve within the set (default = int.MaxValue).</param>
 /// <param name="strFilterVal">Optionally, specifies the filter value that the description must match (default = <i>null</i>, which ignores this parameter).</param>
 /// <param name="nBoostVal">Optionally, specifies the boost value that the boost must match (default = <i>null</i>, which ignores this parameter).</param>
 /// <param name="bBoostValIsExact">Optionally, specifies whether or the boost value (if specified) is to be used literally (exact = true), or as a minimum boost value.</param>
 /// <returns>The list of images is returned.</returns>
 /// <remarks>When using the 'nBoostValue' negative values are used to test the exact match of the boost value with the absolute value of the 'nBoostValue', ande
 /// positive values are used to test for boost values that are greater than or equal to the 'nBoostValue'.</remarks>
 public List <SimpleDatum> GetImages(QueryState state, DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int?nBoostVal = null, bool bBoostValIsExact = false)
 {
     return(m_masterList.GetImages(state, dtStart, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact));
 }
Ejemplo n.º 8
0
 /// <summary>
 /// Returns the array of images in the image set, possibly filtered with the filtering parameters.
 /// </summary>
 /// <param name="state">Specifies the query state to use.</param>
 /// <param name="nStartIdx">Specifies a starting index from which the query is to start within the set of images.</param>
 /// <param name="nQueryCount">Optionally, specifies a number of images to retrieve within the set (default = int.MaxValue).</param>
 /// <param name="strFilterVal">Optionally, specifies the filter value that the description must match (default = <i>null</i>, which ignores this parameter).</param>
 /// <param name="nBoostVal">Optionally, specifies the boost value that the boost must match (default = <i>null</i>, which ignores this parameter).</param>
 /// <param name="bBoostValIsExact">Optionally, specifies whether or the boost value (if specified) is to be used literally (exact = true), or as a minimum boost value.</param>
 /// <param name="bAttemptDirectLoad">Optionaly, specifies to directly load all images not already loaded.</param>
 /// <returns>The list of images is returned.</returns>
 /// <remarks>When using the 'nBoostValue' negative values are used to test the exact match of the boost value with the absolute value of the 'nBoostValue', ande
 /// positive values are used to test for boost values that are greater than or equal to the 'nBoostValue'.</remarks>
 public List <SimpleDatum> GetImages(QueryState state, int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int?nBoostVal = null, bool bBoostValIsExact = false, bool bAttemptDirectLoad = false)
 {
     return(m_masterList.GetImages(state, nStartIdx, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact, bAttemptDirectLoad));
 }
Ejemplo n.º 9
0
 /// <summary>
 /// Returns the number of images in the image set, optionally with super-boost only.
 /// </summary>
 /// <param name="state">Specifies the query state to use.</param>
 /// <param name="strFilterVal">Optionally, specifies the filter value that the description must match (default = <i>null</i>, which ignores this parameter).</param>
 /// <param name="nBoostVal">Optionally, specifies the boost value that the boost must match (default = <i>null</i>, which ignores this parameter).</param>
 /// <param name="bBoostValIsExact">Optionally, specifies whether or the boost value (if specified) is to be used literally (exact = true), or as a minimum boost value.</param>
 /// <returns>The number of images is returned.</returns>
 /// <remarks>When using the 'nBoostValue' negative values are used to test the exact match of the boost value with the absolute value of the 'nBoostValue', ande
 /// positive values are used to test for boost values that are greater than or equal to the 'nBoostValue'.</remarks>
 public int GetCount(QueryState state, string strFilterVal = null, int?nBoostVal = null, bool bBoostValIsExact = false)
 {
     return(m_masterList.GetCount(state, strFilterVal, nBoostVal, bBoostValIsExact));
 }
Ejemplo n.º 10
0
        /// <summary>
        /// Initialize the DatasetEx by loading the training and testing data sources into memory.
        /// </summary>
        /// <param name="ds">Specifies the dataset to load.</param>
        /// <param name="rgAbort">Specifies a set of wait handles used to cancel the load.</param>
        /// <param name="nPadW">Optionally, specifies a pad to apply to the width of each item (default = 0).</param>
        /// <param name="nPadH">Optionally, specifies a pad to apply to the height of each item (default = 0).</param>
        /// <param name="log">Optionally, specifies an external Log to output status (default = null).</param>
        /// <param name="loadMethod">Optionally, specifies the load method to use (default = LOAD_ALL).</param>
        /// <param name="bSkipMeanCheck">Optionally, specifies to skip the mean check (default = false).</param>
        /// <param name="nImageDbLoadLimit">Optionally, specifies the load limit (default = 0).</param>
        /// <param name="nImageDbAutoRefreshScheduledUpdateInMs">Optionally, specifies the scheduled refresh update period in ms (default = 0).</param>
        /// <param name="dfImageDbAutoRefreshScheduledReplacementPct">Optionally, specifies the scheduled refresh replacement percent (default = 0).</param>
        /// <returns>Upon loading the dataset a handle to the default QueryState is returned, or 0 on cancel.</returns>
        public long Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, IMAGEDB_LOAD_METHOD loadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL, bool bSkipMeanCheck = false, int nImageDbLoadLimit = 0, int nImageDbAutoRefreshScheduledUpdateInMs = 0, double dfImageDbAutoRefreshScheduledReplacementPct = 0)
        {
            lock (m_syncObj)
            {
                m_log = log;

                if (ds != null)
                {
                    m_ds = ds;
                }

                if (m_ds.TrainingSource.ImageWidth == -1 || m_ds.TrainingSource.ImageHeight == -1)
                {
                    log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images.  The mean check will be skipped.");
                    bSkipMeanCheck = true;
                }

                bool bSilentLoad = (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND) ? true : false;

                m_TrainingImages = new ImageSet2(ImageSet2.TYPE.TRAIN, log, m_factory, m_ds.TrainingSource, loadMethod, m_random, rgAbort);
                m_TrainingImages.OnCalculateImageMean += OnCalculateImageMean;
                QueryState  qsTraining = m_TrainingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit);
                SimpleDatum sdMean     = null;

                if (!bSkipMeanCheck)
                {
                    bool bQueryOnly = false;
                    if (EntitiesConnection.GlobalDatabaseConnectInfo.Location == ConnectInfo.TYPE.AZURE)
                    {
                        bQueryOnly = true;
                    }

                    sdMean = m_TrainingImages.GetImageMean(log, rgAbort, bQueryOnly);
                }

                if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
                {
                    return(0);
                }

                m_TestingImages = new ImageSet2(ImageSet2.TYPE.TEST, log, m_factory, m_ds.TestingSource, loadMethod, m_random, rgAbort);
                m_TestingImages.OnCalculateImageMean += OnCalculateImageMean;
                QueryState qsTesting = m_TestingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit);

                if (!bSkipMeanCheck)
                {
                    bool bSave = true;
                    if (EntitiesConnection.GlobalDatabaseConnectInfo.Location == ConnectInfo.TYPE.AZURE)
                    {
                        bSave = false;
                    }

                    m_TestingImages.SetImageMean(sdMean, bSave);
                }

                if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
                {
                    return(0);
                }

                if (loadMethod == IMAGEDB_LOAD_METHOD.LOAD_ALL && nImageDbAutoRefreshScheduledUpdateInMs > 0 && dfImageDbAutoRefreshScheduledReplacementPct > 0)
                {
                    StartAutomaticRefreshSchedule(true, true, nImageDbAutoRefreshScheduledUpdateInMs, dfImageDbAutoRefreshScheduledReplacementPct);
                }

                m_lDefaultQueryState = m_queryStates.CreateNewState(qsTraining, qsTesting);
                return(m_lDefaultQueryState);
            }
        }
Ejemplo n.º 11
0
        /// <summary>
        /// Returns the number of images in the image set, optionally with super-boosted values only.
        /// </summary>
        /// <param name="state">Specifies the query state to use.</param>
        /// <param name="strFilterVal">Optionally, specifies the filter value that the description must match (default = <i>null</i>, which ignores this parameter).</param>
        /// <param name="nBoostVal">Optionally, specifies the boost value that the boost must match (default = <i>null</i>, which ignores this parameter).</param>
        /// <param name="bBoostValIsExact">Optionally, specifies whether or the boost value (if specified) is to be used literally (exact = true), or as a minimum boost value.</param>
        /// <returns>The number of images is returned.</returns>
        /// <remarks>When using the 'nBoostValue' negative values are used to test the exact match of the boost value with the absolute value of the 'nBoostValue', ande
        /// positive values are used to test for boost values that are greater than or equal to the 'nBoostValue'.</remarks>
        public int GetCount(QueryState state, string strFilterVal = null, int?nBoostVal = null, bool bBoostValIsExact = false)
        {
            List <int> rgIdx = state.GetIndexes(0, int.MaxValue, strFilterVal, nBoostVal, bBoostValIsExact);

            return(rgIdx.Count());
        }