Пример #1
0
        public override void  doInitAction(DoInitAction tag)
        {
            if (header_field.version > 6 && tag.sprite != null)
            {
                String __Packages = idRef(tag.sprite);
                className = (__Packages != null && __Packages.StartsWith("__Packages"))?__Packages.Substring(11):null;                 // length("__Packages.") = 11

                if (isRegisterClass(tag.actionList))
                {
                    DebugModule dm = new DebugModule();
                    // C: We actually want the class name here, not the linkage ID.
                    dm.name = "<" + __Packages + ".2>";
                    // C: We want the class name as the second input argument. Fortunately, we don't
                    //    really do anything with the source, so it's okay.
                    dm.Text   = "Object.registerClass(" + __Packages + ", " + __Packages + ");";
                    dm.bitmap = 1;

                    LineRecord lr = new LineRecord(1, dm);

                    int startOffset = tag.actionList.getOffset(0);
                    dm.addOffset(lr, startOffset);

                    tag.actionList.insert(startOffset, lr);
                    modules.put((int)(SupportClass.Random.NextDouble() * System.Int32.MaxValue), dm);
                }
            }

            String[] temp = pool;
            collectActions(tag.actionList);
            pool = temp;

            className = null;
        }
Пример #2
0
        private void RenderLinePrimitive(Graphics g, LineRecord line)
        {
            var penWidth = ScaleLineWidth(line.LineWidth);

            using (var pen = CreatePen(line.Color, penWidth))
            {
                var point1 = ScreenFromWorld(line.Location);
                var point2 = ScreenFromWorld(line.Corner);
                g.DrawLine(pen, point1, point2);
            }
        }
Пример #3
0
 public override void  lineRecord(LineRecord line)
 {
     if (!showLineRecord)
     {
     }
     else if (showDebugSource)
     {
         printLines(line, out_Renamed);
     }
     else
     {
         start(line);
         out_Renamed.WriteLine(" " + line.module.name + ":" + line.lineno);
     }
 }
Пример #4
0
        public static void WordCountExample()
        {
#if local
			// This overload runs the computation on your local computer using a single worker
            var config = new DryadLinqContext(1);

            var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
            // You can create inputs from any IEnumerable source using this method
            var input = config.FromEnumerable(lines);
#else
			string clusterName = "Replace with your HDInsight 3.0 cluster name";
            // to use the davinci.txt example input below, select your cluster's default
            // storage account and container, which automatically includes the sample text
			string accountName = "Replace with a storage account name";
			string containerName = "Replace with a storage container name";

			// This overload creates an Azure-based computation
            var config = new DryadLinqContext(clusterName);
            config.JobFriendlyName = "DryadLINQ Sample Wordcount";

            // plain text files should be read as type LineRecord
			var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
					                                 "example/data/gutenberg/davinci.txt"));
#endif

            var words = input.SelectMany(x => x.Line.Split(' '));
            var groups = words.GroupBy(x => x);
            var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
            var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));

#if local
            // any collection computed by the query can be materialized back at the client,
            // not just the 'output' collection. For large collections this is expensive!
            foreach (LineRecord line in toOutput)
            {
                Console.WriteLine(line.Line);
            }
#else
            // the 'true' parameter to ToStore means the output will be over-written if you run
            // the job more than once
            var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
			           "wc-out.txt"), true).SubmitAndWait();
#endif
        }
Пример #5
0
        public static void WordCountExample()
        {
#if local
            // This overload runs the computation on your local computer using a single worker
            var config = new DryadLinqContext(1);

            var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
            // You can create inputs from any IEnumerable source using this method
            var input = config.FromEnumerable(lines);
#else
            string clusterName = "Replace with your HDInsight 3.0 cluster name";
            // to use the davinci.txt example input below, select your cluster's default
            // storage account and container, which automatically includes the sample text
            string accountName   = "Replace with a storage account name";
            string containerName = "Replace with a storage container name";

            // This overload creates an Azure-based computation
            var config = new DryadLinqContext(clusterName);
            config.JobFriendlyName = "DryadLINQ Sample Wordcount";

            // plain text files should be read as type LineRecord
            var input = config.FromStore <LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
                                                                            "example/data/gutenberg/davinci.txt"));
#endif

            var words    = input.SelectMany(x => x.Line.Split(' '));
            var groups   = words.GroupBy(x => x);
            var counts   = groups.Select(x => new KeyValuePair <string, int>(x.Key, x.Count()));
            var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));

#if local
            // any collection computed by the query can be materialized back at the client,
            // not just the 'output' collection. For large collections this is expensive!
            foreach (LineRecord line in toOutput)
            {
                Console.WriteLine(line.Line);
            }
#else
            // the 'true' parameter to ToStore means the output will be over-written if you run
            // the job more than once
            var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
                                                              "wc-out.txt"), true).SubmitAndWait();
#endif
        }
Пример #6
0
        /// <summary> process any dangling line records that belong to the given module</summary>
        /// <param name="lineRecords">
        /// </param>
        /// <param name="moduleId">
        /// </param>
        /// <param name="handler">
        /// </param>
        /// <returns>
        /// </returns>
        private System.Collections.ArrayList purgeLineRecords(System.Collections.ArrayList lineRecords, int moduleId, DebugHandler handler)
        {
            System.Collections.ArrayList newLineRecords = new System.Collections.ArrayList();
            DebugModule module = (DebugModule)modules.get_Renamed(moduleId);
            int         i = 0, size = lineRecords.Count;

            while (i < size)
            {
                System.Int32 id     = (System.Int32)lineRecords[i];
                LineRecord   lr     = (LineRecord)lineRecords[i + 1];
                System.Int32 offset = (System.Int32)lineRecords[i + 2];

                if (id == moduleId)
                {
                    lr.module = module;

                    if (lr.module != null)
                    {
                        lr.module.addOffset(lr, offset);
                        handler.offset(offset, lr);
                    }
                    else
                    {
                        handler.error("Could not find kDebugScript with module ID = " + id);
                    }
                }
                else
                {
                    newLineRecords.Add(id);
                    newLineRecords.Add(lr);
                    newLineRecords.Add(offset);
                }

                i += 3;
            }

            return(newLineRecords);
        }
Пример #7
0
        public virtual void  printLines(LineRecord lr, StreamWriter out_Renamed)
        {
            DebugModule script = lr.module;

            if (script != null)
            {
                int lineno = lr.lineno;
                if (lineno > 0)
                {
                    while (lineno - 1 > 0 && script.offsets[lineno - 1] == 0)
                    {
                        lineno--;
                    }
                    if (lineno == 1)
                    {
                        indent();
                        out_Renamed.WriteLine(script.name);
                    }
                    int off = script.index[lineno - 1];
                    int len = script.index[lr.lineno] - off;
                    out_Renamed.Write(script.text.ToCharArray(), off, len);
                }
            }
        }
Пример #8
0
        // Helper for IncrementalMeasureLinesAfterInsert, IncrementalMeasureLinesAfterDelete.
        // Formats the line preceding the first directly affected line after a TextContainer change.
        // In general this line might grow as content in the following line is absorbed.
        private void FormatFirstIncrementalLine(int lineIndex, double constraintWidth, LineProperties lineProperties, TextBoxLine line,
            out int lineOffset, out bool endOfParagraph)
        {
            int originalEndOffset = _lineMetrics[lineIndex].EndOffset;
            lineOffset = _lineMetrics[lineIndex].Offset;

            using (line)
            {
                line.Format(lineOffset, constraintWidth, constraintWidth, lineProperties, _cache.TextRunCache, _cache.TextFormatter);

                _lineMetrics[lineIndex] = new LineRecord(lineOffset, line);

                lineOffset += line.Length;
                endOfParagraph = line.EndOfParagraph;
            }

            // Don't clear the cached Visual unless something changed.
            if (originalEndOffset != _lineMetrics[lineIndex].EndOffset)
            {
                ClearLineVisual(lineIndex);
            }
        }
Пример #9
0
        /// <summary>
        /// Instantiates a record according to its record type number.
        /// </summary>
        /// <param name="recordType">Integer representing the record type.</param>
        /// <returns>A new empty instance of a record primitive.</returns>
        private SchPrimitive CreateRecord(int recordType)
        {
            SchPrimitive record;

            switch (recordType)
            {
            case 1:
                record = new SchComponent();
                break;

            case 2:
                record = new PinRecord();
                break;

            case 3:
                record = new SymbolRecord();
                break;

            case 4:
                record = new TextStringRecord();
                break;

            case 5:
                record = new BezierRecord();
                break;

            case 6:
                record = new PolylineRecord();
                break;

            case 7:
                record = new PolygonRecord();
                break;

            case 8:
                record = new EllipseRecord();
                break;

            case 9:
                record = new PieChartRecord();
                break;

            case 10:
                record = new RoundedRectangleRecord();
                break;

            case 11:
                record = new EllipticalArcRecord();
                break;

            case 12:
                record = new ArcRecord();
                break;

            case 13:
                record = new LineRecord();
                break;

            case 14:
                record = new RectangleRecord();
                break;

            case 28:
            case 209:
                record = new TextFrameRecord();
                break;

            case 30:
                record = new ImageRecord();
                break;

            case 34:
                record = new Record34();
                break;

            case 41:
                record = new Record41();
                break;

            case 44:
                record = new Record44();
                break;

            case 45:
                record = new Record45();
                break;

            case 46:
                record = new Record46();
                break;

            case 48:
                record = new Record48();
                break;

            default:
                EmitWarning($"Record {recordType} not supported");
                record = new SchPrimitive();
                break;
            }

            return(record);
        }
Пример #10
0
 public virtual void  lineRecord(LineRecord line)
 {
 }
Пример #11
0
 public override void offset(int offset, LineRecord lr)
 {
     Console.Out.WriteLine("DebugOffset #" + lr.module.id + ":" + lr.lineno + " " + offset);
 }
Пример #12
0
        public virtual void  readTags(DebugHandler handler)
        {
            System.Collections.ArrayList lineRecords = new System.Collections.ArrayList();

            do
            {
                int tag = (int)in_Renamed.readUI32();
                switch (tag)
                {
                case kDebugScript:
                    DebugModule m  = new DebugModule();
                    int         id = (int)in_Renamed.readUI32();
                    m.id     = id;
                    m.bitmap = (int)in_Renamed.readUI32();
                    m.name   = in_Renamed.readString();
                    m.Text   = in_Renamed.readString();

                    adjustModuleName(m);

                    if (modules.contains(id))
                    {
                        DebugModule m2 = (DebugModule)modules.get_Renamed(id);
                        if (!m.Equals(m2))
                        {
                            handler.error("Module '" + m2.name + "' has the same ID as Module '" + m.name + "'");
                            handler.error("Let's check for kDebugOffset that came before Module '" + m2.name + "'");
                            handler.error("Before: Number of accumulated line records: " + lineRecords.Count);
                            lineRecords = purgeLineRecords(lineRecords, id, handler);
                            handler.error("After: Number of accumulated line records: " + lineRecords.Count);
                        }
                    }
                    modules.put(id, m);
                    handler.module(m);
                    break;

                case kDebugOffset:
                    id = (int)in_Renamed.readUI32();
                    int         lineno = (int)in_Renamed.readUI32();
                    DebugModule module = (DebugModule)modules.get_Renamed(id);
                    LineRecord  lr     = new LineRecord(lineno, module);
                    int         offset = (int)in_Renamed.readUI32();

                    if (module != null)
                    {
                        // not corrupted before we add the offset and offset add fails
                        bool wasCorrupt = module.corrupt;
                        if (!module.addOffset(lr, offset) && !wasCorrupt)
                        {
                            handler.error(module.name + ":" + lineno + " does not exist for offset " + offset + ", module marked for exclusion from debugging");
                        }
                        handler.offset(offset, lr);
                    }
                    else
                    {
                        lineRecords.Add((System.Int32)id);
                        lineRecords.Add(lr);
                        lineRecords.Add((System.Int32)offset);
                    }
                    break;

                case kDebugBreakpoint:
                    handler.breakpoint((int)in_Renamed.readUI32());
                    break;

                case kDebugRegisters:
                {
                    offset = (int)in_Renamed.readUI32();
                    int            size = in_Renamed.readUI8();
                    RegisterRecord r    = new RegisterRecord(offset, size);
                    for (int i = 0; i < size; i++)
                    {
                        int    nbr  = in_Renamed.readUI8();
                        String name = in_Renamed.readString();
                        r.addRegister(nbr, name);
                    }
                    handler.registers(offset, r);
                    break;
                }


                case kDebugID:
                    FlashUUID uuid = new FlashUUID();
                    in_Renamed.readFully(uuid.bytes);
                    handler.uuid(uuid);
                    break;

                case -1:
                    break;

                default:
                    throw new SwfFormatException("Unexpected tag id " + tag);
                }

                if (tag == -1)
                {
                    break;
                }
            }while (true);

            int i2 = 0, size2 = lineRecords.Count;

            while (i2 < size2)
            {
                int        id     = ((System.Int32)lineRecords[i2]);
                LineRecord lr     = (LineRecord)lineRecords[i2 + 1];
                int        offset = ((System.Int32)lineRecords[i2 + 2]);
                lr.module = (DebugModule)modules.get_Renamed(id);

                if (lr.module != null)
                {
                    //System.out.println("updated module "+id+" out of order");
                    // not corrupted before we add the offset and offset add fails
                    bool wasCorrupt = lr.module.corrupt;
                    if (!lr.module.addOffset(lr, offset) && !wasCorrupt)
                    {
                        handler.error(lr.module.name + ":" + lr.lineno + " does not exist for offset " + offset + ", module marked for exclusion from debugging");
                    }

                    handler.offset(offset, lr);
                }
                else
                {
                    handler.error("Could not find debug module (id = " + id + ") for offset = " + offset);
                }

                i2 += 3;
            }
        }
Пример #13
0
        // Helper for IncrementalMeasureLinesAfterInsert, IncrementalMeasureLinesAfterDelete.
        // Formats line until we hit a synchronization point, a position where we know
        // following lines could not be affected by the change.
        private void SyncLineMetrics(DirtyTextRange range, double constraintWidth, LineProperties lineProperties, TextBoxLine line,
            bool endOfParagraph, int lineIndex, int lineOffset)
        {
            bool offsetSyncOk = (range.PositionsAdded == 0 || range.PositionsRemoved == 0);
            int lastCoveredCharOffset = range.StartIndex + Math.Max(range.PositionsAdded, range.PositionsRemoved);

            // Keep updating lines until we find a synchronized position.
            while (!endOfParagraph &&
                   (lineIndex == _lineMetrics.Count ||
                    !offsetSyncOk ||
                    lineOffset != _lineMetrics[lineIndex].Offset))
            {
                if (lineIndex < _lineMetrics.Count &&
                    lineOffset >= _lineMetrics[lineIndex].EndOffset)
                {
                    // If the current line offset starts past the current line metric offset,
                    // remove the metric.  This happens when the previous line
                    // frees up enough space to completely consume the following line.
                    // We can't simply replace the record without potentially missing our
                    // [....] position.
                    _lineMetrics.RemoveAt(lineIndex); // 
                    RemoveLineVisualRange(lineIndex, 1);
                }
                else
                {
                    using (line)
                    {
                        line.Format(lineOffset, constraintWidth, constraintWidth, lineProperties, _cache.TextRunCache, _cache.TextFormatter);

                        LineRecord record = new LineRecord(lineOffset, line);

                        if (lineIndex == _lineMetrics.Count ||
                            lineOffset + line.Length <= _lineMetrics[lineIndex].Offset)
                        {
                            // The new line preceeds the old line, insert a new record.

                            // 
                            _lineMetrics.Insert(lineIndex, record);
                            AddLineVisualPlaceholder(lineIndex);
                        }
                        else
                        {
                            // We expect to be colliding with the old line directly.
                            // If we extend past it, we're in danger of needlessly
                            // re-formatting the entire doc (ie, we miss the real
                            // [....] position and don't stop until EndOfParagraph).
                            Invariant.Assert(lineOffset < _lineMetrics[lineIndex].EndOffset);

                            _lineMetrics[lineIndex] = record;
                            ClearLineVisual(lineIndex);

                            // If this line ends past the invalidated region, and it
                            // has a hard line break, it's safe to synchronize on the next
                            // line metric with a matching start offset.
                            offsetSyncOk |= lastCoveredCharOffset <= record.EndOffset && line.HasLineBreak;
                        }

                        lineIndex++;
                        lineOffset += line.Length;
                        endOfParagraph = line.EndOfParagraph;
                    }
                }
            }

            // Remove any trailing lines that got absorbed into the new last line.
            if (endOfParagraph && lineIndex < _lineMetrics.Count)
            {
                int count = _lineMetrics.Count - lineIndex;
                _lineMetrics.RemoveRange(lineIndex, count);
                RemoveLineVisualRange(lineIndex, count);
            }
        }
Пример #14
0
        private void  collectActions(ActionList c)
        {
            // assumption: ActionContext c is always not null! try-catch-finally may be busted.
            if (c == null)
            {
                return;
            }

            // interprets the actions. try to assign names to anonymous functions...
            evalActions(c);

            DebugModule d = findDebugModule(c);

            String emptyMethodName = null;

            // loop again, this time, we register all the actions...
            for (int i = 0; i < c.size(); i++)
            {
                int    ioffset = c.getOffset(i);
                Action a       = c.getAction(i);

                if (emptyMethodName != null && emptyMethodName.Length != 0)
                {
                    functionNames.put(ioffset, emptyMethodName);
                    emptyMethodName = null;
                }

                if (a.code == ActionList.sactionLineRecord)
                {
                    LineRecord line = (LineRecord)a;
                    if (line.module != null)
                    {
                        d = line.module;
                        if (d.name.EndsWith(".mxml"))
                        {
                            mxml[d.name] = d;
                        }
                    }

                    continue;
                }

                if (a.code >= 256)
                {
                    // something synthetic we don't care about
                    continue;
                }

                actions.put(ioffset, (Object)codes[a.code]);
                modules.put(ioffset, d);

                switch (a.code)
                {
                case ActionConstants.sactionDefineFunction:
                case ActionConstants.sactionDefineFunction2:
                    DefineFunction f    = (DefineFunction)a;
                    Int32          size = (Int32)f.codeSize;

                    if (f.actionList.size() == 0)
                    {
                        emptyMethodName = f.name;
                    }
                    else
                    {
                        Int32 lineno = -1;

                        // map all the offsets in this function to the function name
                        for (int j = 0; j < f.actionList.size(); j++)
                        {
                            int    o     = f.actionList.getOffset(j);
                            Action child = f.actionList.getAction(j);
                            if (child.code == ActionList.sactionLineRecord)
                            {
                                if (lineno == -1)
                                {
                                    lineno = (Int32)((LineRecord)child).lineno;
                                }

                                preciseLines.put(o, (Object)((LineRecord)child).lineno);
                            }
                            functionNames.put(o, f.name);
                            functionSizes.put(o, (Object)size);
                        }


                        // map all the offsets in this function to the first line number of this function.
                        for (int j = 0; j < f.actionList.size(); j++)
                        {
                            int o = f.actionList.getOffset(j);
                            functionLines.put(o, (Object)lineno);
                        }
                    }

                    collectActions(f.actionList);
                    break;
                }
            }
        }
Пример #15
0
        // Measures content invalidated due to a TextContainer change.
        private void IncrementalMeasureLinesAfterDelete(double constraintWidth, LineProperties lineProperties, DirtyTextRange range, ref Size desiredSize)
        {
            int delta = range.PositionsAdded - range.PositionsRemoved;
            Invariant.Assert(delta < 0);

            int firstLineIndex = GetLineIndexFromOffset(range.StartIndex);

            // Clip the scope of the affected lines to the region of the document
            // we've already inspected.  Clipping happens when background layout
            // has not yet completed but an incremental update happens.
            int endOffset = range.StartIndex + -delta - 1;
            if (endOffset > _lineMetrics[_lineMetrics.Count - 1].EndOffset)
            {
                Invariant.Assert(this.IsBackgroundLayoutPending);
                endOffset = _lineMetrics[_lineMetrics.Count - 1].EndOffset;
                if (range.StartIndex == endOffset)
                {
                    // Nothing left to do until background layout runs.
                    return;
                }
            }

            int lastLineIndex = GetLineIndexFromOffset(endOffset);

            // Increment the offsets of all following lines.
            // 
            for (int i = lastLineIndex + 1; i < _lineMetrics.Count; i++)
            {
                _lineMetrics[i].Offset += delta;
            }

            TextBoxLine line = new TextBoxLine(this);
            int lineIndex = firstLineIndex;
            int lineOffset;
            bool endOfParagraph;

            // We need to re-format the previous line, because if someone inserted
            // a hard break, the first directly affected line might now be shorter
            // and mergeable with its predecessor.
            if (lineIndex > 0) // 
            {
                FormatFirstIncrementalLine(lineIndex - 1, constraintWidth, lineProperties, line, out lineOffset, out endOfParagraph);
            }
            else
            {
                lineOffset = _lineMetrics[lineIndex].Offset;
                endOfParagraph = false;
            }

            // 



            // Update the first affected line.  If it's completely covered, remove it entirely below.
            if (!endOfParagraph &&
                (range.StartIndex > lineOffset || range.StartIndex + -delta < _lineMetrics[lineIndex].EndOffset))
            {
                // Only part of the line is covered, reformat it.
                using (line)
                {
                    line.Format(lineOffset, constraintWidth, constraintWidth, lineProperties, _cache.TextRunCache, _cache.TextFormatter);

                    _lineMetrics[lineIndex] = new LineRecord(lineOffset, line);

                    lineOffset += line.Length;
                    endOfParagraph = line.EndOfParagraph;
                }
                ClearLineVisual(lineIndex);
                lineIndex++;
            }

            // Remove all the following lines that are completely covered.
            // 
            _lineMetrics.RemoveRange(lineIndex, lastLineIndex - lineIndex + 1);
            RemoveLineVisualRange(lineIndex, lastLineIndex - lineIndex + 1);

            // Recalc the following lines not directly affected as needed.
            SyncLineMetrics(range, constraintWidth, lineProperties, line, endOfParagraph, lineIndex, lineOffset);

            desiredSize = BruteForceCalculateDesiredSize();
        }
Пример #16
0
        // Measures content invalidated due to a TextContainer change.
        private void IncrementalMeasureLinesAfterInsert(double constraintWidth, LineProperties lineProperties, DirtyTextRange range, ref Size desiredSize)
        {
            int delta = range.PositionsAdded - range.PositionsRemoved;
            Invariant.Assert(delta >= 0);
            
            int lineIndex = GetLineIndexFromOffset(range.StartIndex, LogicalDirection.Forward);

            if (delta > 0)
            {
                // Increment of the offsets of all following lines.
                // 
                for (int i = lineIndex + 1; i < _lineMetrics.Count; i++)
                {
                    _lineMetrics[i].Offset += delta;
                }
            }

            TextBoxLine line = new TextBoxLine(this);
            int lineOffset;
            bool endOfParagraph = false;

            // We need to re-format the previous line, because if someone inserted
            // a hard break, the first directly affected line might now be shorter
            // and mergeable with its predecessor.
            if (lineIndex > 0) // 
            {
                FormatFirstIncrementalLine(lineIndex - 1, constraintWidth, lineProperties, line, out lineOffset, out endOfParagraph);
            }
            else
            {
                lineOffset = _lineMetrics[lineIndex].Offset;
            }

            // Format the line directly affected by the change.
            // If endOfParagraph == true, then the line was absorbed into its
            // predessor (because its new content is thinner, or because the
            // TextWrapping property changed).
            if (!endOfParagraph)
            {
                using (line)
                {
                    line.Format(lineOffset, constraintWidth, constraintWidth, lineProperties, _cache.TextRunCache, _cache.TextFormatter);

                    _lineMetrics[lineIndex] = new LineRecord(lineOffset, line);

                    lineOffset += line.Length;
                    endOfParagraph = line.EndOfParagraph;
                }
                ClearLineVisual(lineIndex);
                lineIndex++;
            }

            // Recalc the following lines not directly affected as needed.
            SyncLineMetrics(range, constraintWidth, lineProperties, line, endOfParagraph, lineIndex, lineOffset);

            desiredSize = BruteForceCalculateDesiredSize();
        }
Пример #17
0
 public abstract void offset(int offset, LineRecord lr);
Пример #18
0
        public static void WordCountExample()
        {
#if local
			// This overload runs the computation on your local computer using a single worker
            var config = new DryadLinqContext(1);

            var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
            // You can create inputs from any IEnumerable source using this method
            var input = config.FromEnumerable(lines);
#else
#if azure
			string clusterName = "Replace with your HDInsight 3.1 cluster name";
            // to use the davinci.txt example input below, select your cluster's default
            // storage account and container, which automatically includes the sample text
			string accountName = "Replace with a storage account name";
			string containerName = "Replace with a storage container name";

			// This overload creates an Azure-based computation
            var config = new DryadLinqContext(clusterName);
            config.JobFriendlyName = "DryadLINQ Sample Wordcount";

            // plain text files should be read as type LineRecord
			var input = config.FromStore<LineRecord>(Utils.ToAzureUri(accountName, containerName,
					                                 "example/data/gutenberg/davinci.txt"));
#else
            // to use a yarn cluster, fill in the username, resource node machine name and port, and name node and hdfs port below (use -1 for the default hdfs port).
            string user = "******";
            string resourceNode = "Replace with the name of the computer your resource node is running on";
            int rmPort = 8088;
            string nameNode = "Replace with the name of the computer your name node is running on";
            int hdfsPort = -1;
            // set the YARN queue to submit your job on below. Leave null to use the default queue
            string queue = null;
            // set the number of worker containers to start for the DryadLINQ job below
            int numberOfWorkers = 2;
            // set the amount of memory requested for the DryadLINQ job manager container below: 8GB should be enough for even the largest jobs, and 2GB will normally suffice
            int amMemoryMB = 2000;
            // set the amount of memory requested for the DryadLINQ worker containers below. The amount needed will depend on the code you are running
            int workerMemoryMB = 8000;
			// This overload runs the computation on your local computer using a single worker
            var cluster = new DryadLinqYarnCluster(user, numberOfWorkers, amMemoryMB, workerMemoryMB, queue, resourceNode, rmPort, nameNode, hdfsPort);

            var config = new DryadLinqContext(cluster);

            var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
            // You can create inputs from any IEnumerable source using this method
            var input = config.FromEnumerable(lines);
#endif
#endif

            var words = input.SelectMany(x => x.Line.Split(' '));
            var groups = words.GroupBy(x => x);
            var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
            var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));

#if azure
            // the 'true' parameter to ToStore means the output will be over-written if you run
            // the job more than once
            var info = toOutput.ToStore(Utils.ToAzureUri(accountName, containerName,
			           "wc-out.txt"), true).SubmitAndWait();
#else
            // any collection computed by the query can be materialized back at the client,
            // not just the 'output' collection. For large collections this is expensive!
            foreach (LineRecord line in toOutput)
            {
                Console.WriteLine(line.Line);
            }
#endif
        }