private void file_reader() { string fileName = System.Environment.GetEnvironmentVariable("PATH_GRAPH_FILE"); IInputFormatInstance extractor_input_format = (IInputFormatInstance)server.fetchFileContentObject(); extractor_input_format.PARTITION_SIZE = partition_size; IDictionary <int, IInputFormatInstance> sub_formats = extractor_input_format.extractBins(fileName); this.PartitionTABLE = extractor_input_format.PartitionTABLE; foreach (IInputFormatInstance shard in sub_formats.Values) { IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)client.createItem(); int first_vertex_id_from_partition = shard.firstVertex(shard.PARTID); ((IIntegerInstance)item.Key).Value = first_vertex_id_from_partition; //counter_write_global; item.Value = shard; client.put(item); counter_write_chunk++; //counter_write_global++; if (counter_write_chunk >= CHUNK_SIZE) { Console.WriteLine("NEW CHUNK size=" + counter_write_chunk); counter_write_chunk = 0; client.finish(); } } client.finish(); client.finish(); Console.WriteLine("FINISHING READING DATA SOURCE"); }
public void graph_creator() { IKVPairInstance <IInteger, IIterator <IInputFormat> > input_gifs_instance = (IKVPairInstance <IInteger, IIterator <IInputFormat> >)Graph_values.Instance; IIteratorInstance <IInputFormat> vgifs = (IIteratorInstance <IInputFormat>)input_gifs_instance.Value; object o; if (partition_own == null) { if (vgifs.fetch_next(out o)) { IInputFormatInstance gif = (IInputFormatInstance)o; partition = gif.PartitionTABLE; partid = gif.PARTID; partition_size = gif.PARTITION_SIZE; g = Graph.newInstance(gif.VSIZE); // pega-se uma instancia do graph, com vertices do tipo inteiro, com tamanho previsto VSIZE g.DataContainer.AllowingLoops = false; // não serão premitidos laços g.DataContainer.AllowingMultipleEdges = false; // não serão permitidas múltiplas arestas graph_creator_aux(gif); // inserem-se dados no grafo partition_own = new bool[partition_size]; partition_own [partid] = true; } } while (vgifs.fetch_next(out o)) { graph_creator_aux((IInputFormatInstance)o); partition_own [((IInputFormatInstance)o).PARTID] = true; } }
public void graph_creator() { IKVPairInstance <IInteger, IIterator <IInputFormat> > input_gifs_instance = (IKVPairInstance <IInteger, IIterator <IInputFormat> >)Graph_values.Instance; IIteratorInstance <IInputFormat> vgifs = (IIteratorInstance <IInputFormat>)input_gifs_instance.Value; object o; if (partition_own == null) { if (vgifs.fetch_next(out o)) { IInputFormatInstance gif = (IInputFormatInstance)o; partition = gif.PartitionTABLE; partition_size = gif.PARTITION_SIZE; g = Graph.newInstance(gif.VSIZE); g.DataContainer.AllowingLoops = false; g.DataContainer.AllowingMultipleEdges = false; graph_creator_aux(gif); partition_own = new bool[partition_size]; partition_own [gif.PARTID] = true; this.partid = gif.PARTID; } } while (vgifs.fetch_next(out o)) { graph_creator_aux((IInputFormatInstance)o); partition_own [((IInputFormatInstance)o).PARTID] = true; } }
public override void main() { //Reader.Server = new DataSourceReader(); IInputFormatInstance input_format_instance = Data_format.newInstanceIF(); Reader.Server = new DataSourceReader(input_format_instance); }
private void graph_creator_aux(IInputFormatInstance gif) { bool weighted = gif.Weight.Length == gif.Source.Length; float f = 1.0f; for (int i = 0; i < gif.ESIZE; i++) { int s = gif.Source [i]; int t = gif.Target [i]; if (weighted) { f = gif.Weight [i]; } g.addVertex(s); g.addVertex(t); g.noSafeAdd(s, t, f); if (s == 0 || t == 0) { throw new ArgumentNullException("WARNING: Vertex id is 0! "); } } IIteratorInstance <IKVPair <IInteger, IInputFormat> > output_gifs_instance = (IIteratorInstance <IKVPair <IInteger, IInputFormat> >)Output_gif.Instance; IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)Output_gif.createItem(); ((IIntegerInstance)item.Key).Value = gif.PARTID; item.Value = gif; output_gifs_instance.put(item); }
public IDictionary <int, IInputFormatInstance> extractBins(string fileName) { //fileName = System.Environment.GetEnvironmentVariable ("PATH_GRAPH_FILE"); this.extractFile(fileName); IDictionary <int, IInputFormatInstance> dic = new Dictionary <int, IInputFormatInstance> (this.PARTITION_SIZE); bool weighted = this.Weight.Length > 1; for (int i = 0; i < this.PARTITION_SIZE; i++) { dic [i] = new IInputFormatInstanceImpl(); dic [i].PARTID = i; } for (int i = 0; i < this.ESIZE; i++) { int s = this.Source [i]; int t = this.Target [i]; int spart = this.PartitionTABLE [s - 1]; int tpart = this.PartitionTABLE [t - 1]; if (!weighted) { dic [spart].Add(s, t); if (spart != tpart) { dic [tpart].Add(s, t); } } else { float f = this.Weight[i]; dic [spart].Add(s, t, f); if (spart != tpart) { dic [tpart].Add(s, t, f); } } } for (int i = 0; i < this.PARTITION_SIZE; i++) { IInputFormatInstance tmp = dic [i]; tmp.Trim(); tmp.PartitionTABLE = this.PartitionTABLE; tmp.PARTITION_SIZE = this.PARTITION_SIZE; } source = new int[1]; target = new int[1]; weight = new float[1]; esize = 0; vsize = 0; partid = -1; count = 0; return(dic); }
public override void main() { IInputFormatInstance input_gif_instance = (IInputFormatInstance)Input_key.Instance; IIntegerInstance output_integer_instance = (IIntegerInstance)Output_key.Instance; int value = input_gif_instance.PARTID; //int value = graph_partition_table[((int) input_integer_instance.Value) -1]; //Trace.WriteLine("BIN FUNCTION " + (value % NumberOfPartitions) + "value=" + value + ", npart=" + NumberOfPartitions); output_integer_instance.Value = value % NumberOfPartitions; if (this.graph_partition_table == null) { this.graph_partition_table = input_gif_instance.PartitionTABLE; } Input_key.Instance = null; }
private void graph_creator_aux(IInputFormatInstance gif) { for (int i = 0; i < gif.ESIZE;) { if (gif.Target [i] != 0) // Será usada a forma canonica: i->j, onde i<j, i>0 j>0 { int s = gif.Source [i] < gif.Target [i] ? gif.Source [i] : gif.Target [i]; int t = gif.Target [i] > gif.Source [i] ? gif.Target [i] : gif.Source [i]; g.addVertex(s); g.addVertex(t); g.noSafeAdd(s, t); i++; } } IIteratorInstance <IKVPair <IInteger, IInputFormat> > output_gifs_instance = (IIteratorInstance <IKVPair <IInteger, IInputFormat> >)Output_gif.Instance; IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)Output_gif.createItem(); ((IIntegerInstance)item.Key).Value = gif.PARTID; item.Value = gif; output_gifs_instance.put(item); // Emite-se gif novamente para que a funcão de particionamento do conector receba a instancia PartitionTABLE. } // Isso é necessário no caso de IKey ser do tipo IVertex.
private void graph_creator_aux(IInputFormatInstance gif) { for (int i = 0; i < gif.ESIZE; i++) { int s = gif.Source [i]; int t = gif.Target [i]; g.addVertex(s); g.addVertex(t); g.addEdge(s, t); if (s == 0 || t == 0) { throw new ArgumentNullException("WARNING: Vertex id is 0! "); } } IIteratorInstance <IKVPair <IInteger, IInputFormat> > output_gifs_instance = (IIteratorInstance <IKVPair <IInteger, IInputFormat> >)Output_gif.Instance; IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)Output_gif.createItem(); ((IIntegerInstance)item.Key).Value = gif.PARTID; item.Value = gif; output_gifs_instance.put(item); }
private void graph_creator_aux(IInputFormatInstance gif) { for (int i = 0; i < gif.ESIZE;) { if (gif.Target [i] != 0) // Será usada a forma canonica: i->j, onde i<j, i>0 j>0 { int s = gif.Source [i] < gif.Target [i] ? gif.Source [i] : gif.Target [i]; int t = gif.Target [i] > gif.Source [i] ? gif.Target [i] : gif.Source [i]; g.addVertex(s); g.addVertex(t); g.noSafeAdd(s, t); //Usando noSafeAdd! Isso significa que erros no arquivo fonte não serão tratados. i++; //Ou seja, os dados serão inseridos da forma como chegam. Para tratamento, usa-se g.addEdge(s,t). } } IIteratorInstance <IKVPair <IInteger, IInputFormat> > output_gifs_instance = (IIteratorInstance <IKVPair <IInteger, IInputFormat> >)Output_gif.Instance; IKVPairInstance <IInteger, IInputFormat> item = (IKVPairInstance <IInteger, IInputFormat>)Output_gif.createItem(); ((IIntegerInstance)item.Key).Value = gif.PARTID; item.Value = gif; output_gifs_instance.put(item); // Emite-se gif novamente para que a funcão de particionamento do conector receba a instancia PartitionTABLE. } // Isso é necessário no caso de IKey ser do tipo IVertex.
public IInputFormatInstance newInstanceIF() { IInputFormatInstance instance = (IInputFormatInstance)newInstance(); return(instance); }
public DataSourceReader(IInputFormatInstance o) { input_format_instance = o; }