private static void RunSpeedTest() { Console.WriteLine(DateTime.Now.Millisecond + DateTime.Now.Second * 1000 + DateTime.Now.Minute * 60 * 1000); //IntPtr vw = VowpalWabbitInterface.Initialize("--ngram 2 --skips 4 -l 0.25 -b 22 -d rcv1.train.raw.txt -f out"); //IntPtr vw = VowpalWabbitInterface.Initialize("-d rcv1.train.raw.txt -b 22 --ngram 2 --skips 4 -l 0.25 -c"); //IntPtr vw = VowpalWabbitInterface.Initialize("-d rcv1.train.raw.txt -c"); IntPtr vw = VowpalWabbitInterface.Initialize("-d vw.dat"); VowpalWabbitInterface.StartParser(vw, false); int count = 0; IntPtr example = IntPtr.Zero; Stopwatch s = Stopwatch.StartNew(); while (IntPtr.Zero != (example = VowpalWabbitInterface.GetExample(vw))) { count++; float score = VowpalWabbitInterface.Learn(vw, example); VowpalWabbitInterface.FinishExample(vw, example); } s.Stop(); long t1 = s.ElapsedMilliseconds; VowpalWabbitInterface.EndParser(vw); VowpalWabbitInterface.Finish(vw); Console.WriteLine(DateTime.Now.Millisecond + DateTime.Now.Second * 1000 + DateTime.Now.Minute * 60 * 1000); Debug.WriteLine("RunSpeedTest Elapsed Time: {0} ms", s.ElapsedMilliseconds); Console.WriteLine("RunSpeedTest Elapsed Time: {0} ms", s.ElapsedMilliseconds); }
private static void RunParserTest2() { long maxEx = 10; //long.MaxValue // IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d 0002.dat -f out"); IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d 0002.dat -b 18 --hash strings -f out --ring_size 74748 --examples 10"); VowpalWabbitInterface.StartParser(vw, false); int count = 0; IntPtr example = IntPtr.Zero; while (count < maxEx) { example = VowpalWabbitInterface.GetExample(vw); if (IntPtr.Zero == example) { break; } count++; //int featureSpaceLen = 0; //IntPtr featureSpacePtr = VowpalWabbitInterface.ExportExample(vw, example, ref featureSpaceLen); //VowpalWabbitInterface.FEATURE_SPACE[] featureSpace = new VowpalWabbitInterface.FEATURE_SPACE[featureSpaceLen]; //int featureSpace_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE_SPACE)); //for (int i = 0; i < featureSpaceLen; i++) //{ // IntPtr curfeatureSpacePos = new IntPtr(featureSpacePtr.ToInt32() + i * featureSpace_size); // featureSpace[i] = (VowpalWabbitInterface.FEATURE_SPACE)Marshal.PtrToStructure(curfeatureSpacePos, typeof(VowpalWabbitInterface.FEATURE_SPACE)); // VowpalWabbitInterface.FEATURE[] feature = new VowpalWabbitInterface.FEATURE[featureSpace[i].len]; // int feature_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE)); // for (int j = 0; j < featureSpace[i].len; j++) // { // IntPtr curfeaturePos = new IntPtr((featureSpace[i].features.ToInt32() + j * feature_size)); // feature[j] = (VowpalWabbitInterface.FEATURE)Marshal.PtrToStructure(curfeaturePos, typeof(VowpalWabbitInterface.FEATURE)); // } //} //VowpalWabbitInterface.ReleaseFeatureSpace(featureSpacePtr, featureSpaceLen); //float score = VowpalWabbitInterface.Learn(vw, example); //VowpalWabbitInterface.FinishExample(vw, example); } VowpalWabbitInterface.EndParser(vw); VowpalWabbitInterface.Finish(vw); }
static void Main(string[] args) { IntPtr vw = VowpalWabbitInterface.Initialize("--hash all -q st --noconstant -i train.w"); IntPtr example = VowpalWabbitInterface.ReadExample(vw, "|s p^the_man w^the w^man |t p^un_homme w^un w^homme"); float score = VowpalWabbitInterface.Learn(vw, example); VowpalWabbitInterface.FinishExample(vw, example); Console.Error.WriteLine("p2 = {0}", score); VowpalWabbitInterface.Finish(vw); }
private static void RunFlatExampleTestEx() { //IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d rcv1.train.raw.txt -f out"); IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d ..\\..\\..\\test\\train-sets\\0002.dat -f out"); VowpalWabbitInterface.StartParser(vw, false); uint stride = (uint)VowpalWabbitInterface.Get_Stride(vw); int count = 0; IntPtr example = IntPtr.Zero; while (IntPtr.Zero != (example = VowpalWabbitInterface.GetExample(vw))) { count++; float prediction = VowpalWabbitInterface.GetPrediction(example); float importance = VowpalWabbitInterface.GetImportance(example); float initial = VowpalWabbitInterface.GetInitial(example); float label = VowpalWabbitInterface.GetLabel(example); UInt32 tag_len = (UInt32)VowpalWabbitInterface.GetTagLength(example); byte[] tag = new byte[tag_len]; if (tag_len > 0) { Marshal.Copy(VowpalWabbitInterface.GetTag(example), tag, 0, (int)tag_len); } UInt32 num_features = (UInt32)VowpalWabbitInterface.GetFeatureNumber(example); VowpalWabbitInterface.FEATURE[] f; if (num_features > 0) { f = new VowpalWabbitInterface.FEATURE[num_features]; IntPtr feature_count = (IntPtr)0; IntPtr ret = VowpalWabbitInterface.GetFeatures(vw, example, ref feature_count); int feature_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE)); for (int i = 0; i < (int)feature_count; i++) { IntPtr curfeaturePos = new IntPtr(ret.ToInt32() + i * feature_size); f[i] = (VowpalWabbitInterface.FEATURE)Marshal.PtrToStructure(curfeaturePos, typeof(VowpalWabbitInterface.FEATURE)); } } VowpalWabbitInterface.FinishExample(vw, example); } VowpalWabbitInterface.EndParser(vw); VowpalWabbitInterface.Finish(vw); }
private static void RunFlatExampleTestEx() { //IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d rcv1.train.raw.txt -f out"); IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d 0002.dat -f out"); VowpalWabbitInterface.StartParser(vw, false); uint stride = VowpalWabbitInterface.Get_Stride(vw); int count = 0; IntPtr example = IntPtr.Zero; while (IntPtr.Zero != (example = VowpalWabbitInterface.GetExample(vw))) { count++; float prediction = VowpalWabbitInterface.GetPrediction(example); float importance = VowpalWabbitInterface.GetImportance(example); float initial = VowpalWabbitInterface.GetInitial(example); float label = VowpalWabbitInterface.GetLabel(example); UInt32 tag_len = VowpalWabbitInterface.GetTagLength(example); byte[] tag = new byte[tag_len]; if (tag_len > 0) { //Marshal.Copy(VowpalWabbitInterface.GetTag(example), tag, 0, (int)tag_len); //error CS1502: The best overloaded method match for 'System.Runtime.InteropServices.Marshal.Copy(int[], int, System.IntPtr, int)' has some invalid arguments ; } UInt32 num_features = VowpalWabbitInterface.GetFeatureNumber(example); VowpalWabbitInterface.FEATURE[] f; if (num_features > 0) { f = new VowpalWabbitInterface.FEATURE[num_features]; int feature_count = 0; IntPtr ret = VowpalWabbitInterface.GetFeatures(vw, example, ref feature_count); int feature_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE)); for (int i = 0; i < feature_count; i++) { IntPtr curfeaturePos = new IntPtr(ret.ToInt32() + i * feature_size); f[i] = (VowpalWabbitInterface.FEATURE)Marshal.PtrToStructure(curfeaturePos, typeof(VowpalWabbitInterface.FEATURE)); } } VowpalWabbitInterface.FinishExample(vw, example); } VowpalWabbitInterface.EndParser(vw); VowpalWabbitInterface.Finish(vw); }
static void Main(string[] args) { IntPtr vw = VowpalWabbitInterface.Initialize("--hash all -q st --noconstant -i .\\..\\..\\..\\..\\library\\train.w"); IntPtr example = VowpalWabbitInterface.ReadExample(vw, "|s p^the_man w^the w^man |t p^un_homme w^un w^homme"); float score = VowpalWabbitInterface.Learn(vw, example); VowpalWabbitInterface.FinishExample(vw, example); uint nnum = VowpalWabbitInterface.HashSpace(vw, "p"); uint fnum = VowpalWabbitInterface.HashFeature(vw, "s p^the_man w^the w^man", nnum); VowpalWabbitInterface.FEATURE_SPACE[] featureSpace = new VowpalWabbitInterface.FEATURE_SPACE[11]; for (int i = 0; i < featureSpace.Length; i++) { featureSpace[i].name = (byte)('a' + i); // create an array of features for this feature space VowpalWabbitInterface.FEATURE[] features = new VowpalWabbitInterface.FEATURE[2]; // allocate some space for the unmanaged copies of these features featureSpace[i].features = Marshal.AllocHGlobal(features.Length * Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE))); featureSpace[i].len = features.Length; // fill the features, and copy them to the allocated memory for (int j = 0; j < features.Length; j++) { // fill the feature features[j].x = 1.1F; features[j].weight_index = 100; // copy the feature to the unmanaged heap Marshal.StructureToPtr(features[j], IntPtr.Add(featureSpace[i].features, j * Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE))), false); } } IntPtr importedExample = VowpalWabbitInterface.ImportExample(vw, featureSpace, featureSpace.Length); Console.Error.WriteLine("p2 = {0}", score); VowpalWabbitInterface.Finish(vw); // clean up the memory we allocated for (int i = 0; i < featureSpace.Length; i++) { Marshal.FreeHGlobal(featureSpace[i].features); } }
private static void RunParserTest() { IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d 0002.dat -f out"); VowpalWabbitInterface.StartParser(vw, false); int count = 0; IntPtr example = IntPtr.Zero; while (IntPtr.Zero != (example = VowpalWabbitInterface.GetExample(vw))) { IntPtr labelPtr = VowpalWabbitInterface.GetLabel(vw, example); VowpalWabbitInterface.LABEL label = new VowpalWabbitInterface.LABEL(); label = (VowpalWabbitInterface.LABEL)Marshal.PtrToStructure(labelPtr, typeof(VowpalWabbitInterface.LABEL)); count++; int featureSpaceLen = 0; IntPtr featureSpacePtr = VowpalWabbitInterface.ExportExample(vw, example, ref featureSpaceLen); VowpalWabbitInterface.FEATURE_SPACE[] featureSpace = new VowpalWabbitInterface.FEATURE_SPACE[featureSpaceLen]; int featureSpace_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE_SPACE)); for (int i = 0; i < featureSpaceLen; i++) { IntPtr curfeatureSpacePos = new IntPtr(featureSpacePtr.ToInt32() + i * featureSpace_size); featureSpace[i] = (VowpalWabbitInterface.FEATURE_SPACE)Marshal.PtrToStructure(curfeatureSpacePos, typeof(VowpalWabbitInterface.FEATURE_SPACE)); VowpalWabbitInterface.FEATURE[] feature = new VowpalWabbitInterface.FEATURE[featureSpace[i].len]; int feature_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE)); for (int j = 0; j < featureSpace[i].len; j++) { IntPtr curfeaturePos = new IntPtr((featureSpace[i].features.ToInt32() + j * feature_size)); feature[j] = (VowpalWabbitInterface.FEATURE)Marshal.PtrToStructure(curfeaturePos, typeof(VowpalWabbitInterface.FEATURE)); } } VowpalWabbitInterface.ReleaseFeatureSpace(featureSpacePtr, featureSpaceLen); float score = VowpalWabbitInterface.Learn(vw, example); VowpalWabbitInterface.FinishExample(vw, example); } VowpalWabbitInterface.EndParser(vw); VowpalWabbitInterface.Finish(vw); }
private static void RunVWParce_and_VWLearn() { // parse and cache IntPtr vw0 = VowpalWabbitInterface.Initialize(@"-d rcv1.train.raw.txt -c"); VowpalWabbitInterface.StartParser(vw0, false); long instanceCount = 0; VWInstanceEx[] vwInstanceExs = new VWInstanceEx[781266]; Stopwatch s = Stopwatch.StartNew(); while (instanceCount < 781266) { IntPtr example = VowpalWabbitInterface.GetExample(vw0); if (IntPtr.Zero == example) { break; } vwInstanceExs[instanceCount] = new VWInstanceEx(vw0, example); VowpalWabbitInterface.FinishExample(vw0, example); instanceCount++; } VowpalWabbitInterface.EndParser(vw0); VowpalWabbitInterface.Finish(vw0); // learn instanceCount = 0; IntPtr vw = VowpalWabbitInterface.Initialize(@"--quiet --random_seed 276518665 -f C:\\Users\\niruc\\AppData\\Local\\Temp\\outl1.tmp --readable_model C:\\Users\\niruc\\AppData\\Local\\Temp\\outl2.tmp"); foreach (VWInstanceEx vwInstanceEx in vwInstanceExs) { VowpalWabbitInterface.FEATURE_SPACE[] featureSpace = new VowpalWabbitInterface.FEATURE_SPACE[vwInstanceEx.featureSpace.Length]; GCHandle[] pinnedsFeatures = new GCHandle[vwInstanceEx.featureSpace.Length]; for (int i = 0; i < vwInstanceEx.featureSpace.Length; i++) { pinnedsFeatures[i] = GCHandle.Alloc(vwInstanceEx.featureSpace[i].features, GCHandleType.Pinned); featureSpace[i].features = pinnedsFeatures[i].AddrOfPinnedObject(); } GCHandle pinnedFeatureSpace = GCHandle.Alloc(featureSpace, GCHandleType.Pinned); IntPtr featureSpacePtr = pinnedFeatureSpace.AddrOfPinnedObject(); IntPtr importedExample = VowpalWabbitInterface.ImportExample(vw, featureSpacePtr, vwInstanceEx.featureSpace.Length); VowpalWabbitInterface.Learn(vw, importedExample); VowpalWabbitInterface.FinishExample(vw, importedExample); for (int i = 0; i < vwInstanceEx.featureSpace.Length; i++) { pinnedsFeatures[i].Free(); } pinnedFeatureSpace.Free(); importedExample = IntPtr.Zero; instanceCount++; } VowpalWabbitInterface.Finish(vw); Debug.WriteLine("Elapsed Time: {0} ms", s.ElapsedMilliseconds); Console.WriteLine("Elapsed Time: {0} ms", s.ElapsedMilliseconds); }
private static void RunFeaturesTest() { // this usually requires that the library script to update train.w or its moral equivalent needs to have been run IntPtr vw = VowpalWabbitInterface.Initialize("-q st --noconstant --quiet"); IntPtr example = VowpalWabbitInterface.ReadExample(vw, "1 |s p^the_man w^the w^man |t p^un_homme w^un w^homme"); float score = VowpalWabbitInterface.Learn(vw, example); VowpalWabbitInterface.FinishExample(vw, example); VowpalWabbitInterface.FEATURE_SPACE[] featureSpace = new VowpalWabbitInterface.FEATURE_SPACE[2]; //maximum number of index spaces VowpalWabbitInterface.FEATURE[] sfeatures = new VowpalWabbitInterface.FEATURE[3]; // the maximum number of features VowpalWabbitInterface.FEATURE[] tfeatures = new VowpalWabbitInterface.FEATURE[3]; // the maximum number of features GCHandle pinnedsFeatures = GCHandle.Alloc(sfeatures, GCHandleType.Pinned); GCHandle pinnedtFeatures = GCHandle.Alloc(tfeatures, GCHandleType.Pinned); featureSpace[0].features = pinnedsFeatures.AddrOfPinnedObject(); featureSpace[1].features = pinnedtFeatures.AddrOfPinnedObject(); GCHandle pinnedFeatureSpace = GCHandle.Alloc(featureSpace, GCHandleType.Pinned); IntPtr featureSpacePtr = pinnedFeatureSpace.AddrOfPinnedObject(); uint snum = VowpalWabbitInterface.HashSpace(vw, "s"); featureSpace[0].name = (byte)'s'; sfeatures[0].weight_index = VowpalWabbitInterface.HashFeature(vw, "p^the_man", snum); sfeatures[0].x = 1; // add the character "delta" to test unicode // do it as a string to test the marshaling is doing pinning correctly. const string s = "w^thew^man\u0394"; sfeatures[1].weight_index = VowpalWabbitInterface.HashFeature(vw, s, snum); sfeatures[1].x = 1; sfeatures[2].weight_index = VowpalWabbitInterface.HashFeature(vw, "w^man", snum); sfeatures[2].x = 1; featureSpace[0].len = 3; uint tnum = VowpalWabbitInterface.HashSpace(vw, "t"); featureSpace[1].name = (byte)'t'; tfeatures[0].weight_index = VowpalWabbitInterface.HashFeature(vw, "p^un_homme", tnum); tfeatures[0].x = 1; tfeatures[1].weight_index = VowpalWabbitInterface.HashFeature(vw, "w^un", tnum); tfeatures[1].x = 1; tfeatures[2].weight_index = VowpalWabbitInterface.HashFeature(vw, "w^homme", tnum); tfeatures[2].x = 1; featureSpace[1].len = 3; IntPtr importedExample = VowpalWabbitInterface.ImportExample(vw, featureSpacePtr, featureSpace.Length); VowpalWabbitInterface.AddLabel(importedExample, 1); score = VowpalWabbitInterface.Learn(vw, importedExample); Console.Error.WriteLine("p2 = {0}", score); VowpalWabbitInterface.Finish(vw); // clean up the memory we allocated pinnedsFeatures.Free(); pinnedtFeatures.Free(); pinnedFeatureSpace.Free(); }
//private static void RunFlatExampleTest() //{ // IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d 0002.dat -f out"); // VowpalWabbitInterface.StartParser(vw, false); // int count = 0; // IntPtr example = IntPtr.Zero; // while (IntPtr.Zero != (example = VowpalWabbitInterface.GetExample(vw))) // { // count++; // // make example flat // IntPtr flatec = VowpalWabbitInterface.Flatten_Example(vw, example); // VowpalWabbitInterface.FinishExample(vw, example); // } // VowpalWabbitInterface.EndParser(vw); // VowpalWabbitInterface.Finish(vw); //} private static void RunFlatExampleTestEx() { IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d rcv1.train.raw.txt -f out"); //IntPtr vw = VowpalWabbitInterface.Initialize("-q st -d 0002.dat -f out"); VowpalWabbitInterface.StartParser(vw, false); uint stride = VowpalWabbitInterface.Get_Stride(vw); uint mask = (1 << 18) - 1; int count = 0; IntPtr example = IntPtr.Zero; while (IntPtr.Zero != (example = VowpalWabbitInterface.GetExample(vw))) { count++; IntPtr flatec = IntPtr.Zero; // make example flat flatec = VowpalWabbitInterface.Flatten_Example(vw, example); if (IntPtr.Zero == flatec) { continue; } VowpalWabbitInterface.FLAT_RAW_EXAMPLE flat = new VowpalWabbitInterface.FLAT_RAW_EXAMPLE(); flat = (VowpalWabbitInterface.FLAT_RAW_EXAMPLE)Marshal.PtrToStructure(flatec, typeof(VowpalWabbitInterface.FLAT_RAW_EXAMPLE)); // Get IntPtr data VowpalWabbitInterface.FLAT_EXAMPLE flatEx = new VowpalWabbitInterface.FLAT_EXAMPLE(); flatEx.final_prediction = flat.final_prediction; flatEx.example_counter = flat.example_counter; flatEx.ft_offset = flat.ft_offset; flatEx.global_weight = flat.global_weight; flatEx.num_features = flat.num_features; flatEx.ld = new VowpalWabbitInterface.LABEL(); flatEx.ld = (VowpalWabbitInterface.LABEL)Marshal.PtrToStructure(flat.ld, typeof(VowpalWabbitInterface.LABEL)); if (flat.tag_len > 0) { flatEx.tag = new byte[flat.tag_len]; Marshal.Copy(flat.tag, flatEx.tag, 0, flat.tag_len); } IList <int> indices = new List <int>(); if (flat.num_features > 0) { flatEx.feature_map = new VowpalWabbitInterface.FEATURE[flat.num_features]; int feature_size = Marshal.SizeOf(typeof(VowpalWabbitInterface.FEATURE)); for (int i = 0; i < (int)flat.num_features; i++) { IntPtr curfeaturePos = new IntPtr(flat.feature_map.ToInt32() + i * feature_size); flatEx.feature_map[i] = (VowpalWabbitInterface.FEATURE)Marshal.PtrToStructure(curfeaturePos, typeof(VowpalWabbitInterface.FEATURE)); int val = ((int)(flatEx.feature_map[i].weight_index / stride)) & (int)mask; indices.Add(val); flatEx.feature_map[i].weight_index = (uint)val; } } VowpalWabbitInterface.FreeFlattenExample(flatec); VowpalWabbitInterface.FinishExample(vw, example); } VowpalWabbitInterface.EndParser(vw); VowpalWabbitInterface.Finish(vw); }