From 358fd571a03c475751c23ba60d156b220c548126 Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Mon, 29 Jul 2024 11:05:45 +0300 Subject: [PATCH 01/13] Implemented SVM (support vector machine) algorithm --- ml/svm.v | 230 ++++++++++++++++++++++++++++++++++++++++++++++++++ ml/svm_test.v | 105 +++++++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 ml/svm.v create mode 100644 ml/svm_test.v diff --git a/ml/svm.v b/ml/svm.v new file mode 100644 index 000000000..2332b12ac --- /dev/null +++ b/ml/svm.v @@ -0,0 +1,230 @@ +module ml + +import math +import rand + +pub struct SVMConfig { +pub mut: + max_iterations int = 1000 + learning_rate f64 = 0.01 + tolerance f64 = 1e-6 + c f64 = 1.0 // Regularization parameter +} + +pub struct DataPoint { +pub mut: + x []f64 + y int +} + +pub struct SVMModel { +pub mut: + support_vectors []DataPoint + alphas []f64 + b f64 + kernel KernelFunction @[required] + config SVMConfig +} + +type KernelFunction = fn ([]f64, []f64) f64 + +pub fn linear_kernel(x []f64, y []f64) f64 { + return dot_product(x, y) +} + +pub fn polynomial_kernel(degree int) KernelFunction { + return fn [degree] (x []f64, y []f64) f64 { + return math.pow(dot_product(x, y) + 1.0, f64(degree)) + } +} + +pub fn rbf_kernel(gamma f64) KernelFunction { + return fn [gamma] (x []f64, y []f64) f64 { + diff := vector_subtract(x, y) + return math.exp(-gamma * dot_product(diff, diff)) + } +} + +fn dot_product(a []f64, b []f64) f64 { + mut sum := 0.0 + for i in 0 .. a.len { + sum += a[i] * b[i] + } + return sum +} + +fn vector_subtract(a []f64, b []f64) []f64 { + mut result := []f64{len: a.len} + for i in 0 .. a.len { + result[i] = a[i] - b[i] + } + return result +} + +pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVMModel { + mut model := &SVMModel{ + support_vectors: []DataPoint{} + alphas: []f64{len: data.len, init: 0.0} + b: 0.0 + kernel: kernel + config: config + } + + mut passes := 0 + for { + mut num_changed_alphas := 0 + for i in 0 .. data.len { + ei := predict_raw(model, data[i].x) - f64(data[i].y) + if (data[i].y * ei < -model.config.tolerance && model.alphas[i] < model.config.c) + || (data[i].y * ei > model.config.tolerance && model.alphas[i] > 0) { + j := rand.int_in_range(0, data.len - 1) or { panic(err) } + ej := predict_raw(model, data[j].x) - f64(data[j].y) + + alpha_i_old := model.alphas[i] + alpha_j_old := model.alphas[j] + + mut l, mut h := 0.0, 0.0 + if data[i].y != data[j].y { + l = math.max(0.0, model.alphas[j] - model.alphas[i]) + h = math.min(model.config.c, model.config.c + model.alphas[j] - model.alphas[i]) + } else { + l = math.max(0.0, model.alphas[i] + model.alphas[j] - model.config.c) + h = math.min(model.config.c, model.alphas[i] + model.alphas[j]) + } + + if l == h { + continue + } + + eta := 2 * model.kernel(data[i].x, data[j].x) - model.kernel(data[i].x, + data[i].x) - model.kernel(data[j].x, data[j].x) + + if eta >= 0 { + continue + } + + model.alphas[j] = alpha_j_old - f64(data[j].y) * (ei - ej) / eta + model.alphas[j] = math.max(l, math.min(h, model.alphas[j])) + + if math.abs(model.alphas[j] - alpha_j_old) < 1e-5 { + continue + } + + model.alphas[i] = alpha_i_old + + f64(data[i].y * data[j].y) * (alpha_j_old - model.alphas[j]) + + b1 := model.b - ei - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, + data[i].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[i].x, + data[j].x) + + b2 := model.b - ej - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, + data[j].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[j].x, + data[j].x) + + if 0 < model.alphas[i] && model.alphas[i] < model.config.c { + model.b = b1 + } else if 0 < model.alphas[j] && model.alphas[j] < model.config.c { + model.b = b2 + } else { + model.b = (b1 + b2) / 2 + } + + num_changed_alphas++ + } + } + + if num_changed_alphas == 0 { + passes++ + } else { + passes = 0 + } + + if passes >= model.config.max_iterations { + break + } + } + + for i in 0 .. data.len { + if model.alphas[i] > 0 { + model.support_vectors << data[i] + } + } + + return model +} + +fn predict_raw(model &SVMModel, x []f64) f64 { + mut sum := 0.0 + for i, sv in model.support_vectors { + sum += model.alphas[i] * f64(sv.y) * model.kernel(x, sv.x) + } + return sum + model.b +} + +pub fn predict(model &SVMModel, x []f64) int { + return if predict_raw(model, x) >= 0 { 1 } else { -1 } +} + +pub struct MulticlassSVM { +pub mut: + models [][]&SVMModel + classes []int +} + +pub fn train_multiclass_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &MulticlassSVM { + mut classes := []int{} + for point in data { + if point.y !in classes { + classes << point.y + } + } + classes.sort() + + mut models := [][]&SVMModel{len: classes.len, init: []&SVMModel{}} + + for i in 0 .. classes.len { + models[i] = []&SVMModel{len: classes.len, init: unsafe { nil }} // unsafe { nil } kullanarak initialize ediyoruz + for j in i + 1 .. classes.len { + mut binary_data := []DataPoint{} + for point in data { + if point.y == classes[i] || point.y == classes[j] { + binary_data << DataPoint{ + x: point.x + y: if point.y == classes[i] { 1 } else { -1 } + } + } + } + models[i][j] = train_svm(binary_data, kernel, config) + } + } + + return &MulticlassSVM{ + models: models + classes: classes + } +} + +pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { + mut votes := map[int]int{} + for i in 0 .. model.classes.len { + for j in i + 1 .. model.classes.len { + prediction := predict(model.models[i][j], x) + if prediction == 1 { + votes[model.classes[i]]++ + } else { + votes[model.classes[j]]++ + } + } + } + + mut max_votes := 0 + mut predicted_class := 0 + for class, vote_count in votes { + if vote_count > max_votes { + max_votes = vote_count + predicted_class = class + } + } + + return predicted_class +} diff --git a/ml/svm_test.v b/ml/svm_test.v new file mode 100644 index 000000000..20dc26e8d --- /dev/null +++ b/ml/svm_test.v @@ -0,0 +1,105 @@ +module ml + +import math + +fn test_polynomial_kernel() { + x := [1.0, 2.0, 3.0] + y := [4.0, 5.0, 6.0] + kernel := polynomial_kernel(3) + result := kernel(x, y) + expected := math.pow(1 * 4 + 2 * 5 + 3 * 6 + 1, 3) // (32 + 1)^3 + assert result == expected +} + +fn test_rbf_kernel() { + x := [1.0, 2.0, 3.0] + y := [4.0, 5.0, 6.0] + gamma := 0.5 + kernel := rbf_kernel(gamma) + result := kernel(x, y) + expected := math.exp(-gamma * ((1 - 4) * (1 - 4) + (2 - 5) * (2 - 5) + (3 - 6) * (3 - 6))) // exp(-0.5 * 27) + assert math.abs(result - expected) < 1e-6 +} + +fn test_train_svm() { + kernel := linear_kernel + data := [ + DataPoint{[2.0, 3.0], 1}, + DataPoint{[1.0, 1.0], -1}, + DataPoint{[3.0, 4.0], 1}, + DataPoint{[0.0, 0.0], -1}, + ] + config := SVMConfig{} + model := train_svm(data, kernel, config) + + for point in data { + assert predict(model, point.x) == point.y + } +} + +fn test_predict_svm() { + kernel := linear_kernel + data := [ + DataPoint{[2.0, 3.0], 1}, + DataPoint{[1.0, 1.0], -1}, + DataPoint{[3.0, 4.0], 1}, + DataPoint{[0.0, 0.0], -1}, + ] + config := SVMConfig{} + model := train_svm(data, kernel, config) + + assert predict(model, [2.0, 3.0]) == 1 + assert predict(model, [1.0, 1.0]) == -1 + assert predict(model, [3.0, 4.0]) == 1 + assert predict(model, [0.0, 0.0]) == -1 +} + +fn test_train_multiclass_svm() { + kernel := linear_kernel + data := [ + DataPoint{[2.0, 3.0], 1}, + DataPoint{[1.0, 1.0], 2}, + DataPoint{[3.0, 4.0], 1}, + DataPoint{[0.0, 0.0], 2}, + DataPoint{[3.0, 3.0], 3}, + ] + config := SVMConfig{} + model := train_multiclass_svm(data, kernel, config) + + for point in data { + assert predict_multiclass(model, point.x) == point.y + } +} + +fn test_predict_multiclass_svm() { + kernel := linear_kernel + data := [ + DataPoint{[2.0, 3.0], 1}, + DataPoint{[1.0, 1.0], 2}, + DataPoint{[3.0, 4.0], 1}, + DataPoint{[0.0, 0.0], 2}, + DataPoint{[3.0, 3.0], 3}, + ] + config := SVMConfig{} + model := train_multiclass_svm(data, kernel, config) + + assert predict_multiclass(model, [2.0, 3.0]) == 1 + assert predict_multiclass(model, [1.0, 1.0]) == 2 + assert predict_multiclass(model, [3.0, 4.0]) == 1 + assert predict_multiclass(model, [0.0, 0.0]) == 2 + assert predict_multiclass(model, [3.0, 3.0]) == 3 +} + +fn test_kernels() { + kernels := [ + linear_kernel, + polynomial_kernel(3), + rbf_kernel(0.5), + ] + for kernel in kernels { + test_train_svm() + test_predict_svm() + test_train_multiclass_svm() + test_predict_multiclass_svm() + } +} From 73acf44eccae2fcb451044cad64bf70b049ef0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=BCleyman=20Kaya?= <111174999+suleyman-kaya@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:16:57 +0300 Subject: [PATCH 02/13] Update svm.v --- ml/svm.v | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/ml/svm.v b/ml/svm.v index 2332b12ac..8132dfdd2 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -71,7 +71,7 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM } mut passes := 0 - for { + for passes < model.config.max_iterations { mut num_changed_alphas := 0 for i in 0 .. data.len { ei := predict_raw(model, data[i].x) - f64(data[i].y) @@ -96,8 +96,7 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM continue } - eta := 2 * model.kernel(data[i].x, data[j].x) - model.kernel(data[i].x, - data[i].x) - model.kernel(data[j].x, data[j].x) + eta := 2 * model.kernel(data[i].x, data[j].x) - model.kernel(data[i].x, data[i].x) - model.kernel(data[j].x, data[j].x) if eta >= 0 { continue @@ -110,16 +109,11 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM continue } - model.alphas[i] = alpha_i_old + - f64(data[i].y * data[j].y) * (alpha_j_old - model.alphas[j]) + model.alphas[i] = alpha_i_old + f64(data[i].y * data[j].y) * (alpha_j_old - model.alphas[j]) - b1 := model.b - ei - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, - data[i].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[i].x, - data[j].x) + b1 := model.b - ei - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, data[i].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[i].x, data[j].x) - b2 := model.b - ej - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, - data[j].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[j].x, - data[j].x) + b2 := model.b - ej - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, data[j].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[j].x, data[j].x) if 0 < model.alphas[i] && model.alphas[i] < model.config.c { model.b = b1 @@ -138,10 +132,6 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM } else { passes = 0 } - - if passes >= model.config.max_iterations { - break - } } for i in 0 .. data.len { @@ -205,23 +195,24 @@ pub fn train_multiclass_svm(data []DataPoint, kernel KernelFunction, config SVMC } pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { - mut votes := map[int]int{} + mut class_votes := map[int]int{} + for i in 0 .. model.classes.len { for j in i + 1 .. model.classes.len { prediction := predict(model.models[i][j], x) if prediction == 1 { - votes[model.classes[i]]++ + class_votes[model.classes[i]]++ } else { - votes[model.classes[j]]++ + class_votes[model.classes[j]]++ } } } mut max_votes := 0 mut predicted_class := 0 - for class, vote_count in votes { - if vote_count > max_votes { - max_votes = vote_count + for class, votes in class_votes { + if votes > max_votes { + max_votes = votes predicted_class = class } } From 0a27e541c476c182c6621b93e17cef52edd7a7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=BCleyman=20Kaya?= <111174999+suleyman-kaya@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:17:29 +0300 Subject: [PATCH 03/13] Update svm_test.v --- ml/svm_test.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml/svm_test.v b/ml/svm_test.v index 20dc26e8d..5e412bb0a 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -7,7 +7,7 @@ fn test_polynomial_kernel() { y := [4.0, 5.0, 6.0] kernel := polynomial_kernel(3) result := kernel(x, y) - expected := math.pow(1 * 4 + 2 * 5 + 3 * 6 + 1, 3) // (32 + 1)^3 + expected := math.pow((1 * 4 + 2 * 5 + 3 * 6 + 1), 3) // (32 + 1)^3 assert result == expected } From 726651f64e40af9a7037e79378a83d83bbc59137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=BCleyman=20Kaya?= <111174999+suleyman-kaya@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:24:44 +0300 Subject: [PATCH 04/13] run fmt -w svm.v --- ml/svm.v | 1 + 1 file changed, 1 insertion(+) diff --git a/ml/svm.v b/ml/svm.v index 8132dfdd2..fda0451d6 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -219,3 +219,4 @@ pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { return predicted_class } + From cb8f49bbb201f81e6bb7d4f93d0e8a0fa16cb5ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=BCleyman=20Kaya?= <111174999+suleyman-kaya@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:25:22 +0300 Subject: [PATCH 05/13] runfmt -w svm_test.v --- ml/svm_test.v | 1 + 1 file changed, 1 insertion(+) diff --git a/ml/svm_test.v b/ml/svm_test.v index 5e412bb0a..57a4738d3 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -103,3 +103,4 @@ fn test_kernels() { test_predict_multiclass_svm() } } + From aedcbf83307b396a1d0332f3afeae7837859c704 Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Mon, 29 Jul 2024 14:06:36 +0300 Subject: [PATCH 06/13] run `v fmt -w .` in the top of the project --- ml/svm.v | 15 ++++++++++----- ml/svm_test.v | 1 - 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ml/svm.v b/ml/svm.v index fda0451d6..026550091 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -96,7 +96,8 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM continue } - eta := 2 * model.kernel(data[i].x, data[j].x) - model.kernel(data[i].x, data[i].x) - model.kernel(data[j].x, data[j].x) + eta := 2 * model.kernel(data[i].x, data[j].x) - model.kernel(data[i].x, + data[i].x) - model.kernel(data[j].x, data[j].x) if eta >= 0 { continue @@ -109,11 +110,16 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM continue } - model.alphas[i] = alpha_i_old + f64(data[i].y * data[j].y) * (alpha_j_old - model.alphas[j]) + model.alphas[i] = alpha_i_old + + f64(data[i].y * data[j].y) * (alpha_j_old - model.alphas[j]) - b1 := model.b - ei - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, data[i].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[i].x, data[j].x) + b1 := model.b - ei - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, + data[i].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[i].x, + data[j].x) - b2 := model.b - ej - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, data[j].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[j].x, data[j].x) + b2 := model.b - ej - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, + data[j].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[j].x, + data[j].x) if 0 < model.alphas[i] && model.alphas[i] < model.config.c { model.b = b1 @@ -219,4 +225,3 @@ pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { return predicted_class } - diff --git a/ml/svm_test.v b/ml/svm_test.v index 57a4738d3..5e412bb0a 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -103,4 +103,3 @@ fn test_kernels() { test_predict_multiclass_svm() } } - From 66d1c58069997710be31df236129f40b46c71fe5 Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Mon, 29 Jul 2024 14:42:41 +0300 Subject: [PATCH 07/13] Update SVM files --- ml/svm.v | 121 +++++++++++++++++--------------------------------- ml/svm_test.v | 118 ++++++++++++++++++++++++++++-------------------- 2 files changed, 111 insertions(+), 128 deletions(-) diff --git a/ml/svm.v b/ml/svm.v index 026550091..143652f18 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -26,39 +26,61 @@ pub mut: config SVMConfig } +pub struct SVM { +pub mut: + model &SVMModel = unsafe { nil } + kernel KernelFunction @[required] + config SVMConfig +} + type KernelFunction = fn ([]f64, []f64) f64 +fn vector_dot(x []f64, y []f64) f64 { + mut sum := 0.0 + for i := 0; i < x.len; i++ { + sum += x[i] * y[i] + } + return sum +} + +fn vector_subtract(x []f64, y []f64) []f64 { + mut result := []f64{len: x.len} + for i := 0; i < x.len; i++ { + result[i] = x[i] - y[i] + } + return result +} + pub fn linear_kernel(x []f64, y []f64) f64 { - return dot_product(x, y) + return vector_dot(x, y) } pub fn polynomial_kernel(degree int) KernelFunction { return fn [degree] (x []f64, y []f64) f64 { - return math.pow(dot_product(x, y) + 1.0, f64(degree)) + return math.pow(vector_dot(x, y) + 1.0, f64(degree)) } } pub fn rbf_kernel(gamma f64) KernelFunction { return fn [gamma] (x []f64, y []f64) f64 { diff := vector_subtract(x, y) - return math.exp(-gamma * dot_product(diff, diff)) + return math.exp(-gamma * vector_dot(diff, diff)) } } -fn dot_product(a []f64, b []f64) f64 { - mut sum := 0.0 - for i in 0 .. a.len { - sum += a[i] * b[i] +pub fn SVM.new(kernel KernelFunction, config SVMConfig) &SVM { + return &SVM{ + kernel: kernel + config: config } - return sum } -fn vector_subtract(a []f64, b []f64) []f64 { - mut result := []f64{len: a.len} - for i in 0 .. a.len { - result[i] = a[i] - b[i] - } - return result +pub fn (mut s SVM) train(data []DataPoint) { + s.model = train_svm(data, s.kernel, s.config) +} + +pub fn (s &SVM) predict(x []f64) int { + return predict(s.model, x) } pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVMModel { @@ -71,7 +93,7 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM } mut passes := 0 - for passes < model.config.max_iterations { + for { mut num_changed_alphas := 0 for i in 0 .. data.len { ei := predict_raw(model, data[i].x) - f64(data[i].y) @@ -138,6 +160,10 @@ pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVM } else { passes = 0 } + + if passes >= model.config.max_iterations { + break + } } for i in 0 .. data.len { @@ -160,68 +186,3 @@ fn predict_raw(model &SVMModel, x []f64) f64 { pub fn predict(model &SVMModel, x []f64) int { return if predict_raw(model, x) >= 0 { 1 } else { -1 } } - -pub struct MulticlassSVM { -pub mut: - models [][]&SVMModel - classes []int -} - -pub fn train_multiclass_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &MulticlassSVM { - mut classes := []int{} - for point in data { - if point.y !in classes { - classes << point.y - } - } - classes.sort() - - mut models := [][]&SVMModel{len: classes.len, init: []&SVMModel{}} - - for i in 0 .. classes.len { - models[i] = []&SVMModel{len: classes.len, init: unsafe { nil }} // unsafe { nil } kullanarak initialize ediyoruz - for j in i + 1 .. classes.len { - mut binary_data := []DataPoint{} - for point in data { - if point.y == classes[i] || point.y == classes[j] { - binary_data << DataPoint{ - x: point.x - y: if point.y == classes[i] { 1 } else { -1 } - } - } - } - models[i][j] = train_svm(binary_data, kernel, config) - } - } - - return &MulticlassSVM{ - models: models - classes: classes - } -} - -pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { - mut class_votes := map[int]int{} - - for i in 0 .. model.classes.len { - for j in i + 1 .. model.classes.len { - prediction := predict(model.models[i][j], x) - if prediction == 1 { - class_votes[model.classes[i]]++ - } else { - class_votes[model.classes[j]]++ - } - } - } - - mut max_votes := 0 - mut predicted_class := 0 - for class, votes in class_votes { - if votes > max_votes { - max_votes = votes - predicted_class = class - } - } - - return predicted_class -} diff --git a/ml/svm_test.v b/ml/svm_test.v index 5e412bb0a..6235b4bc2 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -2,13 +2,34 @@ module ml import math +fn test_vector_dot() { + x := [1.0, 2.0, 3.0] + y := [4.0, 5.0, 6.0] + result := vector_dot(x, y) + assert math.abs(result - 32.0) < 1e-6 +} + +fn test_vector_subtract() { + x := [1.0, 2.0, 3.0] + y := [4.0, 5.0, 6.0] + result := vector_subtract(x, y) + assert result == [-3.0, -3.0, -3.0] +} + +fn test_linear_kernel() { + x := [1.0, 2.0, 3.0] + y := [4.0, 5.0, 6.0] + result := linear_kernel(x, y) + assert math.abs(result - 32.0) < 1e-6 +} + fn test_polynomial_kernel() { x := [1.0, 2.0, 3.0] y := [4.0, 5.0, 6.0] kernel := polynomial_kernel(3) result := kernel(x, y) - expected := math.pow((1 * 4 + 2 * 5 + 3 * 6 + 1), 3) // (32 + 1)^3 - assert result == expected + expected := math.pow(32.0 + 1.0, 3) + assert math.abs(result - expected) < 1e-6 } fn test_rbf_kernel() { @@ -17,28 +38,34 @@ fn test_rbf_kernel() { gamma := 0.5 kernel := rbf_kernel(gamma) result := kernel(x, y) - expected := math.exp(-gamma * ((1 - 4) * (1 - 4) + (2 - 5) * (2 - 5) + (3 - 6) * (3 - 6))) // exp(-0.5 * 27) + expected := math.exp(-gamma * 27.0) assert math.abs(result - expected) < 1e-6 } -fn test_train_svm() { - kernel := linear_kernel +fn test_svm_new() { + config := SVMConfig{} + svm := SVM.new(linear_kernel, config) + assert svm.kernel == linear_kernel + assert svm.config == config +} + +fn test_svm_train_and_predict() { + mut svm := SVM.new(linear_kernel, SVMConfig{}) data := [ DataPoint{[2.0, 3.0], 1}, DataPoint{[1.0, 1.0], -1}, DataPoint{[3.0, 4.0], 1}, DataPoint{[0.0, 0.0], -1}, ] - config := SVMConfig{} - model := train_svm(data, kernel, config) + svm.train(data) for point in data { - assert predict(model, point.x) == point.y + prediction := svm.predict(point.x) + assert prediction == point.y } } -fn test_predict_svm() { - kernel := linear_kernel +fn test_train_svm() { data := [ DataPoint{[2.0, 3.0], 1}, DataPoint{[1.0, 1.0], -1}, @@ -46,60 +73,55 @@ fn test_predict_svm() { DataPoint{[0.0, 0.0], -1}, ] config := SVMConfig{} - model := train_svm(data, kernel, config) + model := train_svm(data, linear_kernel, config) - assert predict(model, [2.0, 3.0]) == 1 - assert predict(model, [1.0, 1.0]) == -1 - assert predict(model, [3.0, 4.0]) == 1 - assert predict(model, [0.0, 0.0]) == -1 + for point in data { + prediction := predict(model, point.x) + assert prediction == point.y + } } -fn test_train_multiclass_svm() { - kernel := linear_kernel +fn test_predict_raw() { data := [ DataPoint{[2.0, 3.0], 1}, - DataPoint{[1.0, 1.0], 2}, - DataPoint{[3.0, 4.0], 1}, - DataPoint{[0.0, 0.0], 2}, - DataPoint{[3.0, 3.0], 3}, + DataPoint{[1.0, 1.0], -1}, ] config := SVMConfig{} - model := train_multiclass_svm(data, kernel, config) + model := train_svm(data, linear_kernel, config) - for point in data { - assert predict_multiclass(model, point.x) == point.y - } + result := predict_raw(model, [2.0, 3.0]) + assert result > 0 + + result2 := predict_raw(model, [1.0, 1.0]) + assert result2 < 0 } -fn test_predict_multiclass_svm() { - kernel := linear_kernel +fn test_predict() { data := [ DataPoint{[2.0, 3.0], 1}, - DataPoint{[1.0, 1.0], 2}, + DataPoint{[1.0, 1.0], -1}, DataPoint{[3.0, 4.0], 1}, - DataPoint{[0.0, 0.0], 2}, - DataPoint{[3.0, 3.0], 3}, + DataPoint{[0.0, 0.0], -1}, ] config := SVMConfig{} - model := train_multiclass_svm(data, kernel, config) + model := train_svm(data, linear_kernel, config) - assert predict_multiclass(model, [2.0, 3.0]) == 1 - assert predict_multiclass(model, [1.0, 1.0]) == 2 - assert predict_multiclass(model, [3.0, 4.0]) == 1 - assert predict_multiclass(model, [0.0, 0.0]) == 2 - assert predict_multiclass(model, [3.0, 3.0]) == 3 + for point in data { + prediction := predict(model, point.x) + assert prediction == point.y + } } -fn test_kernels() { - kernels := [ - linear_kernel, - polynomial_kernel(3), - rbf_kernel(0.5), - ] - for kernel in kernels { - test_train_svm() - test_predict_svm() - test_train_multiclass_svm() - test_predict_multiclass_svm() - } +fn main() { + test_vector_dot() + test_vector_subtract() + test_linear_kernel() + test_polynomial_kernel() + test_rbf_kernel() + test_svm_new() + test_svm_train_and_predict() + test_train_svm() + test_predict_raw() + test_predict() + println('All tests passed successfully!') } From 5346992978243c280d4950351a20f7d4900200ad Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Mon, 29 Jul 2024 16:08:02 +0300 Subject: [PATCH 08/13] Fix errors --- ml/svm.v | 162 ++++++++++---------------------------------------- ml/svm_test.v | 62 ++----------------- 2 files changed, 36 insertions(+), 188 deletions(-) diff --git a/ml/svm.v b/ml/svm.v index 143652f18..21b5e3396 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -1,14 +1,12 @@ module ml import math -import rand pub struct SVMConfig { pub mut: max_iterations int = 1000 learning_rate f64 = 0.01 tolerance f64 = 1e-6 - c f64 = 1.0 // Regularization parameter } pub struct DataPoint { @@ -19,170 +17,74 @@ pub mut: pub struct SVMModel { pub mut: - support_vectors []DataPoint - alphas []f64 - b f64 - kernel KernelFunction @[required] - config SVMConfig + weights []f64 + bias f64 + config SVMConfig } pub struct SVM { pub mut: model &SVMModel = unsafe { nil } - kernel KernelFunction @[required] config SVMConfig } -type KernelFunction = fn ([]f64, []f64) f64 - -fn vector_dot(x []f64, y []f64) f64 { - mut sum := 0.0 - for i := 0; i < x.len; i++ { - sum += x[i] * y[i] - } - return sum -} - -fn vector_subtract(x []f64, y []f64) []f64 { - mut result := []f64{len: x.len} - for i := 0; i < x.len; i++ { - result[i] = x[i] - y[i] - } - return result -} - -pub fn linear_kernel(x []f64, y []f64) f64 { - return vector_dot(x, y) -} - -pub fn polynomial_kernel(degree int) KernelFunction { - return fn [degree] (x []f64, y []f64) f64 { - return math.pow(vector_dot(x, y) + 1.0, f64(degree)) - } -} - -pub fn rbf_kernel(gamma f64) KernelFunction { - return fn [gamma] (x []f64, y []f64) f64 { - diff := vector_subtract(x, y) - return math.exp(-gamma * vector_dot(diff, diff)) - } -} - -pub fn SVM.new(kernel KernelFunction, config SVMConfig) &SVM { +pub fn SVM.new(config SVMConfig) &SVM { return &SVM{ - kernel: kernel config: config } } pub fn (mut s SVM) train(data []DataPoint) { - s.model = train_svm(data, s.kernel, s.config) + s.model = train_svm(data, s.config) } pub fn (s &SVM) predict(x []f64) int { return predict(s.model, x) } -pub fn train_svm(data []DataPoint, kernel KernelFunction, config SVMConfig) &SVMModel { +fn vector_dot(x []f64, y []f64) f64 { + mut sum := 0.0 + for i := 0; i < x.len; i++ { + sum += x[i] * y[i] + } + return sum +} + +pub fn train_svm(data []DataPoint, config SVMConfig) &SVMModel { mut model := &SVMModel{ - support_vectors: []DataPoint{} - alphas: []f64{len: data.len, init: 0.0} - b: 0.0 - kernel: kernel + weights: []f64{len: data[0].x.len, init: 0.0} + bias: 0.0 config: config } - mut passes := 0 - for { - mut num_changed_alphas := 0 - for i in 0 .. data.len { - ei := predict_raw(model, data[i].x) - f64(data[i].y) - if (data[i].y * ei < -model.config.tolerance && model.alphas[i] < model.config.c) - || (data[i].y * ei > model.config.tolerance && model.alphas[i] > 0) { - j := rand.int_in_range(0, data.len - 1) or { panic(err) } - ej := predict_raw(model, data[j].x) - f64(data[j].y) - - alpha_i_old := model.alphas[i] - alpha_j_old := model.alphas[j] + for _ in 0 .. config.max_iterations { + mut cost := 0.0 + for point in data { + prediction := vector_dot(model.weights, point.x) + model.bias + margin := f64(point.y) * prediction - mut l, mut h := 0.0, 0.0 - if data[i].y != data[j].y { - l = math.max(0.0, model.alphas[j] - model.alphas[i]) - h = math.min(model.config.c, model.config.c + model.alphas[j] - model.alphas[i]) - } else { - l = math.max(0.0, model.alphas[i] + model.alphas[j] - model.config.c) - h = math.min(model.config.c, model.alphas[i] + model.alphas[j]) + if margin < 1 { + for i in 0 .. model.weights.len { + model.weights[i] += config.learning_rate * (f64(point.y) * point.x[i] - 2 * config.tolerance * model.weights[i]) } - - if l == h { - continue - } - - eta := 2 * model.kernel(data[i].x, data[j].x) - model.kernel(data[i].x, - data[i].x) - model.kernel(data[j].x, data[j].x) - - if eta >= 0 { - continue - } - - model.alphas[j] = alpha_j_old - f64(data[j].y) * (ei - ej) / eta - model.alphas[j] = math.max(l, math.min(h, model.alphas[j])) - - if math.abs(model.alphas[j] - alpha_j_old) < 1e-5 { - continue - } - - model.alphas[i] = alpha_i_old + - f64(data[i].y * data[j].y) * (alpha_j_old - model.alphas[j]) - - b1 := model.b - ei - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, - data[i].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[i].x, - data[j].x) - - b2 := model.b - ej - f64(data[i].y) * (model.alphas[i] - alpha_i_old) * model.kernel(data[i].x, - data[j].x) - f64(data[j].y) * (model.alphas[j] - alpha_j_old) * model.kernel(data[j].x, - data[j].x) - - if 0 < model.alphas[i] && model.alphas[i] < model.config.c { - model.b = b1 - } else if 0 < model.alphas[j] && model.alphas[j] < model.config.c { - model.b = b2 - } else { - model.b = (b1 + b2) / 2 + model.bias += config.learning_rate * f64(point.y) + cost += 1 - margin + } else { + for i in 0 .. model.weights.len { + model.weights[i] -= config.learning_rate * 2 * config.tolerance * model.weights[i] } - - num_changed_alphas++ } } - if num_changed_alphas == 0 { - passes++ - } else { - passes = 0 - } - - if passes >= model.config.max_iterations { + if cost == 0 { break } } - for i in 0 .. data.len { - if model.alphas[i] > 0 { - model.support_vectors << data[i] - } - } - return model } -fn predict_raw(model &SVMModel, x []f64) f64 { - mut sum := 0.0 - for i, sv in model.support_vectors { - sum += model.alphas[i] * f64(sv.y) * model.kernel(x, sv.x) - } - return sum + model.b -} - pub fn predict(model &SVMModel, x []f64) int { - return if predict_raw(model, x) >= 0 { 1 } else { -1 } + prediction := vector_dot(model.weights, x) + model.bias + return if prediction >= 0 { 1 } else { -1 } } diff --git a/ml/svm_test.v b/ml/svm_test.v index 6235b4bc2..b813f573b 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -9,48 +9,14 @@ fn test_vector_dot() { assert math.abs(result - 32.0) < 1e-6 } -fn test_vector_subtract() { - x := [1.0, 2.0, 3.0] - y := [4.0, 5.0, 6.0] - result := vector_subtract(x, y) - assert result == [-3.0, -3.0, -3.0] -} - -fn test_linear_kernel() { - x := [1.0, 2.0, 3.0] - y := [4.0, 5.0, 6.0] - result := linear_kernel(x, y) - assert math.abs(result - 32.0) < 1e-6 -} - -fn test_polynomial_kernel() { - x := [1.0, 2.0, 3.0] - y := [4.0, 5.0, 6.0] - kernel := polynomial_kernel(3) - result := kernel(x, y) - expected := math.pow(32.0 + 1.0, 3) - assert math.abs(result - expected) < 1e-6 -} - -fn test_rbf_kernel() { - x := [1.0, 2.0, 3.0] - y := [4.0, 5.0, 6.0] - gamma := 0.5 - kernel := rbf_kernel(gamma) - result := kernel(x, y) - expected := math.exp(-gamma * 27.0) - assert math.abs(result - expected) < 1e-6 -} - fn test_svm_new() { config := SVMConfig{} - svm := SVM.new(linear_kernel, config) - assert svm.kernel == linear_kernel + svm := SVM.new(config) assert svm.config == config } fn test_svm_train_and_predict() { - mut svm := SVM.new(linear_kernel, SVMConfig{}) + mut svm := SVM.new(SVMConfig{}) data := [ DataPoint{[2.0, 3.0], 1}, DataPoint{[1.0, 1.0], -1}, @@ -73,7 +39,7 @@ fn test_train_svm() { DataPoint{[0.0, 0.0], -1}, ] config := SVMConfig{} - model := train_svm(data, linear_kernel, config) + model := train_svm(data, config) for point in data { prediction := predict(model, point.x) @@ -81,21 +47,6 @@ fn test_train_svm() { } } -fn test_predict_raw() { - data := [ - DataPoint{[2.0, 3.0], 1}, - DataPoint{[1.0, 1.0], -1}, - ] - config := SVMConfig{} - model := train_svm(data, linear_kernel, config) - - result := predict_raw(model, [2.0, 3.0]) - assert result > 0 - - result2 := predict_raw(model, [1.0, 1.0]) - assert result2 < 0 -} - fn test_predict() { data := [ DataPoint{[2.0, 3.0], 1}, @@ -104,7 +55,7 @@ fn test_predict() { DataPoint{[0.0, 0.0], -1}, ] config := SVMConfig{} - model := train_svm(data, linear_kernel, config) + model := train_svm(data, config) for point in data { prediction := predict(model, point.x) @@ -114,14 +65,9 @@ fn test_predict() { fn main() { test_vector_dot() - test_vector_subtract() - test_linear_kernel() - test_polynomial_kernel() - test_rbf_kernel() test_svm_new() test_svm_train_and_predict() test_train_svm() - test_predict_raw() test_predict() println('All tests passed successfully!') } From a7751c8f5dbef51492deebf1d10d684a4b6fdbda Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Mon, 29 Jul 2024 20:02:39 +0300 Subject: [PATCH 09/13] Update implementation and tests --- ml/svm.v | 276 +++++++++++++++++++++++++++++++++++++++----------- ml/svm_test.v | 192 ++++++++++++++++++++++++----------- 2 files changed, 353 insertions(+), 115 deletions(-) diff --git a/ml/svm.v b/ml/svm.v index 21b5e3396..301f835c6 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -1,90 +1,250 @@ module ml import math +import rand pub struct SVMConfig { pub mut: - max_iterations int = 1000 - learning_rate f64 = 0.01 - tolerance f64 = 1e-6 + max_iterations int = 1000 + learning_rate f64 = 0.01 + tolerance f64 = 1e-6 + c f64 = 1.0 + kernel_type KernelType = .linear + kernel_param f64 = 1.0 } +pub enum KernelType { + linear + polynomial + rbf + quadratic + custom +} + +pub type KernelFunction = fn ([]f64, []f64) f64 + pub struct DataPoint { pub mut: - x []f64 - y int + x []f64 + y int } pub struct SVMModel { pub mut: - weights []f64 - bias f64 - config SVMConfig + support_vectors []DataPoint + alphas []f64 + b f64 + kernel KernelFunction = linear_kernel + config SVMConfig } -pub struct SVM { +pub struct MulticlassSVM { pub mut: - model &SVMModel = unsafe { nil } - config SVMConfig + models [][]&SVMModel +} + +pub fn linear_kernel(x []f64, y []f64) f64 { + return dot_product(x, y) } -pub fn SVM.new(config SVMConfig) &SVM { - return &SVM{ - config: config - } +pub fn polynomial_kernel(degree f64) KernelFunction { + return fn [degree] (x []f64, y []f64) f64 { + return math.pow(dot_product(x, y) + 1.0, degree) + } } -pub fn (mut s SVM) train(data []DataPoint) { - s.model = train_svm(data, s.config) +pub fn rbf_kernel(gamma f64) KernelFunction { + return fn [gamma] (x []f64, y []f64) f64 { + diff := vector_subtract(x, y) + return math.exp(-gamma * dot_product(diff, diff)) + } } -pub fn (s &SVM) predict(x []f64) int { - return predict(s.model, x) +pub fn quadratic_kernel(x []f64, y []f64) f64 { + dot := dot_product(x, y) + return dot * dot } -fn vector_dot(x []f64, y []f64) f64 { - mut sum := 0.0 - for i := 0; i < x.len; i++ { - sum += x[i] * y[i] - } - return sum +pub fn custom_kernel(x []f64, y []f64) f64 { + z_x := math.pow(x[0], 2) + math.pow(x[1], 2) + z_y := math.pow(y[0], 2) + math.pow(y[1], 2) + return z_x * z_y +} + +fn dot_product(a []f64, b []f64) f64 { + mut sum := 0.0 + for i := 0; i < a.len; i++ { + sum += a[i] * b[i] + } + return sum +} + +fn vector_subtract(a []f64, b []f64) []f64 { + mut result := []f64{len: a.len} + for i := 0; i < a.len; i++ { + result[i] = a[i] - b[i] + } + return result } pub fn train_svm(data []DataPoint, config SVMConfig) &SVMModel { - mut model := &SVMModel{ - weights: []f64{len: data[0].x.len, init: 0.0} - bias: 0.0 - config: config - } - - for _ in 0 .. config.max_iterations { - mut cost := 0.0 - for point in data { - prediction := vector_dot(model.weights, point.x) + model.bias - margin := f64(point.y) * prediction - - if margin < 1 { - for i in 0 .. model.weights.len { - model.weights[i] += config.learning_rate * (f64(point.y) * point.x[i] - 2 * config.tolerance * model.weights[i]) - } - model.bias += config.learning_rate * f64(point.y) - cost += 1 - margin - } else { - for i in 0 .. model.weights.len { - model.weights[i] -= config.learning_rate * 2 * config.tolerance * model.weights[i] - } - } - } - - if cost == 0 { - break - } - } - - return model + kernel := match config.kernel_type { + .linear { linear_kernel } + .polynomial { polynomial_kernel(config.kernel_param) } + .rbf { rbf_kernel(config.kernel_param) } + .quadratic { quadratic_kernel } + .custom { custom_kernel } + } + + mut model := &SVMModel{ + config: config + kernel: kernel + } + + mut alphas := []f64{len: data.len, init: 0.0} + mut b := 0.0 + + for _ in 0 .. config.max_iterations { + mut alpha_pairs_changed := 0 + + for i := 0; i < data.len; i++ { + ei := predict_raw(model, data[i].x) - f64(data[i].y) + if (data[i].y * ei < -config.tolerance && alphas[i] < config.c) || + (data[i].y * ei > config.tolerance && alphas[i] > 0) { + mut j := rand.intn(data.len - 1) or { 0 } + if j >= i { + j += 1 + } + + ej := predict_raw(model, data[j].x) - f64(data[j].y) + + old_alpha_i, old_alpha_j := alphas[i], alphas[j] + l, h := compute_l_h(config.c, alphas[i], alphas[j], data[i].y, data[j].y) + + if l == h { + continue + } + + eta := 2 * kernel(data[i].x, data[j].x) - kernel(data[i].x, data[i].x) - kernel(data[j].x, data[j].x) + if eta >= 0 { + continue + } + + alphas[j] -= f64(data[j].y) * (ei - ej) / eta + alphas[j] = math.max(l, math.min(h, alphas[j])) + + if math.abs(alphas[j] - old_alpha_j) < 1e-5 { + continue + } + + alphas[i] += f64(data[i].y * data[j].y) * (old_alpha_j - alphas[j]) + + b1 := b - ei - data[i].y * (alphas[i] - old_alpha_i) * kernel(data[i].x, data[i].x) - + data[j].y * (alphas[j] - old_alpha_j) * kernel(data[i].x, data[j].x) + b2 := b - ej - data[i].y * (alphas[i] - old_alpha_i) * kernel(data[i].x, data[j].x) - + data[j].y * (alphas[j] - old_alpha_j) * kernel(data[j].x, data[j].x) + + if 0 < alphas[i] && alphas[i] < config.c { + b = b1 + } else if 0 < alphas[j] && alphas[j] < config.c { + b = b2 + } else { + b = (b1 + b2) / 2 + } + + alpha_pairs_changed += 1 + } + } + + if alpha_pairs_changed == 0 { + break + } + } + + model.b = b + model.alphas = alphas + mut support_vectors := []DataPoint{} + for i, d in data { + if alphas[i] > 0 { + support_vectors << d + } + } + model.support_vectors = support_vectors + + return model +} + +fn compute_l_h(c f64, alpha_i f64, alpha_j f64, y_i int, y_j int) (f64, f64) { + if y_i != y_j { + return math.max(0.0, alpha_j - alpha_i), math.min(c, c + alpha_j - alpha_i) + } else { + return math.max(0.0, alpha_i + alpha_j - c), math.min(c, alpha_i + alpha_j) + } +} + +pub fn predict_raw(model &SVMModel, x []f64) f64 { + mut result := 0.0 + for i, sv in model.support_vectors { + result += model.alphas[i] * f64(sv.y) * model.kernel(x, sv.x) + } + return result + model.b } pub fn predict(model &SVMModel, x []f64) int { - prediction := vector_dot(model.weights, x) + model.bias - return if prediction >= 0 { 1 } else { -1 } + return if predict_raw(model, x) >= 0 { 1 } else { -1 } } + +pub fn train_multiclass_svm(data []DataPoint, config SVMConfig) &MulticlassSVM { + mut classes := []int{} + for point in data { + if point.y !in classes { + classes << point.y + } + } + + mut models := [][]&SVMModel{len: classes.len, init: []&SVMModel{}} + + for i := 0; i < classes.len; i++ { + models[i] = []&SVMModel{len: classes.len, init: 0} + for j := i + 1; j < classes.len; j++ { + mut binary_data := []DataPoint{} + for point in data { + if point.y == classes[i] || point.y == classes[j] { + binary_data << DataPoint{ + x: point.x + y: if point.y == classes[i] { 1 } else { -1 } + } + } + } + models[i][j] = train_svm(binary_data, config) + } + } + + return &MulticlassSVM{models: models} +} + +pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { + mut votes := map[int]int{} + for i := 0; i < model.models.len; i++ { + for j := i + 1; j < model.models.len; j++ { + prediction := predict(model.models[i][j], x) + if prediction == 1 { + votes[i]++ + } else { + votes[j]++ + } + } + } + + mut max_votes := 0 + mut predicted_class := 0 + for class, vote_count in votes { + if vote_count > max_votes { + max_votes = vote_count + predicted_class = class + } + } + + return predicted_class +} + diff --git a/ml/svm_test.v b/ml/svm_test.v index b813f573b..558882f4f 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -1,73 +1,151 @@ module ml +import rand import math -fn test_vector_dot() { - x := [1.0, 2.0, 3.0] - y := [4.0, 5.0, 6.0] - result := vector_dot(x, y) - assert math.abs(result - 32.0) < 1e-6 +fn test_linear_kernel() { + x := [1.0, 2.0] + y := [2.0, 3.0] + result := ml.linear_kernel(x, y) + expected := ml.dot_product(x, y) + assert math.abs(result - expected) < 1e-6 + println('Linear kernel test passed.') } -fn test_svm_new() { - config := SVMConfig{} - svm := SVM.new(config) - assert svm.config == config +fn test_polynomial_kernel() { + x := [1.0, 2.0] + y := [2.0, 3.0] + degree := 2.0 + kernel_fn := ml.polynomial_kernel(degree) + result := kernel_fn(x, y) + expected := math.pow(ml.dot_product(x, y) + 1.0, degree) + assert math.abs(result - expected) < 1e-6 + println('Polynomial kernel test passed.') } -fn test_svm_train_and_predict() { - mut svm := SVM.new(SVMConfig{}) - data := [ - DataPoint{[2.0, 3.0], 1}, - DataPoint{[1.0, 1.0], -1}, - DataPoint{[3.0, 4.0], 1}, - DataPoint{[0.0, 0.0], -1}, - ] - svm.train(data) - - for point in data { - prediction := svm.predict(point.x) - assert prediction == point.y - } +fn test_rbf_kernel() { + x := [1.0, 2.0] + y := [2.0, 3.0] + gamma := 0.5 + kernel_fn := ml.rbf_kernel(gamma) + result := kernel_fn(x, y) + diff := ml.vector_subtract(x, y) + expected := math.exp(-gamma * ml.dot_product(diff, diff)) + assert math.abs(result - expected) < 1e-6 + println('RBF kernel test passed.') } -fn test_train_svm() { - data := [ - DataPoint{[2.0, 3.0], 1}, - DataPoint{[1.0, 1.0], -1}, - DataPoint{[3.0, 4.0], 1}, - DataPoint{[0.0, 0.0], -1}, - ] - config := SVMConfig{} - model := train_svm(data, config) - - for point in data { - prediction := predict(model, point.x) - assert prediction == point.y - } +fn test_quadratic_kernel() { + x := [1.0, 2.0] + y := [2.0, 3.0] + result := ml.quadratic_kernel(x, y) + expected := math.pow(ml.dot_product(x, y), 2) + assert math.abs(result - expected) < 1e-6 + println('Quadratic kernel test passed.') } -fn test_predict() { - data := [ - DataPoint{[2.0, 3.0], 1}, - DataPoint{[1.0, 1.0], -1}, - DataPoint{[3.0, 4.0], 1}, - DataPoint{[0.0, 0.0], -1}, - ] - config := SVMConfig{} - model := train_svm(data, config) - - for point in data { - prediction := predict(model, point.x) - assert prediction == point.y - } +fn test_custom_kernel() { + x := [1.0, 2.0] + y := [2.0, 3.0] + result := ml.custom_kernel(x, y) + z_x := math.pow(x[0], 2) + math.pow(x[1], 2) + z_y := math.pow(y[0], 2) + math.pow(y[1], 2) + expected := z_x * z_y + assert math.abs(result - expected) < 1e-6 + println('Custom kernel test passed.') +} + +fn test_dot_product() { + a := [1.0, 2.0] + b := [3.0, 4.0] + result := ml.dot_product(a, b) + expected := 11.0 + assert math.abs(result - expected) < 1e-6 + println('Dot product test passed.') +} + +fn test_vector_subtract() { + a := [1.0, 2.0] + b := [3.0, 4.0] + result := ml.vector_subtract(a, b) + expected := [-2.0, -2.0] + assert result == expected + println('Vector subtract test passed.') +} + +fn test_svm() { + data := [ + ml.DataPoint{x: [2.0, 3.0], y: 1}, + ml.DataPoint{x: [1.0, 2.0], y: -1}, + ml.DataPoint{x: [3.0, 3.0], y: 1}, + ml.DataPoint{x: [2.0, 1.0], y: -1}, + ] + + config := ml.SVMConfig{ + max_iterations: 100, + learning_rate: 0.01, + tolerance: 1e-5, + c: 1.0, + kernel_type: .linear, + } + + model := ml.train_svm(data, config) + + mut predictions := 0 + for point in data { + pred := ml.predict(model, point.x) + if pred == point.y { + predictions += 1 + } + } + + assert predictions == data.len + + println('SVM model training and prediction test passed.') +} + +fn test_multiclass_svm() { + data := [ + ml.DataPoint{x: [1.0, 2.0], y: 0}, + ml.DataPoint{x: [2.0, 3.0], y: 1}, + ml.DataPoint{x: [3.0, 1.0], y: 2}, + ml.DataPoint{x: [1.5, 2.5], y: 0}, + ml.DataPoint{x: [2.5, 3.5], y: 1}, + ml.DataPoint{x: [3.5, 1.5], y: 2}, + ] + + config := ml.SVMConfig{ + max_iterations: 100, + learning_rate: 0.01, + tolerance: 1e-5, + c: 1.0, + kernel_type: .linear, + } + + multiclass_model := ml.train_multiclass_svm(data, config) + + mut correct_predictions := 0 + for point in data { + predicted_class := ml.predict_multiclass(multiclass_model, point.x) + if predicted_class == point.y { + correct_predictions += 1 + } + } + + assert correct_predictions == data.len + + println('Multiclass SVM model training and prediction test passed.') } fn main() { - test_vector_dot() - test_svm_new() - test_svm_train_and_predict() - test_train_svm() - test_predict() - println('All tests passed successfully!') + test_linear_kernel() + test_polynomial_kernel() + test_rbf_kernel() + test_quadratic_kernel() + test_custom_kernel() + test_dot_product() + test_vector_subtract() + test_svm() + test_multiclass_svm() } + From 0bc135169159caa08434d52710976993779fbbd8 Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Mon, 29 Jul 2024 20:03:51 +0300 Subject: [PATCH 10/13] run v fmt -w . --- ml/svm.v | 376 +++++++++++++++++++++++++------------------------- ml/svm_test.v | 263 +++++++++++++++++++---------------- 2 files changed, 336 insertions(+), 303 deletions(-) diff --git a/ml/svm.v b/ml/svm.v index 301f835c6..e858a5bf6 100644 --- a/ml/svm.v +++ b/ml/svm.v @@ -5,246 +5,250 @@ import rand pub struct SVMConfig { pub mut: - max_iterations int = 1000 - learning_rate f64 = 0.01 - tolerance f64 = 1e-6 - c f64 = 1.0 - kernel_type KernelType = .linear - kernel_param f64 = 1.0 + max_iterations int = 1000 + learning_rate f64 = 0.01 + tolerance f64 = 1e-6 + c f64 = 1.0 + kernel_type KernelType = .linear + kernel_param f64 = 1.0 } pub enum KernelType { - linear - polynomial - rbf - quadratic - custom + linear + polynomial + rbf + quadratic + custom } pub type KernelFunction = fn ([]f64, []f64) f64 pub struct DataPoint { pub mut: - x []f64 - y int + x []f64 + y int } pub struct SVMModel { pub mut: - support_vectors []DataPoint - alphas []f64 - b f64 - kernel KernelFunction = linear_kernel - config SVMConfig + support_vectors []DataPoint + alphas []f64 + b f64 + kernel KernelFunction = linear_kernel + config SVMConfig } pub struct MulticlassSVM { pub mut: - models [][]&SVMModel + models [][]&SVMModel } pub fn linear_kernel(x []f64, y []f64) f64 { - return dot_product(x, y) + return dot_product(x, y) } pub fn polynomial_kernel(degree f64) KernelFunction { - return fn [degree] (x []f64, y []f64) f64 { - return math.pow(dot_product(x, y) + 1.0, degree) - } + return fn [degree] (x []f64, y []f64) f64 { + return math.pow(dot_product(x, y) + 1.0, degree) + } } pub fn rbf_kernel(gamma f64) KernelFunction { - return fn [gamma] (x []f64, y []f64) f64 { - diff := vector_subtract(x, y) - return math.exp(-gamma * dot_product(diff, diff)) - } + return fn [gamma] (x []f64, y []f64) f64 { + diff := vector_subtract(x, y) + return math.exp(-gamma * dot_product(diff, diff)) + } } pub fn quadratic_kernel(x []f64, y []f64) f64 { - dot := dot_product(x, y) - return dot * dot + dot := dot_product(x, y) + return dot * dot } pub fn custom_kernel(x []f64, y []f64) f64 { - z_x := math.pow(x[0], 2) + math.pow(x[1], 2) - z_y := math.pow(y[0], 2) + math.pow(y[1], 2) - return z_x * z_y + z_x := math.pow(x[0], 2) + math.pow(x[1], 2) + z_y := math.pow(y[0], 2) + math.pow(y[1], 2) + return z_x * z_y } fn dot_product(a []f64, b []f64) f64 { - mut sum := 0.0 - for i := 0; i < a.len; i++ { - sum += a[i] * b[i] - } - return sum + mut sum := 0.0 + for i := 0; i < a.len; i++ { + sum += a[i] * b[i] + } + return sum } fn vector_subtract(a []f64, b []f64) []f64 { - mut result := []f64{len: a.len} - for i := 0; i < a.len; i++ { - result[i] = a[i] - b[i] - } - return result + mut result := []f64{len: a.len} + for i := 0; i < a.len; i++ { + result[i] = a[i] - b[i] + } + return result } pub fn train_svm(data []DataPoint, config SVMConfig) &SVMModel { - kernel := match config.kernel_type { - .linear { linear_kernel } - .polynomial { polynomial_kernel(config.kernel_param) } - .rbf { rbf_kernel(config.kernel_param) } - .quadratic { quadratic_kernel } - .custom { custom_kernel } - } - - mut model := &SVMModel{ - config: config - kernel: kernel - } - - mut alphas := []f64{len: data.len, init: 0.0} - mut b := 0.0 - - for _ in 0 .. config.max_iterations { - mut alpha_pairs_changed := 0 - - for i := 0; i < data.len; i++ { - ei := predict_raw(model, data[i].x) - f64(data[i].y) - if (data[i].y * ei < -config.tolerance && alphas[i] < config.c) || - (data[i].y * ei > config.tolerance && alphas[i] > 0) { - mut j := rand.intn(data.len - 1) or { 0 } - if j >= i { - j += 1 - } - - ej := predict_raw(model, data[j].x) - f64(data[j].y) - - old_alpha_i, old_alpha_j := alphas[i], alphas[j] - l, h := compute_l_h(config.c, alphas[i], alphas[j], data[i].y, data[j].y) - - if l == h { - continue - } - - eta := 2 * kernel(data[i].x, data[j].x) - kernel(data[i].x, data[i].x) - kernel(data[j].x, data[j].x) - if eta >= 0 { - continue - } - - alphas[j] -= f64(data[j].y) * (ei - ej) / eta - alphas[j] = math.max(l, math.min(h, alphas[j])) - - if math.abs(alphas[j] - old_alpha_j) < 1e-5 { - continue - } - - alphas[i] += f64(data[i].y * data[j].y) * (old_alpha_j - alphas[j]) - - b1 := b - ei - data[i].y * (alphas[i] - old_alpha_i) * kernel(data[i].x, data[i].x) - - data[j].y * (alphas[j] - old_alpha_j) * kernel(data[i].x, data[j].x) - b2 := b - ej - data[i].y * (alphas[i] - old_alpha_i) * kernel(data[i].x, data[j].x) - - data[j].y * (alphas[j] - old_alpha_j) * kernel(data[j].x, data[j].x) - - if 0 < alphas[i] && alphas[i] < config.c { - b = b1 - } else if 0 < alphas[j] && alphas[j] < config.c { - b = b2 - } else { - b = (b1 + b2) / 2 - } - - alpha_pairs_changed += 1 - } - } - - if alpha_pairs_changed == 0 { - break - } - } - - model.b = b - model.alphas = alphas - mut support_vectors := []DataPoint{} - for i, d in data { - if alphas[i] > 0 { - support_vectors << d - } - } - model.support_vectors = support_vectors - - return model + kernel := match config.kernel_type { + .linear { linear_kernel } + .polynomial { polynomial_kernel(config.kernel_param) } + .rbf { rbf_kernel(config.kernel_param) } + .quadratic { quadratic_kernel } + .custom { custom_kernel } + } + + mut model := &SVMModel{ + config: config + kernel: kernel + } + + mut alphas := []f64{len: data.len, init: 0.0} + mut b := 0.0 + + for _ in 0 .. config.max_iterations { + mut alpha_pairs_changed := 0 + + for i := 0; i < data.len; i++ { + ei := predict_raw(model, data[i].x) - f64(data[i].y) + if (data[i].y * ei < -config.tolerance && alphas[i] < config.c) + || (data[i].y * ei > config.tolerance && alphas[i] > 0) { + mut j := rand.intn(data.len - 1) or { 0 } + if j >= i { + j += 1 + } + + ej := predict_raw(model, data[j].x) - f64(data[j].y) + + old_alpha_i, old_alpha_j := alphas[i], alphas[j] + l, h := compute_l_h(config.c, alphas[i], alphas[j], data[i].y, data[j].y) + + if l == h { + continue + } + + eta := 2 * kernel(data[i].x, data[j].x) - kernel(data[i].x, data[i].x) - kernel(data[j].x, + data[j].x) + if eta >= 0 { + continue + } + + alphas[j] -= f64(data[j].y) * (ei - ej) / eta + alphas[j] = math.max(l, math.min(h, alphas[j])) + + if math.abs(alphas[j] - old_alpha_j) < 1e-5 { + continue + } + + alphas[i] += f64(data[i].y * data[j].y) * (old_alpha_j - alphas[j]) + + b1 := b - ei - data[i].y * (alphas[i] - old_alpha_i) * kernel(data[i].x, + data[i].x) - data[j].y * (alphas[j] - old_alpha_j) * kernel(data[i].x, + data[j].x) + b2 := b - ej - data[i].y * (alphas[i] - old_alpha_i) * kernel(data[i].x, + data[j].x) - data[j].y * (alphas[j] - old_alpha_j) * kernel(data[j].x, + data[j].x) + + if 0 < alphas[i] && alphas[i] < config.c { + b = b1 + } else if 0 < alphas[j] && alphas[j] < config.c { + b = b2 + } else { + b = (b1 + b2) / 2 + } + + alpha_pairs_changed += 1 + } + } + + if alpha_pairs_changed == 0 { + break + } + } + + model.b = b + model.alphas = alphas + mut support_vectors := []DataPoint{} + for i, d in data { + if alphas[i] > 0 { + support_vectors << d + } + } + model.support_vectors = support_vectors + + return model } fn compute_l_h(c f64, alpha_i f64, alpha_j f64, y_i int, y_j int) (f64, f64) { - if y_i != y_j { - return math.max(0.0, alpha_j - alpha_i), math.min(c, c + alpha_j - alpha_i) - } else { - return math.max(0.0, alpha_i + alpha_j - c), math.min(c, alpha_i + alpha_j) - } + if y_i != y_j { + return math.max(0.0, alpha_j - alpha_i), math.min(c, c + alpha_j - alpha_i) + } else { + return math.max(0.0, alpha_i + alpha_j - c), math.min(c, alpha_i + alpha_j) + } } pub fn predict_raw(model &SVMModel, x []f64) f64 { - mut result := 0.0 - for i, sv in model.support_vectors { - result += model.alphas[i] * f64(sv.y) * model.kernel(x, sv.x) - } - return result + model.b + mut result := 0.0 + for i, sv in model.support_vectors { + result += model.alphas[i] * f64(sv.y) * model.kernel(x, sv.x) + } + return result + model.b } pub fn predict(model &SVMModel, x []f64) int { - return if predict_raw(model, x) >= 0 { 1 } else { -1 } + return if predict_raw(model, x) >= 0 { 1 } else { -1 } } pub fn train_multiclass_svm(data []DataPoint, config SVMConfig) &MulticlassSVM { - mut classes := []int{} - for point in data { - if point.y !in classes { - classes << point.y - } - } - - mut models := [][]&SVMModel{len: classes.len, init: []&SVMModel{}} - - for i := 0; i < classes.len; i++ { - models[i] = []&SVMModel{len: classes.len, init: 0} - for j := i + 1; j < classes.len; j++ { - mut binary_data := []DataPoint{} - for point in data { - if point.y == classes[i] || point.y == classes[j] { - binary_data << DataPoint{ - x: point.x - y: if point.y == classes[i] { 1 } else { -1 } - } - } - } - models[i][j] = train_svm(binary_data, config) - } - } - - return &MulticlassSVM{models: models} + mut classes := []int{} + for point in data { + if point.y !in classes { + classes << point.y + } + } + + mut models := [][]&SVMModel{len: classes.len, init: []&SVMModel{}} + + for i := 0; i < classes.len; i++ { + models[i] = []&SVMModel{len: classes.len, init: 0} + for j := i + 1; j < classes.len; j++ { + mut binary_data := []DataPoint{} + for point in data { + if point.y == classes[i] || point.y == classes[j] { + binary_data << DataPoint{ + x: point.x + y: if point.y == classes[i] { 1 } else { -1 } + } + } + } + models[i][j] = train_svm(binary_data, config) + } + } + + return &MulticlassSVM{ + models: models + } } pub fn predict_multiclass(model &MulticlassSVM, x []f64) int { - mut votes := map[int]int{} - for i := 0; i < model.models.len; i++ { - for j := i + 1; j < model.models.len; j++ { - prediction := predict(model.models[i][j], x) - if prediction == 1 { - votes[i]++ - } else { - votes[j]++ - } - } - } - - mut max_votes := 0 - mut predicted_class := 0 - for class, vote_count in votes { - if vote_count > max_votes { - max_votes = vote_count - predicted_class = class - } - } - - return predicted_class + mut votes := map[int]int{} + for i := 0; i < model.models.len; i++ { + for j := i + 1; j < model.models.len; j++ { + prediction := predict(model.models[i][j], x) + if prediction == 1 { + votes[i]++ + } else { + votes[j]++ + } + } + } + + mut max_votes := 0 + mut predicted_class := 0 + for class, vote_count in votes { + if vote_count > max_votes { + max_votes = vote_count + predicted_class = class + } + } + + return predicted_class } - diff --git a/ml/svm_test.v b/ml/svm_test.v index 558882f4f..517d51790 100644 --- a/ml/svm_test.v +++ b/ml/svm_test.v @@ -4,148 +4,177 @@ import rand import math fn test_linear_kernel() { - x := [1.0, 2.0] - y := [2.0, 3.0] - result := ml.linear_kernel(x, y) - expected := ml.dot_product(x, y) - assert math.abs(result - expected) < 1e-6 - println('Linear kernel test passed.') + x := [1.0, 2.0] + y := [2.0, 3.0] + result := linear_kernel(x, y) + expected := dot_product(x, y) + assert math.abs(result - expected) < 1e-6 + println('Linear kernel test passed.') } fn test_polynomial_kernel() { - x := [1.0, 2.0] - y := [2.0, 3.0] - degree := 2.0 - kernel_fn := ml.polynomial_kernel(degree) - result := kernel_fn(x, y) - expected := math.pow(ml.dot_product(x, y) + 1.0, degree) - assert math.abs(result - expected) < 1e-6 - println('Polynomial kernel test passed.') + x := [1.0, 2.0] + y := [2.0, 3.0] + degree := 2.0 + kernel_fn := polynomial_kernel(degree) + result := kernel_fn(x, y) + expected := math.pow(dot_product(x, y) + 1.0, degree) + assert math.abs(result - expected) < 1e-6 + println('Polynomial kernel test passed.') } fn test_rbf_kernel() { - x := [1.0, 2.0] - y := [2.0, 3.0] - gamma := 0.5 - kernel_fn := ml.rbf_kernel(gamma) - result := kernel_fn(x, y) - diff := ml.vector_subtract(x, y) - expected := math.exp(-gamma * ml.dot_product(diff, diff)) - assert math.abs(result - expected) < 1e-6 - println('RBF kernel test passed.') + x := [1.0, 2.0] + y := [2.0, 3.0] + gamma := 0.5 + kernel_fn := rbf_kernel(gamma) + result := kernel_fn(x, y) + diff := vector_subtract(x, y) + expected := math.exp(-gamma * dot_product(diff, diff)) + assert math.abs(result - expected) < 1e-6 + println('RBF kernel test passed.') } fn test_quadratic_kernel() { - x := [1.0, 2.0] - y := [2.0, 3.0] - result := ml.quadratic_kernel(x, y) - expected := math.pow(ml.dot_product(x, y), 2) - assert math.abs(result - expected) < 1e-6 - println('Quadratic kernel test passed.') + x := [1.0, 2.0] + y := [2.0, 3.0] + result := quadratic_kernel(x, y) + expected := math.pow(dot_product(x, y), 2) + assert math.abs(result - expected) < 1e-6 + println('Quadratic kernel test passed.') } fn test_custom_kernel() { - x := [1.0, 2.0] - y := [2.0, 3.0] - result := ml.custom_kernel(x, y) - z_x := math.pow(x[0], 2) + math.pow(x[1], 2) - z_y := math.pow(y[0], 2) + math.pow(y[1], 2) - expected := z_x * z_y - assert math.abs(result - expected) < 1e-6 - println('Custom kernel test passed.') + x := [1.0, 2.0] + y := [2.0, 3.0] + result := custom_kernel(x, y) + z_x := math.pow(x[0], 2) + math.pow(x[1], 2) + z_y := math.pow(y[0], 2) + math.pow(y[1], 2) + expected := z_x * z_y + assert math.abs(result - expected) < 1e-6 + println('Custom kernel test passed.') } fn test_dot_product() { - a := [1.0, 2.0] - b := [3.0, 4.0] - result := ml.dot_product(a, b) - expected := 11.0 - assert math.abs(result - expected) < 1e-6 - println('Dot product test passed.') + a := [1.0, 2.0] + b := [3.0, 4.0] + result := dot_product(a, b) + expected := 11.0 + assert math.abs(result - expected) < 1e-6 + println('Dot product test passed.') } fn test_vector_subtract() { - a := [1.0, 2.0] - b := [3.0, 4.0] - result := ml.vector_subtract(a, b) - expected := [-2.0, -2.0] - assert result == expected - println('Vector subtract test passed.') + a := [1.0, 2.0] + b := [3.0, 4.0] + result := vector_subtract(a, b) + expected := [-2.0, -2.0] + assert result == expected + println('Vector subtract test passed.') } fn test_svm() { - data := [ - ml.DataPoint{x: [2.0, 3.0], y: 1}, - ml.DataPoint{x: [1.0, 2.0], y: -1}, - ml.DataPoint{x: [3.0, 3.0], y: 1}, - ml.DataPoint{x: [2.0, 1.0], y: -1}, - ] - - config := ml.SVMConfig{ - max_iterations: 100, - learning_rate: 0.01, - tolerance: 1e-5, - c: 1.0, - kernel_type: .linear, - } - - model := ml.train_svm(data, config) - - mut predictions := 0 - for point in data { - pred := ml.predict(model, point.x) - if pred == point.y { - predictions += 1 - } - } - - assert predictions == data.len - - println('SVM model training and prediction test passed.') + data := [ + DataPoint{ + x: [2.0, 3.0] + y: 1 + }, + DataPoint{ + x: [1.0, 2.0] + y: -1 + }, + DataPoint{ + x: [3.0, 3.0] + y: 1 + }, + DataPoint{ + x: [2.0, 1.0] + y: -1 + }, + ] + + config := SVMConfig{ + max_iterations: 100 + learning_rate: 0.01 + tolerance: 1e-5 + c: 1.0 + kernel_type: .linear + } + + model := train_svm(data, config) + + mut predictions := 0 + for point in data { + pred := predict(model, point.x) + if pred == point.y { + predictions += 1 + } + } + + assert predictions == data.len + + println('SVM model training and prediction test passed.') } fn test_multiclass_svm() { - data := [ - ml.DataPoint{x: [1.0, 2.0], y: 0}, - ml.DataPoint{x: [2.0, 3.0], y: 1}, - ml.DataPoint{x: [3.0, 1.0], y: 2}, - ml.DataPoint{x: [1.5, 2.5], y: 0}, - ml.DataPoint{x: [2.5, 3.5], y: 1}, - ml.DataPoint{x: [3.5, 1.5], y: 2}, - ] - - config := ml.SVMConfig{ - max_iterations: 100, - learning_rate: 0.01, - tolerance: 1e-5, - c: 1.0, - kernel_type: .linear, - } - - multiclass_model := ml.train_multiclass_svm(data, config) - - mut correct_predictions := 0 - for point in data { - predicted_class := ml.predict_multiclass(multiclass_model, point.x) - if predicted_class == point.y { - correct_predictions += 1 - } - } - - assert correct_predictions == data.len - - println('Multiclass SVM model training and prediction test passed.') + data := [ + DataPoint{ + x: [1.0, 2.0] + y: 0 + }, + DataPoint{ + x: [2.0, 3.0] + y: 1 + }, + DataPoint{ + x: [3.0, 1.0] + y: 2 + }, + DataPoint{ + x: [1.5, 2.5] + y: 0 + }, + DataPoint{ + x: [2.5, 3.5] + y: 1 + }, + DataPoint{ + x: [3.5, 1.5] + y: 2 + }, + ] + + config := SVMConfig{ + max_iterations: 100 + learning_rate: 0.01 + tolerance: 1e-5 + c: 1.0 + kernel_type: .linear + } + + multiclass_model := train_multiclass_svm(data, config) + + mut correct_predictions := 0 + for point in data { + predicted_class := predict_multiclass(multiclass_model, point.x) + if predicted_class == point.y { + correct_predictions += 1 + } + } + + assert correct_predictions == data.len + + println('Multiclass SVM model training and prediction test passed.') } fn main() { - test_linear_kernel() - test_polynomial_kernel() - test_rbf_kernel() - test_quadratic_kernel() - test_custom_kernel() - test_dot_product() - test_vector_subtract() - test_svm() - test_multiclass_svm() + test_linear_kernel() + test_polynomial_kernel() + test_rbf_kernel() + test_quadratic_kernel() + test_custom_kernel() + test_dot_product() + test_vector_subtract() + test_svm() + test_multiclass_svm() } - From b4852431e9b1662b1e71cf3dac988a454677747e Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Thu, 1 Aug 2024 00:20:17 +0300 Subject: [PATCH 11/13] Implementation of Decision Trees Algorithm --- ml/decision_tree.v | 205 ++++++++++++++++++++++++++++++++++++++++ ml/decision_tree_test.v | 94 ++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100644 ml/decision_tree.v create mode 100644 ml/decision_tree_test.v diff --git a/ml/decision_tree.v b/ml/decision_tree.v new file mode 100644 index 000000000..dcccb125c --- /dev/null +++ b/ml/decision_tree.v @@ -0,0 +1,205 @@ +module ml + +import math + +pub struct Sample { +pub mut: + features []f64 +pub: + label int +} + +pub struct Dataset { +pub mut: + samples []Sample +pub: + n_features int + n_classes int +} + +struct Node { +mut: + feature int + threshold f64 + label int + left &Node + right &Node +} + +pub struct DecisionTree { +mut: + root &Node + max_depth int + min_samples_split int +} + +pub fn DecisionTree.new(max_depth int, min_samples_split int) &DecisionTree { + return &DecisionTree{ + root: &Node(unsafe { nil }) + max_depth: max_depth + min_samples_split: min_samples_split + } +} + +pub fn index_of_max(arr []int) int { + mut max_index := 0 + for i := 1; i < arr.len; i++ { + if arr[i] > arr[max_index] { + max_index = i + } + } + return max_index +} + +pub fn create_dataset(n_features int, n_classes int) &Dataset { + return &Dataset{ + samples: []Sample{} + n_features: n_features + n_classes: n_classes + } +} + +pub fn (mut dataset Dataset) add_sample(features []f64, label int) bool { + if label < 0 || label >= dataset.n_classes { + return false + } + dataset.samples << Sample{ + features: features.clone() + label: label + } + return true +} + +pub fn (dataset &Dataset) calculate_entropy() f64 { + mut class_counts := []int{len: dataset.n_classes, init: 0} + for sample in dataset.samples { + class_counts[sample.label]++ + } + + mut entropy := 0.0 + for count in class_counts { + if count > 0 { + p := f64(count) / f64(dataset.samples.len) + entropy -= p * math.log2(p) + } + } + return entropy +} + +fn find_best_split(dataset &Dataset) (int, f64, f64) { + mut best_gain := -1.0 + mut best_feature := 0 + mut best_threshold := 0.0 + + for feature in 0 .. dataset.n_features { + for sample in dataset.samples { + threshold := sample.features[feature] + mut left := create_dataset(dataset.n_features, dataset.n_classes) + mut right := create_dataset(dataset.n_features, dataset.n_classes) + + for s in dataset.samples { + if s.features[feature] <= threshold { + left.add_sample(s.features, s.label) + } else { + right.add_sample(s.features, s.label) + } + } + + if left.samples.len > 0 && right.samples.len > 0 { + p_left := f64(left.samples.len) / f64(dataset.samples.len) + p_right := f64(right.samples.len) / f64(dataset.samples.len) + gain := dataset.calculate_entropy() - (p_left * left.calculate_entropy() + p_right * right.calculate_entropy()) + + if gain > best_gain { + best_gain = gain + best_feature = feature + best_threshold = threshold + } + } + } + } + + return best_feature, best_threshold, best_gain +} + +fn build_tree(dataset &Dataset, max_depth int, min_samples_split int) &Node { + if dataset.samples.len < min_samples_split || max_depth == 0 { + mut class_counts := []int{len: dataset.n_classes, init: 0} + for sample in dataset.samples { + class_counts[sample.label]++ + } + label := index_of_max(class_counts) + return &Node{ + feature: -1 + threshold: 0 + label: label + left: &Node(unsafe { nil }) + right: &Node(unsafe { nil }) + } + } + + best_feature, best_threshold, best_gain := find_best_split(dataset) + + if best_gain <= 0 { + mut class_counts := []int{len: dataset.n_classes, init: 0} + for sample in dataset.samples { + class_counts[sample.label]++ + } + label := index_of_max(class_counts) + return &Node{ + feature: -1 + threshold: 0 + label: label + left: &Node(unsafe { nil }) + right: &Node(unsafe { nil }) + } + } + + mut left := create_dataset(dataset.n_features, dataset.n_classes) + mut right := create_dataset(dataset.n_features, dataset.n_classes) + + for sample in dataset.samples { + if sample.features[best_feature] <= best_threshold { + left.add_sample(sample.features, sample.label) + } else { + right.add_sample(sample.features, sample.label) + } + } + + left_subtree := build_tree(left, max_depth - 1, min_samples_split) + right_subtree := build_tree(right, max_depth - 1, min_samples_split) + + return &Node{ + feature: best_feature + threshold: best_threshold + label: -1 + left: left_subtree + right: right_subtree + } +} + +pub fn (mut dt DecisionTree) train(dataset &Dataset) { + dt.root = build_tree(dataset, dt.max_depth, dt.min_samples_split) +} + +pub fn (dt &DecisionTree) predict(features []f64) int { + return predict_recursive(dt.root, features) +} + +fn predict_recursive(node &Node, features []f64) int { + if node.left == unsafe { nil } && node.right == unsafe { nil } { + return node.label + } + + if features[node.feature] <= node.threshold { + return predict_recursive(node.left, features) + } else { + return predict_recursive(node.right, features) + } +} + +pub fn calculate_information_gain(parent &Dataset, left &Dataset, right &Dataset) f64 { + p_left := f64(left.samples.len) / f64(parent.samples.len) + p_right := f64(right.samples.len) / f64(parent.samples.len) + return parent.calculate_entropy() - (p_left * left.calculate_entropy() + p_right * right.calculate_entropy()) +} diff --git a/ml/decision_tree_test.v b/ml/decision_tree_test.v new file mode 100644 index 000000000..5b37756ec --- /dev/null +++ b/ml/decision_tree_test.v @@ -0,0 +1,94 @@ +module ml + +import math + +fn test_decision_tree_creation() { + max_depth := 3 + min_samples_split := 2 + dt := DecisionTree.new(max_depth, min_samples_split) + assert dt.max_depth == max_depth + assert dt.min_samples_split == min_samples_split +} + +fn test_dataset_creation() { + n_features := 3 + n_classes := 4 + dataset := create_dataset(n_features, n_classes) + assert dataset.n_features == n_features + assert dataset.n_classes == n_classes + assert dataset.samples.len == 0 +} + +fn test_add_sample() { + mut dataset := create_dataset(3, 4) + features := [1.0, 2.0, 3.0] + label := 2 + assert dataset.add_sample(features, label) == true + assert dataset.samples.len == 1 + assert dataset.samples[0].features == features + assert dataset.samples[0].label == label + + // Test invalid label + assert dataset.add_sample(features, 5) == false + assert dataset.samples.len == 1 +} + +fn test_entropy_calculation() { + mut dataset := create_dataset(3, 4) + dataset.add_sample([1.0, 2.0, 0.5], 0) + dataset.add_sample([2.0, 3.0, 1.0], 1) + dataset.add_sample([3.0, 4.0, 1.5], 2) + dataset.add_sample([4.0, 5.0, 2.0], 3) + dataset.add_sample([2.5, 3.5, 1.2], 1) + + entropy := dataset.calculate_entropy() + expected_entropy := 1.9219280948873623 // Manually calculated + assert math.abs(entropy - expected_entropy) < 1e-6 +} + +fn test_decision_tree_training_and_prediction() { + mut dataset := create_dataset(3, 4) + dataset.add_sample([1.0, 2.0, 0.5], 0) + dataset.add_sample([2.0, 3.0, 1.0], 1) + dataset.add_sample([3.0, 4.0, 1.5], 2) + dataset.add_sample([4.0, 5.0, 2.0], 3) + dataset.add_sample([2.5, 3.5, 1.2], 1) + + mut dt := DecisionTree.new(3, 2) + dt.train(dataset) + + // Test predictions + assert dt.predict([2.5, 3.5, 1.3]) == 1 // Manually calculated +} + +fn test_information_gain() { + mut parent := create_dataset(3, 3) + parent.add_sample([2.0, 3.5, 1.1], 0) + parent.add_sample([3.0, 4.0, 1.5], 1) + parent.add_sample([1.5, 2.0, 0.5], 0) + parent.add_sample([2.5, 3.0, 1.0], 1) + parent.add_sample([4.0, 5.0, 2.0], 2) + + mut left := create_dataset(3, 3) + left.add_sample([2.0, 3.5, 1.1], 0) + left.add_sample([1.5, 2.0, 0.5], 0) + + mut right := create_dataset(3, 3) + right.add_sample([3.0, 4.0, 1.5], 1) + right.add_sample([2.5, 3.0, 1.0], 1) + right.add_sample([4.0, 5.0, 2.0], 2) + + info_gain := calculate_information_gain(parent, left, right) + expected_gain := 0.9709505944546686 // Manually calculated + assert math.abs(info_gain - expected_gain) < 1e-6 +} + +fn main() { + test_decision_tree_creation() + test_dataset_creation() + test_add_sample() + test_entropy_calculation() + test_decision_tree_training_and_prediction() + test_information_gain() +} + From 493453f688059d4cf942a23aca11d6a971fdd83f Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Thu, 1 Aug 2024 00:23:00 +0300 Subject: [PATCH 12/13] run v fmt --- ml/decision_tree.v | 28 +++++++++++++++------------- ml/decision_tree_test.v | 1 - 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ml/decision_tree.v b/ml/decision_tree.v index dcccb125c..83606216a 100644 --- a/ml/decision_tree.v +++ b/ml/decision_tree.v @@ -6,12 +6,12 @@ pub struct Sample { pub mut: features []f64 pub: - label int + label int } pub struct Dataset { pub mut: - samples []Sample + samples []Sample pub: n_features int n_classes int @@ -28,8 +28,8 @@ mut: pub struct DecisionTree { mut: - root &Node - max_depth int + root &Node + max_depth int min_samples_split int } @@ -42,13 +42,13 @@ pub fn DecisionTree.new(max_depth int, min_samples_split int) &DecisionTree { } pub fn index_of_max(arr []int) int { - mut max_index := 0 - for i := 1; i < arr.len; i++ { - if arr[i] > arr[max_index] { - max_index = i - } - } - return max_index + mut max_index := 0 + for i := 1; i < arr.len; i++ { + if arr[i] > arr[max_index] { + max_index = i + } + } + return max_index } pub fn create_dataset(n_features int, n_classes int) &Dataset { @@ -108,7 +108,8 @@ fn find_best_split(dataset &Dataset) (int, f64, f64) { if left.samples.len > 0 && right.samples.len > 0 { p_left := f64(left.samples.len) / f64(dataset.samples.len) p_right := f64(right.samples.len) / f64(dataset.samples.len) - gain := dataset.calculate_entropy() - (p_left * left.calculate_entropy() + p_right * right.calculate_entropy()) + gain := dataset.calculate_entropy() - (p_left * left.calculate_entropy() + + p_right * right.calculate_entropy()) if gain > best_gain { best_gain = gain @@ -201,5 +202,6 @@ fn predict_recursive(node &Node, features []f64) int { pub fn calculate_information_gain(parent &Dataset, left &Dataset, right &Dataset) f64 { p_left := f64(left.samples.len) / f64(parent.samples.len) p_right := f64(right.samples.len) / f64(parent.samples.len) - return parent.calculate_entropy() - (p_left * left.calculate_entropy() + p_right * right.calculate_entropy()) + return parent.calculate_entropy() - (p_left * left.calculate_entropy() + + p_right * right.calculate_entropy()) } diff --git a/ml/decision_tree_test.v b/ml/decision_tree_test.v index 5b37756ec..9be2efbd9 100644 --- a/ml/decision_tree_test.v +++ b/ml/decision_tree_test.v @@ -91,4 +91,3 @@ fn main() { test_decision_tree_training_and_prediction() test_information_gain() } - From 11121e7b36e2fe2a66f27afa3b832c40580f2972 Mon Sep 17 00:00:00 2001 From: "scriptalert(\"suleyman\");/script" Date: Thu, 1 Aug 2024 21:51:40 +0300 Subject: [PATCH 13/13] Implementation of Random Forest algorithm --- ml/random_forest.v | 165 ++++++++++++++++++++++++++++++++++++++++ ml/random_forest_test.v | 62 +++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 ml/random_forest.v create mode 100644 ml/random_forest_test.v diff --git a/ml/random_forest.v b/ml/random_forest.v new file mode 100644 index 000000000..0f21632d8 --- /dev/null +++ b/ml/random_forest.v @@ -0,0 +1,165 @@ +module ml + +import rand + +pub struct RandomForest { +pub mut: + trees []DecisionTree + n_trees int + max_depth int + min_samples_split int + feature_subset_size int +} + +pub fn RandomForest.new(n_trees int, max_depth int, min_samples_split int, feature_subset_size int) &RandomForest { + return &RandomForest{ + trees: []DecisionTree{} + n_trees: n_trees + max_depth: max_depth + min_samples_split: min_samples_split + feature_subset_size: feature_subset_size + } +} + +fn bootstrap_sample(dataset &Dataset) &Dataset { + mut bootstrap := create_dataset(dataset.n_features, dataset.n_classes) + for _ in 0 .. dataset.samples.len { + sample_index := rand.intn(dataset.samples.len) or { 0 } + sample := dataset.samples[sample_index] + bootstrap.add_sample(sample.features, sample.label) + } + return bootstrap +} + +fn select_feature_subset(n_features int, subset_size int) []int { + mut features := []int{len: n_features, init: index} + rand.shuffle(mut features) or { return features[..subset_size] } + return features[..subset_size] +} + +pub fn (mut rf RandomForest) train(dataset &Dataset) { + for _ in 0 .. rf.n_trees { + mut tree := DecisionTree.new(rf.max_depth, rf.min_samples_split) + bootstrap := bootstrap_sample(dataset) + + feature_subset := select_feature_subset(dataset.n_features, rf.feature_subset_size) + + tree.train_with_feature_subset(bootstrap, feature_subset) + rf.trees << tree + } +} + +pub fn (rf &RandomForest) predict(features []f64) int { + if rf.trees.len == 0 { + return -1 + } + + mut votes := []int{len: rf.trees[0].root.label + 1, init: 0} + for tree in rf.trees { + prediction := tree.predict(features) + if prediction >= 0 && prediction < votes.len { + votes[prediction]++ + } + } + return index_of_max(votes) +} + +pub fn (mut dt DecisionTree) train_with_feature_subset(dataset &Dataset, feature_subset []int) { + dt.root = build_tree_with_feature_subset(dataset, dt.max_depth, dt.min_samples_split, + feature_subset) +} + +fn build_tree_with_feature_subset(dataset &Dataset, max_depth int, min_samples_split int, feature_subset []int) &Node { + if dataset.samples.len < min_samples_split || max_depth == 0 { + mut class_counts := []int{len: dataset.n_classes, init: 0} + for sample in dataset.samples { + class_counts[sample.label]++ + } + label := index_of_max(class_counts) + return &Node{ + feature: -1 + threshold: 0 + label: label + left: &Node(unsafe { nil }) + right: &Node(unsafe { nil }) + } + } + + best_feature, best_threshold, best_gain := find_best_split_with_subset(dataset, feature_subset) + + if best_gain <= 0 { + mut class_counts := []int{len: dataset.n_classes, init: 0} + for sample in dataset.samples { + class_counts[sample.label]++ + } + label := index_of_max(class_counts) + return &Node{ + feature: -1 + threshold: 0 + label: label + left: &Node(unsafe { nil }) + right: &Node(unsafe { nil }) + } + } + + mut left := create_dataset(dataset.n_features, dataset.n_classes) + mut right := create_dataset(dataset.n_features, dataset.n_classes) + + for sample in dataset.samples { + if sample.features[best_feature] <= best_threshold { + left.add_sample(sample.features, sample.label) + } else { + right.add_sample(sample.features, sample.label) + } + } + + left_subtree := build_tree_with_feature_subset(left, max_depth - 1, min_samples_split, + feature_subset) + right_subtree := build_tree_with_feature_subset(right, max_depth - 1, min_samples_split, + feature_subset) + + return &Node{ + feature: best_feature + threshold: best_threshold + label: -1 + left: left_subtree + right: right_subtree + } +} + +fn find_best_split_with_subset(dataset &Dataset, feature_subset []int) (int, f64, f64) { + mut best_gain := -1.0 + mut best_feature := 0 + mut best_threshold := 0.0 + + for feature in feature_subset { + for sample in dataset.samples { + threshold := sample.features[feature] + mut left := create_dataset(dataset.n_features, dataset.n_classes) + mut right := create_dataset(dataset.n_features, dataset.n_classes) + + for s in dataset.samples { + if s.features[feature] <= threshold { + left.add_sample(s.features, s.label) + } else { + right.add_sample(s.features, s.label) + } + } + + if left.samples.len > 0 && right.samples.len > 0 { + p_left := f64(left.samples.len) / f64(dataset.samples.len) + p_right := f64(right.samples.len) / f64(dataset.samples.len) + gain := dataset.calculate_entropy() - (p_left * left.calculate_entropy() + + p_right * right.calculate_entropy()) + + if gain > best_gain { + best_gain = gain + best_feature = feature + best_threshold = threshold + } + } + } + } + + return best_feature, best_threshold, best_gain +} diff --git a/ml/random_forest_test.v b/ml/random_forest_test.v new file mode 100644 index 000000000..a3bf9d2d6 --- /dev/null +++ b/ml/random_forest_test.v @@ -0,0 +1,62 @@ +module ml + +import rand + +fn test_random_forest_creation() { + rf := RandomForest.new(10, 5, 2, 3) + assert rf.n_trees == 10 + assert rf.max_depth == 5 + assert rf.min_samples_split == 2 + assert rf.feature_subset_size == 3 + assert rf.trees.len == 0 +} + +fn test_bootstrap_sample() { + mut dataset := create_dataset(5, 2) + for i in 0 .. 100 { + dataset.add_sample([f64(i), f64(i * 2), f64(i * 3), f64(i * 4), f64(i * 5)], i % 2) + } + + bootstrap := bootstrap_sample(dataset) + assert bootstrap.samples.len == dataset.samples.len + assert bootstrap.n_features == dataset.n_features + assert bootstrap.n_classes == dataset.n_classes +} + +fn test_select_feature_subset() { + n_features := 10 + subset_size := 5 + subset := select_feature_subset(n_features, subset_size) + assert subset.len == subset_size + assert subset.all(it >= 0 && it < n_features) +} + +fn test_random_forest_train_and_predict() { + mut dataset := create_dataset(4, 2) + for i in 0 .. 1000 { + if i % 2 == 0 { + dataset.add_sample([f64(i), f64(i * 2), f64(i * 3), f64(i * 4)], 0) + } else { + dataset.add_sample([f64(i), f64(i * 2), f64(i * 3), f64(i * 4)], 1) + } + } + + mut rf := RandomForest.new(10, 5, 5, 2) + rf.train(dataset) + + assert rf.trees.len == 10 + + for i in 0 .. 100 { + features := [f64(i * 10), f64(i * 20), f64(i * 30), f64(i * 40)] + prediction := rf.predict(features) + assert prediction == 0 || prediction == 1 + } +} + +fn main() { + test_random_forest_creation() + test_bootstrap_sample() + test_select_feature_subset() + test_random_forest_train_and_predict() + println('All Random Forest tests passed successfully!') +}