Skip to content

Commit 8032032

Browse files
behricabehrica
committed
added score-fn
1 parent 3796b53 commit 8032032

4 files changed

Lines changed: 144 additions & 48 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@
66
/target/
77
.calva
88
.dir-locals.el
9+
test/scicloj/ml/tribuo_test.clj
10+
pom.properties
11+
pom.xml

‎deps.edn‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
{:paths ["src" "resources"]
22
:deps {org.clojure/clojure {:mvn/version "1.12.4"}
3-
org.scicloj/metamorph.ml {:mvn/version "1.3.2"}
3+
org.scicloj/metamorph.ml
4+
{:git/url "https://github.com/scicloj/metamorph.ml"
5+
:git/sha "bbd4d390981fbb77d0f93d489f5ed299d2c80080"}
6+
;{:mvn/version "1.3.2"}
7+
48
cheshire/cheshire {:mvn/version "6.1.0"}
59
;; tribuo core deps
610
org.tribuo/tribuo-classification-core {:mvn/version "4.3.2"}

‎src/scicloj/ml/tribuo.clj‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
(ns scicloj.ml.tribuo
22
(:require
33
[fastmath.stats :as stats]
4+
[scicloj.metamorph.ml.loss :as loss]
45
[scicloj.metamorph.ml :as ml]
56
[tech.v3.dataset :as ds]
67
[tech.v3.dataset.modelling :as ds-mod]
78
[tech.v3.datatype :as dt]
89
[tech.v3.datatype.errors :as errors]
910
[tech.v3.datatype.functional :as fun]
10-
[tech.v3.libs.tribuo :as tribuo])
11+
[tech.v3.libs.tribuo :as tribuo]
12+
[tech.v3.dataset.categorical :as dscat]
13+
[tech.v3.dataset.column-filters :as cf])
1114
(:import
1215
[org.tribuo.regression Regressor]
1316
[org.tribuo.regression.evaluation RegressionEvaluator]))
@@ -45,8 +48,13 @@
4548
casted-ds (reduce-kv
4649
(fn [m k v]
4750
(cast-target m k v target-categorical-maps))
48-
renamed-ds target-datatypes)]
49-
casted-ds))
51+
renamed-ds target-datatypes)
52+
prob-column-names
53+
(-> casted-ds
54+
(ds/drop-columns [target-column-name])
55+
(ds/column-names))]
56+
57+
(ds/assoc-metadata casted-ds prob-column-names :column-type :probability-distribution)))
5058

5159
(defn- post-process-prediction-regression [ds target-column-name]
5260
(-> ds
@@ -178,21 +186,53 @@
178186
(tribuo/predict-regression model feature-ds)
179187
(post-process-prediction-regression (first target-columns)))))
180188

189+
(defn- safety-first! [values-1 values-2]
190+
(let [different-types
191+
(into #{}
192+
(concat (map type values-1)
193+
(map type values-2)))]
194+
(assert (= 1 (count different-types))
195+
(format "All values need to be of same type, but found: %s\nvalues-1: %s\nvalues-2: %s"
196+
different-types (frequencies values-1) (frequencies values-2)))))
197+
198+
199+
(defn- score
200+
([model scoring-ds options]
201+
(let [prediction (ml/predict (cf/feature scoring-ds) model)
202+
trueth (cf/target scoring-ds)
203+
prediction-values
204+
(->
205+
(cf/prediction prediction)
206+
(dscat/reverse-map-categorical-xforms)
207+
(get (-> model :target-columns first)))
208+
trueth-values
209+
(->
210+
trueth
211+
(dscat/reverse-map-categorical-xforms)
212+
(get (-> model :target-columns first)))
213+
_ (safety-first! prediction-values trueth-values)]
214+
(loss/classification-accuracy prediction-values trueth-values)))
215+
([model scoring-ds](score model scoring-ds nil))
216+
)
181217

182218

183219
(ml/define-model! :scicloj.ml.tribuo/regression
184220
train-regression
185221
predict-regression
186222
{:glance-fn glance-fn-regression
187223
:augment-fn augment-fn-regression
188-
:thaw-fn thaw})
224+
:thaw-fn thaw
225+
226+
})
189227

190228

191229

192230
(ml/define-model! :scicloj.ml.tribuo/classification
193231
train-classification
194232
predict-classification
195-
{:thaw-fn thaw})
233+
{:thaw-fn thaw
234+
:score-fn score
235+
})
196236

197237

198238

‎test/scicloj/ml/tribuo_test.clj‎

Lines changed: 91 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@
99
[tablecloth.api :as tc]
1010
[tech.v3.dataset :as ds]
1111
[tech.v3.dataset.categorical :as dscat]
12-
[tech.v3.dataset.column-filters :as ds-cf]
13-
[tech.v3.dataset.modelling :as ds-mod]))
12+
[tech.v3.dataset.modelling :as ds-mod]
13+
[tech.v3.dataset.column-filters :as cf]))
14+
1415

1516
(def iris-target-raw
1617
(->>
@@ -46,28 +47,32 @@
4647
(ds-mod/set-inference-target :species)))
4748

4849
(defn- validate [ds expected-target-val expected-accuracy]
49-
(let [split (ds-mod/train-test-split ds {:seed 123})
50+
(let [options {:model-type :scicloj.ml.tribuo/classification
51+
:tribuo-components [{:name "trainer"
52+
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"}]
53+
:tribuo-trainer-name "trainer"}
5054

51-
model (ml/train (:train-ds split)
52-
{:model-type :scicloj.ml.tribuo/classification
53-
:tribuo-components [{:name "trainer"
54-
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"}]
55-
:tribuo-trainer-name "trainer"})
55+
split (ds-mod/train-test-split ds {:seed 123})
56+
57+
model (ml/train (:train-ds split) options)
5658
predictions (-> (ml/predict (:test-ds split) model))
5759

5860
accuracy (loss/classification-accuracy (-> split :test-ds
5961
dscat/reverse-map-categorical-xforms
6062
:species)
6163
(-> predictions
6264
dscat/reverse-map-categorical-xforms
63-
:species))]
65+
:species))
66+
score-fn (:score-fn (ml/options->model-def options))
67+
score (score-fn model (:test-ds split))]
6468

6569

70+
(t/is (< expected-accuracy score))
6671
(t/is (< expected-accuracy accuracy))
6772
(t/is (= expected-target-val
6873
(->
6974
predictions
70-
(ds-cf/prediction)
75+
(cf/prediction)
7176
ds/columns
7277
first
7378
first)))))
@@ -85,7 +90,7 @@
8590
nil)
8691
2
8792
0.94)
88-
93+
8994
(validate
9095
(make-iris-ds
9196
(make-species-column :float32
@@ -98,6 +103,20 @@
98103
2.0
99104
0.94)
100105

106+
(validate
107+
(make-iris-ds
108+
(make-species-column :float32
109+
true ;categorical?
110+
true ;inference-target?
111+
{:setosa 0.0
112+
:versicolor 1.0
113+
:virginica 2.0})
114+
nil)
115+
2.0
116+
0.94)
117+
118+
119+
101120
(validate
102121

103122
(make-iris-ds
@@ -225,25 +244,26 @@
225244
"12"
226245
0.94))))
227246

228-
(ds/new-column :x [:a :b :c])
229247

230248
(defn- verify-evaluate [ds]
231-
(let [make-pipefn (fn [opts]
232-
(mm/pipeline
233-
{:metamorph/id :model}
234-
(ml/model {:model-type :scicloj.ml.tribuo/classification
235-
:tribuo-components [{:name "trainer"
236-
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"
237-
:properties {:maxDepth "8"}}]
249+
(let [options
250+
{:model-type :scicloj.ml.tribuo/classification
251+
:tribuo-components [{:name "trainer"
252+
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"
253+
:properties {:maxDepth "8"}}]
238254

239255

240256

241-
:tribuo-trainer-name "trainer"})))
257+
:tribuo-trainer-name "trainer"}
258+
make-pipefn (fn []
259+
(mm/pipeline
260+
{:metamorph/id :model}
261+
(ml/model options)))
242262

243263
splits
244264
(tc/split->seq ds :kfold {:seed 1234})
245265

246-
pipefns [(make-pipefn {})]
266+
pipefns [(make-pipefn)]
247267

248268
evaluations
249269
(ml/evaluate-pipelines pipefns splits
@@ -254,7 +274,8 @@
254274
:return-best-pipeline-only true
255275
:evaluation-handler-fn
256276
(fn [eval-result]
257-
eval-result)})]
277+
eval-result)})
278+
]
258279

259280
(t/is (= "org.tribuo.common.tree.TreeModel"
260281
(->
@@ -286,25 +307,25 @@
286307

287308
(defn- validate-target-symetry [datatype]
288309
(t/is (= datatype
289-
(->>
290-
(ml/train
291-
(-> (ds/->dataset {:x [1 2 3 4]
292-
:y [:a :b :c :d]})
293-
(ds/categorical->number [:y] [] datatype)
294-
(ds-mod/set-inference-target [:y]))
295-
{:model-type :scicloj.ml.tribuo/classification
296-
:tribuo-components [{:name "trainer"
297-
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"
298-
:properties {:maxDepth "8"}}]
299-
:tribuo-trainer-name "trainer"})
300-
(ml/predict
301-
(-> (ds/->dataset {:x [1 2 3 4]})))
302-
:y
303-
meta
304-
:datatype))))
305-
306-
307-
(t/deftest validate-target-sym
310+
(->>
311+
(ml/train
312+
(-> (ds/->dataset {:x [1 2 3 4]
313+
:y [:a :b :c :d]})
314+
(ds/categorical->number [:y] [] datatype)
315+
(ds-mod/set-inference-target [:y]))
316+
{:model-type :scicloj.ml.tribuo/classification
317+
:tribuo-components [{:name "trainer"
318+
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"
319+
:properties {:maxDepth "8"}}]
320+
:tribuo-trainer-name "trainer"})
321+
(ml/predict
322+
(-> (ds/->dataset {:x [1 2 3 4]})))
323+
:y
324+
meta
325+
:datatype))))
326+
327+
328+
(t/deftest validate-target-sym
308329
(validate-target-symetry :int8)
309330
(validate-target-symetry :int16)
310331
(validate-target-symetry :int32)
@@ -313,7 +334,35 @@
313334
(validate-target-symetry :float64))
314335

315336

337+
(t/deftest xxx
338+
(let [iris
339+
(make-iris-ds
340+
(make-species-column :int32
341+
true ;categorical?
342+
true ;inference-target?
343+
{:setosa :setosa
344+
:versicolor :versicolor
345+
:virginica :virginica})
346+
:int)
347+
split (ds-mod/train-test-split iris {:seed 123})
348+
options {:model-type :scicloj.ml.tribuo/classification
349+
:tribuo-components [{:name "trainer"
350+
:type "org.tribuo.classification.dtree.CARTClassificationTrainer"}]
351+
:tribuo-trainer-name "trainer"}
352+
model (ml/train (:train-ds split)
353+
options)
354+
355+
p (ml/predict (:test-ds split) model)]
356+
(t/is (= (-> p cf/prediction ds/column-names) [:species]))
357+
(t/is (= (-> p cf/probability-distribution ds/column-names) ["virginica" "setosa" "versicolor"]))
358+
359+
(t/is (.equals
360+
{:name :species,
361+
:datatype :int32,
362+
:n-elems 45,
363+
:column-type :prediction,
364+
:categorical-map {:lookup-table {:versicolor 0, :setosa 1, :virginica 2}, :src-column :species, :result-datatype :int}}
365+
(-> p (cf/prediction) :species meta)))))
316366

317367

318368

319-

0 commit comments

Comments
 (0)