درخت های مدل (Model Trees) — راهنمای کاربردی

1"""
2 model_tree.py  (author: Anson Wong / git: ankonzoid)
3 Given a classification/regression model, this code builds its model tree.
4"""
5import os, pickle, csv
6from src.ModelTree import ModelTree
7from src.utils import load_csv_data, cross_validate
8
9def main():
10    # ====================
11    # Settings
12    # ====================
13    mode = "regr"  # "clf" / "regr"
14    save_model_tree = True  # save model tree?
15    save_model_tree_predictions = True  # save model tree predictions/explanations?
16    cross_validation = True  # cross-validate model tree?
17
18    # ====================
19    # Load data
20    # ====================
21    data_csv_data_filename = os.path.join("data", "data_clf.csv")
22    X, y, header = load_csv_data(data_csv_data_filename, mode=mode, verbose=True)
23
24    # *********************************************
25    #
26    # Insert your models here!
27    #
28    # All models must have the following class instantiations:
29    #
30    #   fit(X, y)
31    #   predict(X)
32    #   loss(X, y, y_pred)
33    #
34    # Below are some ready-for-use regression models:
35    #
36    #   mean regressor  (models/mean_regr.py)
37    #   linear regressor  (models/linear_regr.py)
38    #   logistic regressor  (lmodels/ogistic_regr.py)
39    #   support vector machine regressor  (models/svm_regr.py)
40    #   decision tree regressor (models/DT_sklearn_regr.py)
41    #   neural network regressor (models/DT_sklearn_regr.py)
42    #
43    # as well as some classification models:
44    #
45    #   modal classifier (models/modal_clf.py)
46    #   decision tree classifier (models/DT_sklearn_clf.py)
47    #
48    # *********************************************
49    from models.mean_regr import mean_regr
50    from models.linear_regr import linear_regr
51    from models.logistic_regr import logistic_regr
52    from models.svm_regr import svm_regr
53    from models.DT_sklearn_regr import DT_sklearn_regr
54
55    from models.modal_clf import modal_clf
56    from models.DT_sklearn_clf import DT_sklearn_clf
57
58    # Choose model
59    model = linear_regr()
60
61    # Build model tree
62    model_tree = ModelTree(model, max_depth=4, min_samples_leaf=10,
63                           search_type="greedy", n_search_grid=100)
64
65    # ====================
66    # Train model tree
67    # ====================
68    print("Training model tree with '{}'...".format(model.__class__.__name__))
69    model_tree.fit(X, y, verbose=True)
70    y_pred = model_tree.predict(X)
71    explanations = model_tree.explain(X, header)
72    loss = model_tree.loss(X, y, y_pred)
73    print(" -> loss_train={:.6f}\n".format(loss))
74    model_tree.export_graphviz(os.path.join("output", "model_tree"), header,
75                               export_png=True, export_pdf=False)
76
77    # ====================
78    # Save model tree results
79    # ====================
80    if save_model_tree:
81        model_tree_filename = os.path.join("output", "model_tree.p")
82        print("Saving model tree to '{}'...".format(model_tree_filename))
83        pickle.dump(model, open(model_tree_filename, 'wb'))
84
85    if save_model_tree_predictions:
86        predictions_csv_filename = os.path.join("output", "model_tree_pred.csv")
87        print("Saving mode tree predictions to '{}'".format(predictions_csv_filename))
88        with open(predictions_csv_filename, "w") as f:
89            writer = csv.writer(f)
90            field_names = ["x", "y", "y_pred", "explanation"]
91            writer.writerow(field_names)
92            for (x_i, y_i, y_pred_i, exp_i) in zip(X, y, y_pred, explanations):
93                field_values = [x_i, y_i, y_pred_i, exp_i]
94                writer.writerow(field_values)
95
96    # ====================
97    # Cross-validate model tree
98    # ====================
99    if cross_validation:
100        cross_validate(model_tree, X, y, kfold=5, seed=1)
101
102# Driver
103if __name__ == "__main__":
104    main()

1python3 run_model_tree.py

1Loading data from 'input/data_clf.csv' (mode=regr)...
2 header=['x1', 'x2', 'x3', 'x4', 'y']
3 X.shape=(1372, 4)
4 y.shape=(1372,)
5 len(y_classes)=2
6
7Training model tree with 'linear_regr'...
8 max_depth=4, min_samples_leaf=10, search_type=greedy...
9 node 0 @ depth 0: loss=0.033372, j_feature=1, threshold=-1.862400, N=(339,1033)
10  node 1 @ depth 1: loss=0.016889, j_feature=0, threshold=0.234600, N=(209,130)
11    node 3 @ depth 2: loss=0.006617, j_feature=0, threshold=-0.657670, N=(195,14)
12      *leaf 5 @ depth 3: loss=0.000000, N=195
13      *leaf 6 @ depth 3: loss=0.004635, N=14
14    node 4 @ depth 2: loss=0.010361, j_feature=0, threshold=0.744280, N=(13,117)
15      *leaf 7 @ depth 3: loss=0.000000, N=13
16      *leaf 8 @ depth 3: loss=0.000000, N=117
17  node 2 @ depth 1: loss=0.023927, j_feature=2, threshold=-1.544300, N=(346,687)
18    node 9 @ depth 2: loss=0.014254, j_feature=1, threshold=5.202200, N=(149,197)
19      node 11 @ depth 3: loss=0.007080, j_feature=0, threshold=2.017700, N=(139,10)
20        *leaf 13 @ depth 4: loss=0.000000, N=139
21        *leaf 14 @ depth 4: loss=0.003821, N=10
22      *leaf 12 @ depth 3: loss=0.000000, N=197
23    node 10 @ depth 2: loss=0.018931, j_feature=0, threshold=0.559390, N=(377,310)
24      node 15 @ depth 3: loss=0.020929, j_feature=3, threshold=-1.566800, N=(154,223)
25        *leaf 17 @ depth 4: loss=0.010759, N=154
26        *leaf 18 @ depth 4: loss=0.020452, N=223
27      node 16 @ depth 3: loss=0.002916, j_feature=1, threshold=-0.045533, N=(23,287)
28        *leaf 19 @ depth 4: loss=0.016037, N=23
29        *leaf 20 @ depth 4: loss=0.000000, N=287
30 -> loss_train=0.004876
31
32Saving model tree diagram to 'output/model_tree.png'...
33Saving model tree to 'output/model_tree.p'...
34Saving mode tree predictions to 'output/model_tree_pred.csv'
35Cross-validating (kfold=5, seed=1)...
36 [fold 1/5] loss_train=0.00424305, loss_validation=0.011334
37 [fold 2/5] loss_train=0.00373604, loss_validation=0.0138225
38 [fold 3/5] loss_train=0.00249428, loss_validation=0.00959152
39 [fold 4/5] loss_train=0.00207239, loss_validation=0.0103934
40 [fold 5/5] loss_train=0.00469358, loss_validation=0.010235
41  -> loss_train_avg=0.003448, loss_validation_avg=0.011075

1x,y,y_pred,explanation
2[ 3.6216   8.6661  -2.8073  -0.44699],0,0.0,"['x2 = 8.666100 > -1.862400', 'x3 = -2.807300 <= -1.544300', 'x2 = 8.666100 > 5.202200']"
3[ 4.5459  8.1674 -2.4586 -1.4621],0,0.0,"['x2 = 8.167400 > -1.862400', 'x3 = -2.458600 <= -1.544300', 'x2 = 8.167400 > 5.202200']"
4[ 3.866   -2.6383   1.9242   0.10645],0,0.0,"['x2 = -2.638300 <= -1.862400', 'x1 = 3.866000 > 0.234600', 'x1 = 3.866000 > 0.744280']"
5...

1"""
2 run_tests.py  (author: Anson Wong / git: ankonzoid)
3 Runs 3 tests to make sure our model tree works as expected.
4"""
5import os, csv
6import numpy as np
7import matplotlib.pyplot as plt
8from src.utils import load_csv_data
9from src.ModelTree import ModelTree
10
11def main():
12    # ====================
13    # Run sanity checks on model tree before training (using our own data)
14    #  1) Reproduce model result on depth-0 model tree
15    #  2) Reproduce sklearn DecisionTreeRegressor result using mean regression + mse
16    #  3) Reproduce sklearn DecisionTreeClassifier result using modal class + gini loss
17    # ====================
18    run_tests(ModelTree, os.path.join("data", "data_clf.csv"))
19
20    # ====================
21    # For 1D polynomial data using a model tree with linear regression model
22    # ====================
23
24    # Generate 1D polynomial data and save as a csv
25    func = lambda x: (x-1)*(x-4)*(x-8)*(x-8)
26    data_csv_data_filename = os.path.join("data", "data_poly4_regr.csv")
27    generate_csv_data(func, data_csv_data_filename,  x_range=(0, 10), N=500)
28
29    # Read generated data
30    X, y, header = load_csv_data(data_csv_data_filename, mode="regr", verbose=True)
31    assert X.shape[1] == 1
32
33    # Train different depth model tree fits and plot results
34    from models.mean_regr import mean_regr
35    plot_model_tree_fit(mean_regr(), X, y)
36    from models.linear_regr import linear_regr
37    plot_model_tree_fit(linear_regr(), X, y)
38
39
40# ********************************
41#
42# Side functions
43#
44# ********************************
45
46
47def plot_model_tree_fit(model, X, y):
48        output_filename = os.path.join("output", "test_{}_fit.png".format(model.__class__.__name__))
49        print("Saving model tree predictions plot y vs x to '{}'...".format(output_filename))
50
51        plt.figure(figsize=(20, 10))
52        figure_str = "23"
53        for depth in range(6):
54            # Form model tree
55            print(" -> training model tree depth={}...".format(depth))
56            model_tree = ModelTree(model, max_depth=depth, min_samples_leaf=10,
57                                   search_type="greedy", n_search_grid=100)
58
59            # Train model tree
60            model_tree.fit(X, y, verbose=False)
61            y_pred = model_tree.predict(X)
62
63            # Plot predictions
64            plt.subplot(int(figure_str + str(depth + 1)))
65            plt.plot(X[:, 0], y, '.', markersize=5, color='k')
66            plt.plot(X[:, 0], y_pred, '.', markersize=5, color='r')
67            plt.legend(['data', 'fit'])
68            plt.title("depth = {}".format(depth))
69            plt.xlabel("x", fontsize=15)
70            plt.ylabel("y", fontsize=15)
71            plt.grid()
72
73        plt.suptitle('Model tree (model = {}) fits for different depths'.format(model.__class__.__name__), fontsize=25)
74        plt.savefig(output_filename, bbox_inches='tight')
75        plt.close()
76
77def generate_csv_data(func, output_csv_filename, x_range=(0, 1), N=500):
78    x_vec = np.linspace(x_range[0], x_range[1], N)
79    y_vec = np.vectorize(func)(x_vec)
80    with open(output_csv_filename, "w") as f:
81        writer = csv.writer(f)
82        field_names = ["x1", "y"]
83        writer.writerow(field_names)
84        for (x, y) in zip(x_vec, y_vec):
85            field_values = [x, y]
86            writer.writerow(field_values)
87
88def run_tests(ModelTree, data_csv_filename):
89
90    print("Running model tree tests...")
91    eps = 1E-6 # tolerance for test acceptance
92    X, y, header = load_csv_data(data_csv_filename, mode="regr")
93
94    # Test 1
95    print(" [1/3] Checking depth-0 model tree...")
96    from models.linear_regr import linear_regr
97    model = linear_regr()
98    MTR_0 = ModelTree(model, max_depth=0, min_samples_leaf=20,
99                      search_type="greedy", n_search_grid=100)
100    loss_model = experiment(model, X, y)
101    loss_MTR_0 = experiment(MTR_0, X, y)
102    print("  -> loss(linregr)={:.6f}, loss(MTR_0_linregr)={:.6f}...".format(loss_model, loss_MTR_0))
103    if np.abs(loss_model - loss_MTR_0) > eps:
104        exit("err: passed test 1!")
105    else:
106        print("  -> passed test 1!")
107
108    # Test 2
109    print(" [2/3] Reproducing DecisionTreeRegressor sklearn (depth=20) result...")
110    from models.mean_regr import mean_regr
111    MTR = ModelTree(mean_regr(), max_depth=20, min_samples_leaf=10,
112                    search_type="greedy", n_search_grid=100)
113    from models.DT_sklearn_regr import DT_sklearn_regr
114    DTR_sklearn = DT_sklearn_regr(max_depth=20, min_samples_leaf=10)
115    loss_MTR = experiment(MTR, X, y)
116    loss_DTR_sklearn = experiment(DTR_sklearn, X, y)
117    print("  -> loss(MTR)={:.6f}, loss(DTR_sklearn)={:.6f}...".format(loss_MTR, loss_DTR_sklearn))
118    if np.abs(loss_MTR - loss_DTR_sklearn) > eps:
119        exit("err: passed test 2!")
120    else:
121        print("  -> passed test 2!")
122
123    # Test 3
124    print(" [3/3] Reproducing DecisionTreeClassifier sklearn (depth=20) result...")
125    from models.modal_clf import modal_clf
126    MTC = ModelTree(modal_clf(), max_depth=20, min_samples_leaf=10,
127                    search_type="greedy", n_search_grid=100)
128    from models.DT_sklearn_clf import DT_sklearn_clf
129    DTC_sklearn = DT_sklearn_clf(max_depth=20, min_samples_leaf=10)
130    loss_MTC = experiment(MTC, X, y)
131    loss_DTC_sklearn = experiment(DTC_sklearn, X, y)
132    print("  -> loss(MTC)={:.6f}, loss(DTC_sklearn)={:.6f}...".format(loss_MTC, loss_DTC_sklearn))
133    if np.abs(loss_MTC - loss_DTC_sklearn) > eps:
134        exit("err: passed test 3!")
135    else:
136        print("  -> passed test 3!")
137    print()
138
139def experiment(model, X, y):
140    model.fit(X, y)  # train model
141    y_pred = model.predict(X)
142    loss = model.loss(X, y, y_pred)  # compute loss
143    return loss
144
145# Driver
146if __name__ == "__main__":
147    main()