{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# RF (Random Forest; ランダムフォレスト)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "::::{admonition} matplotlibで日本語を使うための準備 \n", ":class: note, dropdown\n", "\n", "事前に`palmerpenguins`データセットをダウンロードしておこう．\n", "\n", "```py\n", "try:\n", " from palmerpenguins import load_penguins\n", "except:\n", " !pip install palmerpenguins\n", " from palmerpenguins import load_penguins\n", "```\n", "\n", "::::" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "LHARO55yibSN", "outputId": "329cd263-95dc-4424-92af-1e964f05efbc" }, "outputs": [], "source": [ "import numpy as np \n", "import pandas as pd\n", "import scipy\n", "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n", "from sklearn.datasets import load_wine\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import LabelEncoder\n", "from copy import copy\n", "from sklearn.ensemble import RandomForestClassifier\n", "from palmerpenguins import load_penguins\n", "\n", "SEED = 2023_02_15" ] }, { "cell_type": "markdown", "metadata": { "id": "fR-UFq-9WXx0" }, "source": [ "## データの準備" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 467 }, "id": "wyHOpiLQQIMK", "outputId": "07987aba-b56d-444d-d641-37cf8b2f1c21" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "-------Original DataFrame-------------------\n", "(344, 8)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
0AdelieTorgersen39.118.7181.03750.0male2007
1AdelieTorgersen39.517.4186.03800.0female2007
2AdelieTorgersen40.318.0195.03250.0female2007
3AdelieTorgersenNaNNaNNaNNaNNaN2007
4AdelieTorgersen36.719.3193.03450.0female2007
\n", "
" ], "text/plain": [ " species island bill_length_mm bill_depth_mm flipper_length_mm \\\n", "0 Adelie Torgersen 39.1 18.7 181.0 \n", "1 Adelie Torgersen 39.5 17.4 186.0 \n", "2 Adelie Torgersen 40.3 18.0 195.0 \n", "3 Adelie Torgersen NaN NaN NaN \n", "4 Adelie Torgersen 36.7 19.3 193.0 \n", "\n", " body_mass_g sex year \n", "0 3750.0 male 2007 \n", "1 3800.0 female 2007 \n", "2 3250.0 female 2007 \n", "3 NaN NaN 2007 \n", "4 3450.0 female 2007 " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "-------Preprocessed DataFrame-------------------\n", "(333, 8)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
00239.118.7181.03750.012007
10239.517.4186.03800.002007
20240.318.0195.03250.002007
40236.719.3193.03450.002007
50239.320.6190.03650.012007
\n", "
" ], "text/plain": [ " species island bill_length_mm bill_depth_mm flipper_length_mm \\\n", "0 0 2 39.1 18.7 181.0 \n", "1 0 2 39.5 17.4 186.0 \n", "2 0 2 40.3 18.0 195.0 \n", "4 0 2 36.7 19.3 193.0 \n", "5 0 2 39.3 20.6 190.0 \n", "\n", " body_mass_g sex year \n", "0 3750.0 1 2007 \n", "1 3800.0 0 2007 \n", "2 3250.0 0 2007 \n", "4 3450.0 0 2007 \n", "5 3650.0 1 2007 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "print(\"-------Original DataFrame-------------------\")\n", "df = load_penguins()\n", "print(df.shape)\n", "display(df.head())\n", "#display(df.info())\n", "\n", "print(\"-------Preprocessed DataFrame-------------------\")\n", "df = df.dropna()\n", "labelencoder = LabelEncoder()\n", "df.island = labelencoder.fit_transform(df.island)\n", "df.sex = labelencoder.fit_transform(df.sex)\n", "df.species = labelencoder.fit_transform(df.species)\n", "print(df.shape)\n", "display(df.head())\n", "\n", "X = df.drop(\"species\", axis=1).to_numpy()\n", "y = df[\"species\"].to_numpy()\n", "X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.5,)#stratify=y)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "hIZbBTRvWbsr" }, "source": [ "## scikit-learnを使った決定木の実験" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "IjKRdUtoi3t5", "outputId": "39d5cf43-530b-452a-8859-38dc12aeed21" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train_acc=1.0\n", "test_acc=0.9820359281437125\n" ] } ], "source": [ "dtc = DecisionTreeClassifier(max_depth=None,random_state=SEED)\n", "dtc.fit(X_train,y_train)\n", "train_acc=dtc.score(X_train,y_train)\n", "test_acc=dtc.score(X_test,y_test)\n", "print(f\"{train_acc=}\\n{test_acc=}\")" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## scikit-learnを使ったRandom Forestの実験" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(n_jobs=-1, random_state=20230215)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(n_jobs=-1, random_state=20230215)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rfc = RandomForestClassifier(\n", " n_estimators=100, # 弱学習器を何個作るか\n", " criterion=\"gini\", # 損失関数をginiやentropyから指定\n", " max_depth=None, # 弱学習器として使った決定木の深さ上限\n", " max_features=\"sqrt\", # ブートストラップサンプルの特徴数。sqrtはsqrt(n_features)\n", " n_jobs=-1, # 何個並列で計算するか。-1は使えるコアを全て使う。\n", " random_state=SEED, # 擬似乱数のSEED\n", ")\n", "\n", "rfc.fit(X_train,y_train)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([1, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 2, 1, 2, 0, 2, 2, 2, 2, 0, 0, 0,\n", " 2, 2, 1, 1, 1, 0, 1, 0, 2, 1, 2, 2, 2, 0, 2, 2, 0, 0, 0, 0, 2, 2,\n", " 0, 0, 0, 2, 2, 2, 0, 0, 2, 2, 0, 0, 2, 2, 0, 2, 1, 1, 2, 0, 1, 2,\n", " 0, 1, 2, 0, 0, 0, 2, 0, 1, 2, 1, 1, 2, 0, 2, 2, 2, 0, 0, 0, 0, 0,\n", " 0, 2, 2, 0, 1, 0, 0, 0, 0, 2, 1, 2, 0, 0, 2, 2, 1, 1, 0, 0, 0, 1,\n", " 1, 2, 2, 2, 2, 0, 0, 1, 2, 1, 2, 2, 2, 0, 2, 1, 2, 0, 1, 0, 0, 2,\n", " 0, 2, 2, 1, 2, 2, 0, 0, 1, 2, 1, 2, 0, 1, 2, 2, 0, 1, 0, 2, 2, 2,\n", " 0, 0, 0, 0, 1, 0, 2, 0, 2, 0, 2, 1, 2])" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rfc.predict(X_test)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.9820359281437125" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rfc.score(X_test,y_test)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": { "id": "jrQwSn4Tw2Sn" }, "source": [ "## RandomForestClassifierのシンプルな実装例\n", "NumPyを使ってRandom Forest Classifierを実装しましょう。ただし、sklearnのDecisionTreeClassifierを使います。" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "id": "epA-c6l0i3rQ" }, "outputs": [], "source": [ "import numpy as np \n", "import scipy\n", "from sklearn.tree import DecisionTreeClassifier\n", "from copy import copy\n", "\n", "# 最頻値を求める\n", "def mode(Mat: np.ndarray, axis: int = None) -> np.ndarray:\n", " return scipy.stats.mode(Mat, axis=axis, keepdims=False).mode\n", "\n", "def get_bootstrap_sample_indices(rng:np.random._generator.Generator, X:np.ndarray, bootstrap_sample_size:int)->np.ndarray:\n", " \"\"\"ブートストラップサンプルを一つ作る\n", "\n", " Args:\n", " rng (np.random._generator.Generator): 擬似乱数生成器\n", " X (np.ndarray): 二次元配列\n", " bootstrap_sample_size (int): サンプルサイズ\n", "\n", " Returns:\n", " np.ndarray: サンプルのindexを持った一次元配列\n", " \"\"\"\n", " return rng.integers(low=0, high=X.shape[0],size=bootstrap_sample_size,)\n", "\n", "\n", "class MyRandomForestClassifier:\n", " def __init__(self,\n", " bootstrap_sample_size: int,\n", " max_features: int = None,\n", " n_estimators: int = 100,\n", " rng: np.random._generator.Generator = np.random.default_rng(\n", " np.random.randint(2**20)),\n", " **estimator_params,\n", " ):\n", " self.n_estimators = n_estimators\n", " self.bootstrap_sample_size = bootstrap_sample_size\n", " self.max_features = max_features\n", " self.rng = rng\n", " self.estimator_params = estimator_params\n", " self.estimators_ = []\n", " self.selected_features_ = []\n", " self.is_fitted = False\n", "\n", " def fit(self, X, y):\n", " # ブートストラップサンプルを作成\n", " for _x, _y in self.get_bootstrap_sample(X, y):\n", " # 弱識別器の訓練を行う\n", " _estimator = DecisionTreeClassifier(**self.estimator_params,\n", " random_state=self.rng.integers(0, 2**20),)\n", " _estimator.fit(_x, _y)\n", " # 学習済み弱識別器をリストに保存\n", " self.estimators_.append(_estimator)\n", " self.is_fitted = True\n", " return self\n", "\n", " def get_bootstrap_sample(self, X: np.ndarray, y: np.ndarray):\n", " \"\"\"ブートストラップサンプルを作成し、データとラベルのペアを一つ一つ返すメソッド\n", " \"\"\"\n", " if self.is_fitted:\n", " print(\"warning! 2回目以降のfitです。bootstrap sampleの作り方が初期化されます。\")\n", "\n", " for _ in range(self.n_estimators):\n", " _sample_data_indices = get_bootstrap_sample_indices(self.rng,X,self.bootstrap_sample_size)\n", " \n", " # ランダムに特徴を選択する\n", " _feature_indices = np.arange(X.shape[1])\n", " if self.max_features is not None:\n", " self.rng.shuffle(_feature_indices)\n", " _feature_indices = _feature_indices[:self.max_features]\n", " self.selected_features_.append(_feature_indices)\n", "\n", " # ブートストラップサンプルを切り出す\n", " X_sample = X[_sample_data_indices][:, _feature_indices]\n", " y_sample = y[_sample_data_indices]\n", " yield X_sample, y_sample\n", "\n", " def predict(self, X):\n", " assert self.is_fitted, \"このメソッドは訓練後に利用してください。\"\n", " _pred_labels = []\n", "\n", " for _index in range(len(self.estimators_)):\n", " # _index番目の弱識別器を使ってXのラベルを推論する\n", " _estimator = self.estimators_[_index]\n", " _feature_indices = self.selected_features_[_index]\n", " _pred_labels.append(_estimator.predict(X[:, _feature_indices]))\n", " _pred_labels = np.vstack(_pred_labels)\n", "\n", " # 多数決で予測値を決定する（_pred_labelsの各列の最頻値を返す)\n", " pred_labels = mode(_pred_labels, axis=0)\n", " return pred_labels\n", "\n", " def score(self, X, y):\n", " \"正答率を計算する\"\n", " assert self.is_fitted, \"このメソッドは訓練後に利用してください。\"\n", " _pred_labels = self.predict(X,)\n", " return (_pred_labels == y).sum()/y.size" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "3gABw94OMq-I", "outputId": "f5b832a3-f427-4418-8c93-6064c5408243" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train_acc=1.0\n", "test_acc=0.9820359281437125\n" ] } ], "source": [ "rf = MyRandomForestClassifier(\n", " bootstrap_sample_size=int(X_train.shape[0]*0.9),\n", " max_features = int(X_train.shape[1]*0.8),\n", " n_estimators = 100,\n", " rng = np.random.default_rng(SEED),\n", " max_depth=None,\n", " )\n", "rf.fit(X_train,y_train)\n", "\n", "train_acc=rf.score(X_train,y_train)\n", "test_acc=rf.score(X_test,y_test)\n", "print(f\"{train_acc=}\\n{test_acc=}\")" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## RandomForestRegressorのシンプルな実装例\n", "NumPyを使ってRandom Forest Regressorを実装しましょう。ただし、sklearnのDecisionTreeRegressorを使います。" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "id": "lgG4iQoru1nr" }, "outputs": [], "source": [ "class MyRandomForestRegressor:\n", " def __init__(self,\n", " bootstrap_sample_size: int,\n", " max_features: int = None,\n", " n_estimators: int = 100,\n", " rng: np.random._generator.Generator = np.random.default_rng(\n", " np.random.randint(2**20)),\n", " **estimator_params,\n", " ):\n", " self.n_estimators = n_estimators\n", " self.bootstrap_sample_size = bootstrap_sample_size\n", " self.max_features = max_features\n", " self.rng = rng\n", " self.estimator_params = estimator_params\n", " self.estimators_ = []\n", " self.selected_features_ = []\n", " self.is_fitted = False\n", "\n", " def fit(self, X, y):\n", " # ブートストラップサンプルを作成\n", " for _x, _y in self.get_bootstrap_sample(X, y):\n", " # 弱識別器の訓練を行う\n", " _estimator = DecisionTreeRegressor(**self.estimator_params,\n", " random_state=self.rng.integers(0, 2**20),)\n", " _estimator.fit(_x, _y)\n", " # 学習済み弱識別器をリストに保存\n", " self.estimators_.append(_estimator)\n", " self.is_fitted = True\n", " return self\n", "\n", " def get_bootstrap_sample(self, X: np.ndarray, y: np.ndarray):\n", " \"\"\"ブートストラップサンプルを作成し、データとラベルのペアを一つ一つ返すメソッド\n", " \"\"\"\n", " if self.is_fitted:\n", " print(\"warning! 2回目以降のfitです。bootstrap sampleの作り方が初期化されます。\")\n", "\n", " for _ in range(self.n_estimators):\n", " _sample_data_indices = get_bootstrap_sample_indices(self.rng,X,self.bootstrap_sample_size)\n", " \n", " # ランダムに特徴を選択する\n", " _feature_indices = np.arange(X.shape[1])\n", " if self.max_features is not None:\n", " self.rng.shuffle(_feature_indices)\n", " _feature_indices = _feature_indices[:self.max_features]\n", " self.selected_features_.append(_feature_indices)\n", "\n", " # ブートストラップサンプルを切り出す\n", " X_sample = X[_sample_data_indices][:, _feature_indices]\n", " y_sample = y[_sample_data_indices]\n", " yield X_sample, y_sample\n", "\n", " def predict(self, X):\n", " assert self.is_fitted, \"このメソッドは訓練後に利用してください。\"\n", " _pred_labels = []\n", "\n", " for _index in range(len(self.estimators_)):\n", " # _index番目の弱識別器を使ってXのラベルを推論する\n", " _estimator = self.estimators_[_index]\n", " _feature_indices = self.selected_features_[_index]\n", " _pred_labels.append(_estimator.predict(X[:, _feature_indices]))\n", " _pred_labels = np.vstack(_pred_labels)\n", "\n", " # 平均で予測値を決定する（_pred_labelsの各列の平均を返す)\n", " pred_labels = np.mean(_pred_labels, axis=0)\n", " return pred_labels\n", "\n", " def score(self, X, y):\n", " \"正答率を計算する\"\n", " assert self.is_fitted, \"このメソッドは訓練後に利用してください。\"\n", " _pred_labels = self.predict(X,)\n", " return (_pred_labels == y).sum()/y.size" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.8" } }, "nbformat": 4, "nbformat_minor": 0 }