Code a random forest from scratch in python

Here is an implementation of a random forest from scratch in Python:

import numpy as np

class RandomForest:
    def __init__(self, n_estimators, max_depth, min_samples_split, min_impurity_decrease):
        self.n_estimators = n_estimators
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_impurity_decrease = min_impurity_decrease
        self.trees = []

    def fit(self, X, y):
        self.trees = []
        for _ in range(self.n_estimators):
            tree = DecisionTreeClassifier(max_depth=self.max_depth, min_samples_split=self.min_samples_split, 
                                           min_impurity_decrease=self.min_impurity_decrease)
            idx = np.random.choice(X.shape[0], X.shape[0], replace=True)
            X_boot, y_boot = X[idx], y[idx]
            tree.fit(X_boot, y_boot)
            self.trees.append(tree)

    def predict(self, X):
        predictions = []
        for tree in self.trees:
            predictions.append(tree.predict(X))
        return np.array(predictions).mean(axis=0)

class DecisionTreeClassifier:
    def __init__(self, max_depth, min_samples_split, min_impurity_decrease):
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_impurity_decrease = min_impurity_decrease
        self.tree = {}

    def fit(self, X, y):
        self.tree = self._build_tree(X, y, 0)

    def _build_tree(self, X, y, depth):
        if depth < self.max_depth:
            feat_idxs = np.random.choice(X.shape[1], X.shape[1], replace=False)
            feat_idxs = feat_idxs[:int(np.log2(X.shape[1]))]
            best_feat, best_val, best_gain, best_left, best_right = self._find_best_split(X, y, feat_idxs)
            if best_gain > self.min_impurity_decrease and len(y) > self.min_samples_split:
                left_X, right_X, left_y, right_y = self._split_data(X, y, best_feat, best_val)
                left_tree = self._build_tree(left_X, left_y, depth + 1)
                right_tree = self._build_tree(right_X, right_y, depth + 1)
                return { 'feature': best_feat, 'value': best_val, 'left': left_tree, 'right': right_tree }
            else:
                return y
        return y

    def _find_best_split(self, X, y, feat_idxs):
        best_feat, best_val, best_gain, best_left, best_right = None, None, -1, None, None
        for feat in feat_idxs:
            X_column = X[:, feat]
            thresholds, classes = np.unique(X_column), np.unique(y)
            for threshold in thresholds:
                left_idxs, right_idxs = X_column <= threshold, X_column > threshold
                left_y, right_y = y[left_idxs], y[right_idxs]
                gini_left, gini_right = gini(y), gini(y)
                gain = gini(y) - (np.sum(left_idxs) / len(y)) * gini_left - (np.sum(right_idxs) / len(y)) * gini_right
                if gain > best_gain:
                    best_gain = gain
                    best_feat = feat
                    best_val = threshold
                    best_left, best_right = left_y, right_y
        return best_feat, best_val, best_gain, best_left, best_right

    def _split_data(self, X, y, feat, val):
        left_idxs, right_idxs = X[:, feat] <= val, X[:, feat] > val
        return X[left_idxs], X[right_idxs], y[left_idxs], y[right_idxs]

def gini(y):
    return 1 - np.sum(np.mean(y == c) ** 2 for c in np.unique(y))

# Example usage:
rf = RandomForest(n_estimators=10, max_depth=5, min_samples_split=2, min_impurity_decrease=0.01)
X = # your feature data
y = # your target data
rf.fit(X, y)
predictions = rf.predict(X)

This implementation includes the following:

• A RandomForest class that creates multiple decision trees and combines their predictions.
• A DecisionTreeClassifier class that builds a decision tree using a recursive function _build_tree.
• The _find_best_split function finds the best feature and value to split the data.
• The _split_data function splits the data based on the best feature and value.
• The gini function calculates the Gini impurity.

Note that this is a basic implementation, and you may want to add more features, such as handling categorical variables, missing values, and hyperparameter tuning.