chore: import upstream snapshot with attribution

mla/ensemble/tree.py

@@ -0,0 +1,206 @@
+# coding:utf-8
+import random
+
+import numpy as np
+from scipy import stats
+
+from mla.ensemble.base import split, split_dataset, xgb_criterion
+
+random.seed(111)
+
+
+class Tree(object):
+    """Recursive implementation of decision tree."""
+
+    def __init__(self, regression=False, criterion=None, n_classes=None):
+        self.regression = regression
+        self.impurity = None
+        self.threshold = None
+        self.column_index = None
+        self.outcome = None
+        self.criterion = criterion
+        self.loss = None
+        self.n_classes = n_classes  # Only for classification
+
+        self.left_child = None
+        self.right_child = None
+
+    @property
+    def is_terminal(self):
+        return not bool(self.left_child and self.right_child)
+
+    def _find_splits(self, X):
+        """Find all possible split values."""
+        split_values = set()
+
+        # Get unique values in a sorted order
+        x_unique = list(np.unique(X))
+        for i in range(1, len(x_unique)):
+            # Find a point between two values
+            average = (x_unique[i - 1] + x_unique[i]) / 2.0
+            split_values.add(average)
+
+        return list(split_values)
+
+    def _find_best_split(self, X, target, n_features):
+        """Find best feature and value for a split. Greedy algorithm."""
+
+        # Sample random subset of features
+        subset = random.sample(list(range(0, X.shape[1])), n_features)
+        max_gain, max_col, max_val = None, None, None
+
+        for column in subset:
+            split_values = self._find_splits(X[:, column])
+            for value in split_values:
+                if self.loss is None:
+                    # Random forest
+                    splits = split(X[:, column], target["y"], value)
+                    gain = self.criterion(target["y"], splits)
+                else:
+                    # Gradient boosting
+                    left, right = split_dataset(
+                        X, target, column, value, return_X=False
+                    )
+                    gain = xgb_criterion(target, left, right, self.loss)
+
+                if (max_gain is None) or (gain > max_gain):
+                    max_col, max_val, max_gain = column, value, gain
+        return max_col, max_val, max_gain
+
+    def _train(
+        self,
+        X,
+        target,
+        max_features=None,
+        min_samples_split=10,
+        max_depth=None,
+        minimum_gain=0.01,
+    ):
+        try:
+            # Exit from recursion using assert syntax
+            assert X.shape[0] > min_samples_split
+            assert max_depth > 0
+
+            if max_features is None:
+                max_features = X.shape[1]
+
+            column, value, gain = self._find_best_split(X, target, max_features)
+            assert gain is not None
+            if self.regression:
+                assert gain != 0
+            else:
+                assert gain > minimum_gain
+
+            self.column_index = column
+            self.threshold = value
+            self.impurity = gain
+
+            # Split dataset
+            left_X, right_X, left_target, right_target = split_dataset(
+                X, target, column, value
+            )
+
+            # Grow left and right child
+            self.left_child = Tree(self.regression, self.criterion, self.n_classes)
+            self.left_child._train(
+                left_X,
+                left_target,
+                max_features,
+                min_samples_split,
+                max_depth - 1,
+                minimum_gain,
+            )
+
+            self.right_child = Tree(self.regression, self.criterion, self.n_classes)
+            self.right_child._train(
+                right_X,
+                right_target,
+                max_features,
+                min_samples_split,
+                max_depth - 1,
+                minimum_gain,
+            )
+        except AssertionError:
+            self._calculate_leaf_value(target)
+
+    def train(
+        self,
+        X,
+        target,
+        max_features=None,
+        min_samples_split=10,
+        max_depth=None,
+        minimum_gain=0.01,
+        loss=None,
+    ):
+        """Build a decision tree from training set.
+
+        Parameters
+        ----------
+
+        X : array-like
+            Feature dataset.
+        target : dictionary or array-like
+            Target values.
+        max_features : int or None
+            The number of features to consider when looking for the best split.
+        min_samples_split : int
+            The minimum number of samples required to split an internal node.
+        max_depth : int
+            Maximum depth of the tree.
+        minimum_gain : float, default 0.01
+            Minimum gain required for splitting.
+        loss : function, default None
+            Loss function for gradient boosting.
+        """
+
+        if not isinstance(target, dict):
+            target = {"y": target}
+
+        # Loss for gradient boosting
+        if loss is not None:
+            self.loss = loss
+
+        if not self.regression:
+            self.n_classes = len(np.unique(target["y"]))
+
+        self._train(
+            X,
+            target,
+            max_features=max_features,
+            min_samples_split=min_samples_split,
+            max_depth=max_depth,
+            minimum_gain=minimum_gain,
+        )
+
+    def _calculate_leaf_value(self, targets):
+        """Find optimal value for leaf."""
+        if self.loss is not None:
+            # Gradient boosting
+            self.outcome = self.loss.approximate(targets["actual"], targets["y_pred"])
+        else:
+            # Random Forest
+            if self.regression:
+                # Mean value for regression task
+                self.outcome = np.mean(targets["y"])
+            else:
+                # Probability for classification task
+                self.outcome = (
+                    np.bincount(targets["y"], minlength=self.n_classes)
+                    / targets["y"].shape[0]
+                )
+
+    def predict_row(self, row):
+        """Predict single row."""
+        if not self.is_terminal:
+            if row[self.column_index] < self.threshold:
+                return self.left_child.predict_row(row)
+            else:
+                return self.right_child.predict_row(row)
+        return self.outcome
+
+    def predict(self, X):
+        result = np.zeros(X.shape[0])
+        for i in range(X.shape[0]):
+            result[i] = self.predict_row(X[i, :])
+        return result