boost原理与sklearn源码_从sklearn源码简析GBDT
def __init__(self, loss, learning_rate, n_estimators, criterion, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_depth, min_impurity_decrease, min_impurity_split, init, subsample, max_features, ccp_alpha, random_state, alpha=0.9, verbose=0, max_leaf_nodes=None, warm_start=False, presort='deprecated', validation_fraction=0.1, n_iter_no_change=None, tol=1e-4): self.n_estimators = n_estimators self.learning_rate = learning_rate self.loss = loss self.criterion = criterion self.min_samples_split = min_samples_split self.min_samples_leaf = min_samples_leaf self.min_weight_fraction_leaf = min_weight_fraction_leaf self.subsample = subsample self.max_features = max_features self.max_depth = max_depth self.min_impurity_decrease = min_impurity_decrease self.min_impurity_split = min_impurity_split self.ccp_alpha = ccp_alpha self.init = init self.random_state = random_state self.alpha = alpha self.verbose = verbose self.max_leaf_nodes = max_leaf_nodes self.warm_start = warm_start self.presort = presort self.validation_fraction = validation_fraction self.n_iter_no_change = n_iter_no_change self.tol = toldef fit(self, X, y, sample_weight=None, monitor=None): if not self.warm_start: self._clear_state() n_samples, self.n_features_ = X.shape sample_weight_is_none = sample_weight is None if sample_weight_is_none: sample_weight = np.ones(n_samples, dtype=np.float32) else: sample_weight_is_none = False X, X_val, y, y_val, sample_weight, sample_weight_val =train_test_split(X, y, sample_weight, random_state=self.random_state, test_size=self.validation_fraction) if self.init_ == 'zero': raw_predictions = np.zeros(shape=(X.shape[0], self.loss_.K), dtype=np.float64) else: if sample_weight_is_none: self.init_.fit(X, y) else: msg = ("The initial estimator {} does not support sample " "weights.".format(self.init_.__class__.__name__)) try: self.init_.fit(X, y, sample_weight=sample_weight) except TypeError: raise ValueError(msg) except ValueError as e: if "pass parameters to specific steps of " \ "your pipeline using the " \ "stepname__parameter" in str(e): raise ValueError(msg) from e else: raise raw_predictions = \ self.loss_.get_init_raw_predictions(X, self.init_) begin_at_stage = 0 self._rng = self.random_state X_idx_sorted = None n_stages = self._fit_stages( X, y, raw_predictions, sample_weight, self._rng, X_val, y_val, sample_weight_val, begin_at_stage, monitor, X_idx_sorted) self.n_estimators_ = n_stages return selfdef _fit_stage(self, i, X, y, raw_predictions, sample_weight, random_state, X_idx_sorted): loss = self.loss_ original_y = y raw_predictions_copy = raw_predictions.copy() for k in range(loss.K): if loss.is_multi_class: y = np.array(original_y == k, dtype=np.float64) residual = loss.negative_gradient(y, raw_predictions_copy, k=k, sample_weight=sample_weight) tree = DecisionTreeRegressor( criterion=self.criterion, splitter='best', max_depth=self.max_depth, min_samples_split=self.min_samples_split, min_samples_leaf=self.min_samples_leaf, min_weight_fraction_leaf=self.min_weight_fraction_leaf, min_impurity_decrease=self.min_impurity_decrease, min_impurity_split=self.min_impurity_split, max_features=self.max_features, max_leaf_nodes=self.max_leaf_nodes, random_state=random_state, ccp_alpha=self.ccp_alpha) tree.fit(X, residual, sample_weight=sample_weight, check_input=False, X_idx_sorted=X_idx_sorted) loss.update_terminal_regions( tree.tree_, X, y, residual, raw_predictions, sample_weight, learning_rate=self.learning_rate, k=k) self.estimators_[i, k] = tree return raw_predictionsresidual = y - raw_predictions.ravel()· 回归 MAE
residual = 2 * (y - raw_predictions >0) - 1· 回归 Huber
raw_predictions =raw_predictions.ravel()diff = y - raw_predictionsgamma = np.percentile(np.abs(diff), self.alpha * 100)gamma_mask = np.abs(diff) <= gammaresidual = np.zeros((y.shape[0],), dtype=np.float64)residual[gamma_mask] = diff[gamma_mask]residual[~gamma_mask] = gamma * np.sign(diff[~gamma_mask])· 回归 Quantile
raw_predictions =raw_predictions.ravel()mask = y > raw_predictionsresidual = (alpha * mask) - ((1 - alpha) * ~mask)· 二分类 Logistic sigmoid
residual = y -scipy.special.expit(raw_predictions.ravel())· 二分类 Exponential loss，来源于AdaBoost
y_ = -(2. * y - 1.)residual = y_ * np.exp(y_ * raw_predictions.ravel())· 多分类 Softmax
residual = y -np.nan_to_num(np.exp(raw_predictions[:, k] - logsumexp(raw_predictions, axis=1))回归任务不使用上述的第三步。
def update_terminal_regions(self, tree, X, y, residual, raw_predictions, sample_weight, learning_rate=0.1, k=0): terminal_regions = tree.apply(X) for leaf in np.where(tree.children_left == TREE_LEAF)[0]: self._update_terminal_region(tree, terminal_regions, leaf, X, y, residual, raw_predictions[:, k], sample_weight) raw_predictions[:, k] += \ learning_rate * tree.value[:, 0, 0].take(terminal_regions, axis=0) def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, residual, raw_predictions, sample_weight): terminal_region = np.where(terminal_regions == leaf)[0] residual = residual.take(terminal_region, axis=0) y = y.take(terminal_region, axis=0) sample_weight = sample_weight.take(terminal_region, axis=0) numerator = np.sum(sample_weight * residual) denominator = np.sum(sample_weight * (y - residual) * (1 - y + residual)) if abs(denominator) < 1e-150: tree.value[leaf, 0, 0] = 0.0 else: tree.value[leaf, 0, 0] = numerator / denominatordef feature_importances_(self): relevant_trees = [tree for stage in self.estimators_ for tree in stage if tree.tree_.node_count > 1] if not relevant_trees: return np.zeros(shape=self.n_features_, dtype=np.float64) relevant_feature_importances = [ tree.tree_.compute_feature_importances(normalize=False) for tree in relevant_trees ] avg_feature_importances = np.mean(relevant_feature_importances, axis=0, dtype=np.float64) return avg_feature_importances / np.sum(avg_feature_importances)