65579-Python 金融大数据风控建模实战_源代码.zip

# numpy and pandas for data manipulation # model used for feature importances import lightgbm as lgb # visualizations import matplotlib.pyplot as plt import numpy as np import pandas as pd # utility for early stopping with a validation set from sklearn.model_selection import train_test_split plt.switch_backend('agg') import seaborn as sns # memory management import gc # utilities from itertools import chain class FeatureSelector(): """ Class for performing feature selection for machine learning or data preprocessing. Implements five different methods to identify features for removal 1. Find columns with a missing percentage greater than a specified threshold 2. Find columns with a single unique value 3. Find collinear variables with a correlation greater than a specified correlation coefficient 4. Find features with 0.0 feature importance from a gradient boosting machine (gbm) 5. Find low importance features that do not contribute to a specified cumulative feature importance from the gbm Parameters -------- data : dataframe A dataset with observations in the rows and features in the columns labels : array or series, default = None Array of labels for training the machine learning model to find feature importances. These can be either binary labels (if task is 'classification') or continuous targets (if task is 'regression'). If no labels are provided, then the feature importance based methods are not available. Attributes -------- ops : dict Dictionary of operations run and features identified for removal missing_stats : dataframe The fraction of missing values for all features record_missing : dataframe The fraction of missing values for features with missing fraction above threshold unique_stats : dataframe Number of unique values for all features record_single_unique : dataframe Records the features that have a single unique value corr_matrix : dataframe All correlations between all features in the data record_collinear : dataframe Records the pairs of collinear variables with a correlation coefficient above the threshold feature_importances : dataframe All feature importances from the gradient boosting machine record_zero_importance : dataframe Records the zero importance features in the data according to the gbm record_low_importance : dataframe Records the lowest importance features not needed to reach the threshold of cumulative importance according to the gbm Notes -------- - All 5 operations can be run with the `identify_all` method. - If using feature importances, one-hot encoding is used for categorical variables which creates new columns """ def __init__(self, data, labels=None): # Dataset and optional training labels self.data = data self.labels = labels if labels is None: print('No labels provided. Feature importance based methods are not available.') self.base_features = list(data.columns) self.one_hot_features = None # Dataframes recording information about features to remove self.record_missing = None self.record_single_unique = None self.record_collinear = None self.record_zero_importance = None self.record_low_importance = None self.missing_stats = None self.unique_stats = None self.corr_matrix = None self.feature_importances = None # Dictionary to hold removal operations self.ops = {} self.one_hot_correlated = False def identify_missing(self, missing_threshold): """Find the features with a fraction of missing values above `missing_threshold`""" self.missing_threshold = missing_threshold # Calculate the fraction of missing in each column missing_series = self.data.isnull().sum() / self.data.shape[0] self.missing_stats = pd.DataFrame(missing_series).rename(columns = {'index': 'feature', 0: 'missing_fraction'}) # Sort with highest number of missing values on top self.missing_stats = self.missing_stats.sort_values('missing_fraction', ascending = False) # Find the columns with a missing percentage above the threshold record_missing = pd.DataFrame(missing_series[missing_series > missing_threshold]).reset_index().rename(columns = {'index': 'feature', 0: 'missing_fraction'}) to_drop = list(record_missing['feature']) self.record_missing = record_missing self.ops['missing'] = to_drop print('%d features with greater than %0.2f missing values.\n' % (len(self.ops['missing']), self.missing_threshold)) def identify_single_unique(self): """Finds features with only a single unique value. NaNs do not count as a unique value. """ # Calculate the unique counts in each column unique_counts = self.data.nunique() self.unique_stats = pd.DataFrame(unique_counts).rename(columns = {'index': 'feature', 0: 'nunique'}) self.unique_stats = self.unique_stats.sort_values('nunique', ascending = True) # Find the columns with only one unique count record_single_unique = pd.DataFrame(unique_counts[unique_counts == 1]).reset_index().rename(columns = {'index': 'feature', 0: 'nunique'}) to_drop = list(record_single_unique['feature']) self.record_single_unique = record_single_unique self.ops['single_unique'] = to_drop print('%d features with a single unique value.\n' % len(self.ops['single_unique'])) def identify_collinear(self, correlation_threshold, one_hot=False): """ Finds collinear features based on the correlation coefficient between features. For each pair of features with a correlation coefficient greather than `correlation_threshold`, only one of the pair is identified for removal. Using code adapted from: https://chrisalbon.com/machine_learning/feature_selection/drop_highly_correlated_features/ Parameters -------- correlation_threshold : float between 0 and 1 Value of the Pearson correlation cofficient for identifying correlation features one_hot : boolean, default = False Whether to one-hot encode the features before calculating the correlation coefficients """ self.correlation_threshold = correlation_threshold self.one_hot_correlated = one_hot # Calculate the correlations between every column if one_hot: # One hot encoding features = pd.get_dummies(self.data) self.one_hot_features = [column for column in features.columns if column not in self.base_features] # Add one hot encoded data to original data self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1) corr_matrix = pd.get_dummies(features).corr() else: corr_matrix = self.data.corr()