Source code for mlbox.preprocessing.drift_thresholder
# coding: utf-8
# Author: Axel ARONIO DE ROMBLAY <axelderomblay@gmail.com>
# License: BSD 3 clause
import os
import time
from sklearn.pipeline import Pipeline
from .drift import DriftThreshold
from ..encoding.na_encoder import NA_encoder
from ..encoding.categorical_encoder import Categorical_encoder
[docs]class Drift_thresholder():
"""Automatically drops ids and drifting variables between train and test datasets.
Drops on train and test datasets. The list of drift coefficients is available and
saved as "drifts.txt". To get familiar with drift:
https://github.com/AxeldeRomblay/MLBox/blob/master/docs/webinars/features.pdf
Parameters
----------
threshold : float, defaut = 0.6
Drift threshold under which features are kept. Must be between 0. and 1.
The lower the more you keep non-drifting/stable variables: a feature with
a drift measure of 0. is very stable and a one with 1. is highly unstable.
inplace : bool, default = False
If True, train and test datasets are transformed. Returns self.
Otherwise, train and test datasets are not transformed. Returns a new dictionnary with
cleaned datasets.
verbose : bool, default = True
Verbose mode
to_path : str, default = "save"
Name of the folder where the list of drift coefficients is saved.
"""
def __init__(self,
threshold=0.6,
inplace=False,
verbose=True,
to_path="save"):
self.threshold = threshold
self.inplace = inplace
self.verbose = verbose
self.to_path = to_path
self.__Ddrifts = {}
self.__fitOK = False
[docs] def fit_transform(self, df):
"""Fits and transforms train and test datasets
Automatically drops ids and drifting variables between train and test datasets.
The list of drift coefficients is available and saved as "drifts.txt"
Parameters
----------
df : dict, defaut = None
Dictionnary containing :
- 'train' : pandas dataframe for train dataset
- 'test' : pandas dataframe for test dataset
- 'target' : pandas serie for the target on train set
Returns
-------
dict
Dictionnary containing :
- 'train' : transformed pandas dataframe for train dataset
- 'test' : transformed pandas dataframe for test dataset
- 'target' : pandas serie for the target on train set
"""
######################################################
# Deleting IDs
######################################################
# Exception
if (df["test"].shape[0] == 0):
if (self.verbose):
print("")
print("You have no test dataset...")
return df
else:
start_time = time.time()
ds = DriftThreshold(self.threshold)
na = NA_encoder(numerical_strategy=0)
ca = Categorical_encoder()
pp = Pipeline([("na", na), ("ca", ca)])
pp.fit(df['train'], None)
# Deleting IDs with drift threshold method
if (self.verbose):
print("")
print("computing drifts ...")
ds.fit(pp.transform(df['train']), pp.transform(df['test']))
if (self.verbose):
print("CPU time: %s seconds" % (time.time() - start_time))
print("")
self.__fitOK = True
self.__Ddrifts = ds.drifts()
drifts_top = sorted(ds.drifts().items(),
key=lambda x: x[1],
reverse=True)[:10]
if (self.verbose):
print("> Top 10 drifts")
print("")
for d in range(len(drifts_top)):
print(drifts_top[d])
if (self.verbose):
print("")
print("> Deleted "
"variables : " + str(ds.get_support(complement=True)))
######################################################
# Dumping Encoders into directory
######################################################
if (self.to_path is not None):
try:
os.mkdir(self.to_path)
except OSError:
pass
file = open(self.to_path + '/drifts.txt', "w")
file.write("\n")
file.write(
"*******************************************"
" Drifts coefficients "
"*******************************************\n")
file.write("\n")
for var, d in sorted(ds.drifts().items(),
key=lambda x: x[1],
reverse=True):
file.write(str(var) + " = " + str(d) + '\n')
file.close()
if (self.verbose):
print("> Drift coefficients dumped into directory : " + self.to_path)
# Returning datasets with no IDs
if (self.inplace):
df['train'] = ds.transform(df['train'])
df['test'] = ds.transform(df['test'])
else:
return {'train': ds.transform(df['train']),
'test': ds.transform(df['test']),
'target': df['target']}
[docs] def drifts(self):
"""Returns the univariate drifts for all variables.
Returns
-------
dict
Dictionnary containing the drifts for each feature
"""
if self.__fitOK:
return self.__Ddrifts
else:
raise ValueError('Call the fit_transform function before !')