In [2]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from time import time
%matplotlib inline
In [26]:
df = pd.read_csv('TrainingData.csv', index_col=0)
df.shape
Out[26]:
In [27]:
df.info()
In [28]:
df.columns
Out[28]:
In [31]:
df.head()
Out[31]:
In [30]:
# df.columns = ['ID']+list(df.columns[1:])
In [36]:
df['Job_Title_Description'].value_counts().head(10)
Out[36]:
In [43]:
df['Object_Type'].value_counts().head(10)
Out[43]:
columns¶
- FTE: Stands for "full-time equivalent". If the budget item is associated to an employee, this number tells us the percentage of full-time that the employee works. A value of 1 means the associated employee works for the schooll full-time. A value close to 0 means the item is associated to a part-time or contracted employee.
- Total: Stands for the total cost of the expenditure. This number tells us how much the budget item cost.
In [48]:
df['FTE'].value_counts().head()
Out[48]:
In [50]:
df['Total'].describe()
Out[50]:
In [58]:
plt.xlim?
In [67]:
# Create the histogram
plt.hist(df['FTE'].dropna(),bins=500,alpha=.5,color='r')
plt.xlim([-.2,1.2])
# Add title and labels
plt.title('Distribution of %full-time \n employee works')
plt.xlabel('% of full-time')
plt.ylabel('num employees')
# Display the histogram
plt.show()
datatypes¶
In [93]:
# sample_df.label = sample_df.label.astype('category')
Dummy variable encoding¶
In [71]:
df.head(1)
Out[71]:
In [70]:
df.dtypes
Out[70]:
In [75]:
df['Use'].value_counts()
Out[75]:
In [79]:
type(df['Use']), type(df[['Use']])
Out[79]:
In [81]:
dummies = pd.get_dummies(df[['Use']], prefix_sep='_')
dummies.head()
Out[81]:
In [82]:
dummies = pd.get_dummies(df['Use'], prefix_sep='_')
dummies.head()
Out[82]:
Lambda functions¶
In [84]:
categorize_label = lambda x: x.astype('category')
In [85]:
type(categorize_label)
Out[85]:
In [92]:
df_use = df[['Use']].apply(categorize_label, axis=0)
df_use.info()
In [96]:
df.Function.astype('category').dtypes
Out[96]:
In [104]:
df.apply(categorize_label, axis=0).dtypes
Out[104]:
In [103]:
df.dtypes.value_counts()
Out[103]:
In [102]:
df.apply(categorize_label).dtypes.value_counts()
Out[102]:
Counting unique labels¶
In [109]:
# pd.Series.nunique?
# Return number of unique elements in the object.
In [115]:
df['Function'].nunique()
Out[115]:
In [122]:
for i in df: print i, '--- unique values: ', df[i].nunique()
In [123]:
# Calculate number of unique values for each label: num_unique_labels
num_unique_labels = df.apply(pd.Series.nunique)
num_unique_labels
Out[123]:
In [131]:
# Plot number of unique values for each label
plt.figure(figsize=[10,5])
num_unique_labels.plot(kind='bar',alpha=.6)
plt.ylim([0,2000])
# Label the axes
plt.xlabel('Labels')
plt.ylabel('Number of unique values')
# Display the plot
plt.show()
In [145]:
# np.clip?
# np.clip(a, a_min, a_max, out=None)
# Docstring:
# Clip (limit) the values in an array.
# Given an interval, values outside the interval are clipped to
# the interval edges. For example, if an interval of ``[0, 1]``
# is specified, values smaller than 0 become 0, and values larger
# than 1 become 1.
In [143]:
np.clip(range(1,10), 3, 7)
Out[143]:
Log loss binary classification¶
● Log loss for binary classification
● Actual value: y = {1=yes, 0=no}
● Prediction (probability that the value is 1): p
logloss(N=1) = y log(p) + (1 − y) log(1 − p)¶
In [6]:
def compute_log_loss(predicted, actual, eps=1e-14):
import numpy as np
""" Computes the logarithmic loss between predicted and
actual when these are 1D arrays.
:param predicted: The predicted probabilities as floats between 0-1
:param actual: The actual binary labels. Either 0 or 1.
:param eps (optional): log(0) is inf, so we need to offset our
predicted values slightly by eps from 0 or 1.
"""
predicted = np.clip(predicted, eps, 1 - eps)
loss = -1 * np.mean(actual * np.log(predicted) + (1 - actual)* np.log(1 - predicted))
return loss
In [7]:
compute_log_loss(predicted=0.9, actual=0)
Out[7]:
In [8]:
compute_log_loss(predicted=0.5, actual=1)
Out[8]:
In [ ]: