My first foray into creating classes in Python. In this case, the example is for determining statistical properties of data and printing it.
'''
Stats_Class_np.py
Author
Dan Patterson
Dept of Geography and Environmental Studies
Carleton University, Ottawa, Canada
Dan_Patterson@carleton.ca
Date March 2010
Edited June 2013
Purpose
Obtain statistical information for a data list when numpy is available.
Requirements
The data list must contain at least 1 elements in order to use this module.
Some parameters (variance, std dev, skewness and kurtosis) require 2 to 4
elements as a minimum.
Useage
Create a Statistics object by passing a list to it.
stats = Statistics(a_list)
The list will be validated and various base parameters
determined for use in other methods. These include
N size of data set
is_numeric True/False boolean: list of numbers / list of text
is_valid True/False boolean: true if list contains at least one element
total sum of values
avg average
min minimum
max max
The rest of the properties are calculated from these base parameters.
'''
import math
import numpy as np
class Statistics(list):
'''
A simple data class enabling statistical calculations and
frequency counts on valid integer/float values. Data are sorted
to facilitate many operations, see above for requirements
'''
def __init__(self, an_array=None):
'''
Tests for lists of numbers, if the input contains numbers, these values
are processed. String lists or lists that contain strings will be processed
as strings whether they contain numbers or not.
'''
data = [x for x in an_array if isinstance(x, (int,float)) ]
if len(an_array) == 0: #empty list check
self.is_valid = False
elif len(data) == len(an_array): # all entries numeric
self.data = data
self.N = len(data)
self.is_numeric = True
self.is_valid = True
self.total = np.sum(self.data)
self.avg = np.mean(self.data)
self.min = np.min(self.data)
self.max = np.max(self.data)
else: # all string or mixed numbers and string
self.data = data
self.N = len(self.data)
self.is_valid = True
self.is_numeric = False
self.total = None
self.avg = None
self.min = None
self.max = None
def kurtosis(self, a_list=None):
'''Momental, excess and unbiased kurtosis'''
if self.is_numeric and self.N > 3: #at least 4 samples found
avg_x = self.avg
n = float(self.N)
diff =(self.data - self.avg)
x_2 = np.sum(np.power(diff, 2))
x_4 = np.sum(np.power(diff, 4))
var_x = np.var(self.data) #variance, std dev population/batch
std_x = math.sqrt(var_x)
vars_x = var_x * (float(n)/(n-1))
stds_x = math.sqrt(vars_x)
if var_x > 0.0:
kurt_mom = x_4 /( n * (std_x**4)) #momental kurtosis
kurt_excess = kurt_mom - 3.0 #excess kurtosis
a = ((n**2) * (n + 1))/((n - 1)*(n - 2)*(n - 3))
b = 3.0 *((n - 1)**2)/((n - 2)*(n - 3))
kurt_unbias = (x_4 /((stds_x**4)*n)) * a - b #unbiased kurtosis
else:
kurt_mom= None
kurt_excess = None
kurt_unbias = None
return [kurt_mom, kurt_excess, kurt_unbias]
def median(self):
'''Median of a data set'''
med_x = None
if self.is_numeric:
a_list = self.data
N = self.N
avg_x = self.avg
med_x = np.median(self.data)
return med_x
def mode(self):
'''Returns the mode, uni-modal only'''
class_list, count_list = self.freq_man()
a_set = list(set(count_list))
uniq_n = len(a_set)
mode_val = None
if uniq_n > 1:
zip_list = zip(count_list,class_list)
zip_list.sort()
mode_val = zip_list[-1][1]
return mode_val
def skewness(self):
'''Momental and unbiased skewness'''
skew_x = None
skews_x = None
if self.is_numeric and self.N > 1: #at least 2 samples found
avg_x = self.avg
n = float(self.N)
diff =(self.data - self.avg)
x_2 = np.sum(np.power(diff, 2))
x_3 = np.sum(np.power(diff, 3))
var_x = x_2 / n
std_x= math.sqrt(var_x)
vars_x = x_2 / (n-1)
stds_x= math.sqrt(vars_x)
skew_x = x_3 / (n * (std_x**3))
if self.N > 2:
skews_x = (x_3 / (n * (stds_x**3)))*((n**2)/((n-1)*(n-2)))
return [skew_x, skews_x]
def std_dev(self):
'''Standard deviation, population and sample values'''
std_x = None
stds_x = None
if self.is_numeric and self.N > 1:
var_x, vars_x = self.variance()
std_x = math.sqrt(var_x)
stds_x = math.sqrt(vars_x)
return [std_x, stds_x]
def variance(self):
'''Variance, population and sample values'''
var_x = None
vars_x = None
if self.is_numeric and self.N > 1:
n = float(self.N)
var_x = np.var(self.data) #variance, std dev population/batch
vars_x = var_x * (float(n)/(n-1)) #variance, std dev sample
return [var_x, vars_x]
def stats_full(self):
'''Returns basic statistics for an input list'''
#establish defaults
var_x = None; vars_x = None
std_x = None; stds_x = None
skew_x = None; skews_x = None
kurt_mom = None; kurt_excess = None; kurt_unbias = None
if self.is_numeric and self.is_valid:
a_list = self.data
N = self.N
n = float(N)
a_list.sort() #Min and Max
min_x = self.min
max_x = self.max
sum_x = self.total
avg_x = self.avg #Average
x_2 = 0.0; x_3 = 0.0; x_4 = 0.0
diff =(self.data - self.avg)
x_2 = np.sum(np.power(diff, 2))
x_3 = np.sum(np.power(diff, 3))
x_4 = np.sum(np.power(diff, 4))
var_x = x_2 / n #variance, std dev population/batch
std_x = math.sqrt(var_x)
if self.N > 1:
vars_x = x_2 / (n-1) #variance, std dev sample
stds_x = math.sqrt(vars_x)
if var_x != 0.0: #skewness and kurtosis
skew_x = x_3 / (n * (std_x**3))
kurt_mom = x_4 /( n * (std_x**4))
kurt_excess = kurt_mom - 3.0
if self.N > 2:
skews_x = (x_3 / (n * (stds_x**3)))*((n**2) / ((n-1) * (n-2)))
if self.N > 3:
a = ((n**2) * (n + 1))/((n - 1)*(n - 2)*(n - 3))
b = 3.0 *((n - 1)**2)/((n - 2)*(n - 3))
kurt_unbias = (x_4 /((stds_x**4)*n)) * a - b
med_x = self.median() # median
mode_x = self.mode() # mode
return_list = [N, sum_x, min_x, max_x, avg_x, med_x, mode_x,
std_x, stds_x, var_x, vars_x,
skew_x, skews_x,
kurt_mom, kurt_excess, kurt_unbias]
return return_list
def freq_class(self):
'''
Determines the frequency count for a valid list or array
of data using its unique values as the classification scheme.
Returns a dictionary of key:value pairs. Uses numpy if
available.
'''
uni_vals = np.unique(self.data)
histo = np.histogram(self.data, uni_vals)
class_list = histo[1]
count_list = histo[0]
return [class_list, count_list]
def freq_man(self, class_list=None):
'''
Determines the frequency distribution for a valid list of data
given an optional list of class values. If None is specified for class_list
then the set of unique values is used. The class values need not contain all
the possible values in the input list. A useful way of collapsing counts
over shorter intervals that would be obtained by the freq_class method.
integer or text counting can be accomplished by
count_list = [ a_list.count(a_val) for a_val in class_list ]
'''
if class_list == None:
class_list = np.unique(self.data)
histo = np.histogram(self.data, class_list)
count_list = histo[0]
class_list = histo[1]
print "count_list", count_list
print "class_list",class_list
return [class_list, count_list]
def freq_std(self, std=None, step_by=None):
'''
Input a Statistics object, std and std step.
Returns the counts, cum freq and cum rel freq from -3 to +3 std
increments.
'''
a_list = self.data
avg = self.avg
a_list.sort()
N = self.N
if std == None:
std, stds = self.std_dev()
if std == None:
return None
if step_by == None:
step_by = 1
std_class = []; std_val = []; std_count = []; cf = []; crf = []
i = -3.0
while i < (3.0 + step_by):
b = avg + (i * std)
std_class.append(i)
std_val.append(b)
std_count.append(0)
cf.append(0)
crf.append(0.0)
i += step_by
for i in range(N):
x = a_list[i]
for j in range(len(std_class)):
if x <= std_val[j]:
std_count[j] += 1
cf[j] += std_count[j]
break
cf[0] = std_count[0]
crf[0] = cf[0] / float(N)
for i in range(1,len(std_class)):
cf[i] = cf[i-1] + std_count[i]
crf[i] = cf[i] / float(N)
return [std_class, std_val, std_count, cf, crf]
def flatten(a_list):
'''flatten a 2D list and return a 1D list'''
if isinstance(a_list[0], (list,tuple)):
out_list = []
for an_ell in a_list:
out_list.extend(an_ell)
return out_list
return a_list
def print_full(stats):
'''Format stats for printing, produced by stats_full'''
s = "%-20s %-15s" % ("Statistic", "Value")
headers = ["count", "sum", "minimum", "maximum", "average", "median", "mode",
"std dev (pop.)", "std dev (sample)",
"variance (pop.)", "variance (sample)",
"skewness (pop.)", "skewness (sample)",
"kurtosis (momental)", "kurtosis (excess)", "kurtosis (unbiased)"]
for i in range(len(stats)):
s += "\n%-20s %15s" % (headers[i], str(stats[i]))
return s
def print_class(class_list, count_list):
'''
Returns a string of a class-counts dictionary produced by freq_class.
The data need not be numeric.
'''
s = "%10s %10s" % ("Class", "Count")
for i in range(len(count_list)): #class_list)):
s += "\n%10s %5s" % (class_list[i], count_list[i])
return s
def print_man(class_list, count_list):
'''
Returns the frequency distribution for a manual classification for
printing produced by freq_man
'''
s = "%10s %5s" % ("Upper limit", "Count")
for i in range(len(count_list)): #class_list)):
s += "\n%10s %5s" % (class_list[i], count_list[i])
return s
def print_std(stats):
'''
Returns the frequency distribution obtained by freq_std for printing purposes
'''
if stats == None:
s = "Cannot perform 'print_std' since sample size is < 2"
else:
L1, L2, L3, L4, L5 = stats
form = "%10s %12s %10s %10s %10s"
headers = ("std dev", "value", "freq", "cf", "crf")
s = form % headers
for i in range(len(L1)):
s += "\n%10s %12.6f %10s %10s %10.6f" % (L1[i], L2[i], L3[i], L4[i], L5[i])
return s
def validate_data(a_list):
stats = Statistics(a_list)
return stats
if __name__ == "__main__":
'''
invalid data list ( [] )
data list with 1,2,3,4,5 or more values
data list with text (text is removed)
'''
data_list = [ [],
[1],
[1,2],
[1,2,3],
[1,2,3,4],
[1,2,3,4,5],
[1,1,1,2,2,2,2,2,3,3,3,3,3],
[1,1,1,2,2,2,2,2,3,3,3,3,3, "x"],
["a","b","a","c","d","b"]
]
#data_list = [ [1,2,3] ]
data_list = [ [ [1,2,3],[4,5,6],[7,8,9] ] ] #2D array
for a_list in data_list:
an_array = (np.array(a_list)).flatten()
stats = Statistics(an_array)
if not stats.is_valid:
print "\nData list must contain at least 1 element", str(stats)
continue
if stats.is_numeric:
print "\nStats for dataset: \n", stats.data
print "N ", stats.N
print "total", stats.total
print "avg ", stats.avg
print "min ", stats.min
print "max ", stats.max
print "median",stats.median()
print "mode", stats.mode()
print "variance (pop., sample)", stats.variance()
print "std_dev (pop., sample)", stats.std_dev()
print "skewness (pop., sample)", stats.skewness()
print "kurtosis (Momental, excess and unbiased)", stats.kurtosis()
print "\nFull Statistics"
print print_full( stats.stats_full() )
print "\nVarious counting options"
print "\nFreq by std dev breaks"
print print_std( stats.freq_std() )
print "\nFreq by range"
class_list, count_list = stats.freq_man()
print class_list, count_list
print print_man(class_list, count_list)
print "\nFreq by class"
class_list, count_list = stats.freq_class()
print print_class(class_list, count_list)
else:
print "Data are not all numeric, only counts performed"
stats = Statistics(a_list)
print "\nFreq by range"
class_list, count_list = stats.freq_man()
print print_man(class_list, count_list)
print "\nFreq by class"
class_list, count_list = stats.freq_class()
print print_class(class_list, count_list)
No comments:
Post a Comment