From 4be9f36f99376e99a387d828bfa834b56fe1ac42 Mon Sep 17 00:00:00 2001 From: mercierc Date: Thu, 5 Aug 2021 11:32:16 +1200 Subject: [PATCH] stats: fixed the computation of variance when it is equal to 0 --- python/obitools3/commands/stats.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/obitools3/commands/stats.pyx b/python/obitools3/commands/stats.pyx index 7bae6bc..152e915 100755 --- a/python/obitools3/commands/stats.pyx +++ b/python/obitools3/commands/stats.pyx @@ -119,9 +119,12 @@ def mean(values, options): def variance(v): if len(v)==1: - return 0 + return 0 s = reduce(lambda x,y:(x[0]+y,x[1]+y**2),v,(0.,0.)) - return s[1]/(len(v)-1) - s[0]**2/len(v)/(len(v)-1) + var = round(s[1]/(len(v)-1) - s[0]**2/len(v)/(len(v)-1), 5) # round to go around shady python rounding stuff when var is actually 0 + if var == -0.0: # then fix -0 to +0 if was rounded to -0 + var = 0.0 + return var def varpop(values, options):