From 7973d45a92d6625bc6a40bdd10331d820b49c994 Mon Sep 17 00:00:00 2001 From: dfinke Date: Sun, 1 Oct 2017 13:05:55 -0400 Subject: [PATCH 01/19] First pass at adding std deviation to measure-object --- .../commands/utility/Measure-Object.cs | 56 ++++++++++++++++++- .../Measure-Object.Tests.ps1 | 22 ++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 0a16b3a3155..45c4e85eb6a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -34,7 +34,7 @@ public sealed class GenericMeasureInfo : MeasureInfo /// public GenericMeasureInfo() { - Average = Sum = Maximum = Minimum = null; + Average = Sum = Maximum = Minimum = StdDeviation = null; } /// @@ -71,6 +71,13 @@ public GenericMeasureInfo() /// /// public double? Minimum { get; set; } + + /// + /// + /// The StdDeviation of property values + /// + /// + public double? StdDeviation { get; set; } } /// @@ -90,7 +97,7 @@ public sealed class GenericObjectMeasureInfo : MeasureInfo /// public GenericObjectMeasureInfo() { - Average = Sum = null; + Average = Sum = StdDeviation = null; Maximum = Minimum = null; } @@ -128,6 +135,13 @@ public GenericObjectMeasureInfo() /// /// public object Minimum { get; set; } + + /// + /// + /// The StdDeviation of property values + /// + /// + public double? StdDeviation { get; set; } } /// @@ -225,6 +239,8 @@ private class Statistics // Generic/Numeric statistics internal double sum = 0.0; + internal double stdDeviation = 0.0; + internal List stdDeviationNumbers = new List(); internal object max = null; internal object min = null; @@ -263,6 +279,25 @@ public MeasureObjectCommand() #endregion Common parameters in both sets + /// + /// Set to true is StdDeviation is to be returned + /// + /// + [Parameter(ParameterSetName = GenericParameterSet)] + public SwitchParameter StdDeviation + { + get + { + return _measureStdDeviation; + } + set + { + _measureStdDeviation = value; + } + } + + private bool _measureStdDeviation; + /// /// Set to true is Sum is to be returned /// @@ -710,6 +745,8 @@ private void AnalyzeNumber(double numValue, Statistics stat) { if (_measureSum || _measureAverage) stat.sum += numValue; + if (_measureStdDeviation) + stat.stdDeviationNumbers.Add(numValue); } /// @@ -790,6 +827,7 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene { double? sum = null; double? average = null; + double? stdDeviation = null; object max = null; object min = null; @@ -797,8 +835,21 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene { if (_measureSum) sum = stat.sum; + if (_measureAverage && stat.count > 0) average = stat.sum / stat.count; + + if (_measureStdDeviation && _measureAverage && stat.count > 0) + { + var popdev = 0.0; + + foreach (double n in stat.stdDeviationNumbers) + { + popdev += Math.Pow((n - (double)average), 2); + } + + stdDeviation = Math.Round(Math.Sqrt(popdev / (stat.stdDeviationNumbers.Count - 1)), 4); + } } if (_measureMax) @@ -835,6 +886,7 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene gmi.Count = stat.count; gmi.Sum = sum; gmi.Average = average; + gmi.StdDeviation = stdDeviation; if (null != max) { gmi.Maximum = (double)max; diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index 6791d790a1a..9674b384a11 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -17,6 +17,28 @@ Describe "Measure-Object" -Tags "CI" { $($testObject | Measure-Object).Count | Should Be $testObject.Length } + + It "Should calculate Standard Deviation" { + $actual = ($testObject | Measure-Object -Average -StdDeviation) + $actual.StdDeviation | Should Be 1.5275 + } + + It "Should throw if Standard Deviation with -Average not specified" { + #{$testObject | Measure-Object -StdDeviation} | Should Throw + $actual = $testObject | Measure-Object -StdDeviation + $actual.StdDeviation | Should Be 1.5275 + } + + It 'Should throw if Standard Deviation requested and -Average:$false' { + try { + $Error.Clear() + $testObject | Measure-Object -StdDeviation -Average:$false -ErrorAction Stop + } + catch { + $Error.Count | should be 1 + } + } + It "Should be able to count using the Property switch" { $expected = $(Get-ChildItem $TestDrive).Length $actual = $(Get-ChildItem $TestDrive | Measure-Object -Property Length).Count From ea0cc8c16462f2562429057e0c1858e40efe97fa Mon Sep 17 00:00:00 2001 From: dfinke Date: Mon, 2 Oct 2017 16:48:05 -0400 Subject: [PATCH 02/19] Use A*A instead of Math.Pow. Throw if -Average not specified and fixed comments --- .../commands/utility/Measure-Object.cs | 18 ++++++++++++------ .../Measure-Object.Tests.ps1 | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 45c4e85eb6a..5782e316d29 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -74,7 +74,7 @@ public GenericMeasureInfo() /// /// - /// The StdDeviation of property values + /// The Standard Deviation of property values /// /// public double? StdDeviation { get; set; } @@ -138,7 +138,7 @@ public GenericObjectMeasureInfo() /// /// - /// The StdDeviation of property values + /// The Standard Deviation of property values /// /// public double? StdDeviation { get; set; } @@ -280,7 +280,7 @@ public MeasureObjectCommand() #endregion Common parameters in both sets /// - /// Set to true is StdDeviation is to be returned + /// Set to true if Standard Deviation is to be returned /// /// [Parameter(ParameterSetName = GenericParameterSet)] @@ -839,16 +839,22 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene if (_measureAverage && stat.count > 0) average = stat.sum / stat.count; + if(_measureStdDeviation && !_measureAverage) { + var message = "StdDeviation was requested and requires the average to be calculated, please add the -Average switch"; + throw new ArgumentException(message); + } + if (_measureStdDeviation && _measureAverage && stat.count > 0) { - var popdev = 0.0; + var sumOfDerivation = 0.0; foreach (double n in stat.stdDeviationNumbers) { - popdev += Math.Pow((n - (double)average), 2); + var m = n - (double)average; + sumOfDerivation += m * m; } - stdDeviation = Math.Round(Math.Sqrt(popdev / (stat.stdDeviationNumbers.Count - 1)), 4); + stdDeviation = Math.Round(Math.Sqrt(sumOfDerivation / (stat.stdDeviationNumbers.Count - 1)), 4); } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index 9674b384a11..6296f3466a9 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -17,7 +17,6 @@ Describe "Measure-Object" -Tags "CI" { $($testObject | Measure-Object).Count | Should Be $testObject.Length } - It "Should calculate Standard Deviation" { $actual = ($testObject | Measure-Object -Average -StdDeviation) $actual.StdDeviation | Should Be 1.5275 From b68253c9edb0c1e435ddc4626d488630ceba4620 Mon Sep 17 00:00:00 2001 From: dfinke Date: Mon, 2 Oct 2017 19:32:36 -0400 Subject: [PATCH 03/19] Added message to the rex file --- .../commands/utility/Measure-Object.cs | 10 ++++++++-- .../resources/MeasureObjectStrings.resx | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 5782e316d29..4dbeaa5406d 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -840,8 +840,14 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene average = stat.sum / stat.count; if(_measureStdDeviation && !_measureAverage) { - var message = "StdDeviation was requested and requires the average to be calculated, please add the -Average switch"; - throw new ArgumentException(message); + ErrorRecord errorRecord = new ErrorRecord( + PSTraceSource.NewArgumentException("Average"), + "AverageSwitchNotSet", + ErrorCategory.InvalidArgument, + null); + + errorRecord.ErrorDetails = new ErrorDetails(this, "MeasureObjectStrings", "AverageSwitchNotSet", "Average"); + WriteError(errorRecord); } if (_measureStdDeviation && _measureAverage && stat.count > 0) diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx index 8211114c335..c5c624771ac 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx @@ -126,4 +126,7 @@ Input object "{0}" is not numeric. + + StdDeviation was requested and requires the average to be calculated, please add the -Average switch. + From df488a3aa09dfbdcc0e5d99c02392de91207119a Mon Sep 17 00:00:00 2001 From: dfinke Date: Tue, 3 Oct 2017 13:53:11 -0400 Subject: [PATCH 04/19] Set Average true if StdDeviation is set --- .../commands/utility/Measure-Object.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 4dbeaa5406d..e99ad3e5a6e 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -293,6 +293,8 @@ public SwitchParameter StdDeviation set { _measureStdDeviation = value; + if(value == true) + _measureAverage = true; } } From 515fb010f70e5fcef1e95e5e05bb32b7187e537c Mon Sep 17 00:00:00 2001 From: dfinke Date: Tue, 3 Oct 2017 13:53:19 -0400 Subject: [PATCH 05/19] Update message --- .../resources/MeasureObjectStrings.resx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx index c5c624771ac..bccbd5db10c 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx @@ -127,6 +127,6 @@ Input object "{0}" is not numeric. - StdDeviation was requested and requires the average to be calculated, please add the -Average switch. + StdDeviation was requested and requires the average to be calculated, however '-Average' was set to $false. From f719ad9ab1c4eeb42745c9cd9e270f94778f60f5 Mon Sep 17 00:00:00 2001 From: CloudyDino Date: Sat, 24 Feb 2018 18:01:29 -0500 Subject: [PATCH 06/19] Addressed comments by @iSazonov and @SteveL-MSFT --- .../commands/utility/Measure-Object.cs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index e99ad3e5a6e..de34bf7b206 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -293,8 +293,9 @@ public SwitchParameter StdDeviation set { _measureStdDeviation = value; - if(value == true) + if (value) { _measureAverage = true; + } } } @@ -841,28 +842,17 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene if (_measureAverage && stat.count > 0) average = stat.sum / stat.count; - if(_measureStdDeviation && !_measureAverage) { - ErrorRecord errorRecord = new ErrorRecord( - PSTraceSource.NewArgumentException("Average"), - "AverageSwitchNotSet", - ErrorCategory.InvalidArgument, - null); - - errorRecord.ErrorDetails = new ErrorDetails(this, "MeasureObjectStrings", "AverageSwitchNotSet", "Average"); - WriteError(errorRecord); - } - if (_measureStdDeviation && _measureAverage && stat.count > 0) { var sumOfDerivation = 0.0; foreach (double n in stat.stdDeviationNumbers) { - var m = n - (double)average; + var m = n - average; sumOfDerivation += m * m; } - stdDeviation = Math.Round(Math.Sqrt(sumOfDerivation / (stat.stdDeviationNumbers.Count - 1)), 4); + stdDeviation = Math.Sqrt(sumOfDerivation / (stat.stdDeviationNumbers.Count - 1)); } } From de861ab8d86344ba8659cfad6c923f84a5943fd3 Mon Sep 17 00:00:00 2001 From: CloudyDino Date: Sat, 24 Feb 2018 19:24:37 -0500 Subject: [PATCH 07/19] Cast average outside for loop --- .../commands/utility/Measure-Object.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index de34bf7b206..39945e7a835 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -846,9 +846,10 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene { var sumOfDerivation = 0.0; + var averageCasted = (double) average; foreach (double n in stat.stdDeviationNumbers) { - var m = n - average; + var m = n - averageCasted; sumOfDerivation += m * m; } From 0f1c554689a993fd3e1b81154fc450cc99deb16c Mon Sep 17 00:00:00 2001 From: CloudyDino Date: Sat, 24 Feb 2018 19:26:09 -0500 Subject: [PATCH 08/19] Updated Measure-Object.Tests.ps1 to match standard deviation calculations --- .../Measure-Object.Tests.ps1 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index 6296f3466a9..fc7e934a354 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -19,13 +19,14 @@ Describe "Measure-Object" -Tags "CI" { It "Should calculate Standard Deviation" { $actual = ($testObject | Measure-Object -Average -StdDeviation) - $actual.StdDeviation | Should Be 1.5275 + # We check this way since .StdDeviation returns a double value + [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } - It "Should throw if Standard Deviation with -Average not specified" { - #{$testObject | Measure-Object -StdDeviation} | Should Throw - $actual = $testObject | Measure-Object -StdDeviation - $actual.StdDeviation | Should Be 1.5275 + It "Should calculate Standard Deviation without -Average" { + $actual = ($testObject | Measure-Object -StdDeviation) + # We check this way since .StdDeviation returns a double value + [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } It 'Should throw if Standard Deviation requested and -Average:$false' { From e42c10e9170d9f43eaf85333601fcd3d7e756c85 Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Sat, 10 Mar 2018 20:41:13 -0500 Subject: [PATCH 09/19] Change Standard Deviation calculation to be iterative --- .../commands/utility/Measure-Object.cs | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 39945e7a835..9134f64620a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -73,9 +73,7 @@ public GenericMeasureInfo() public double? Minimum { get; set; } /// - /// /// The Standard Deviation of property values - /// /// public double? StdDeviation { get; set; } } @@ -137,9 +135,7 @@ public GenericObjectMeasureInfo() public object Minimum { get; set; } /// - /// /// The Standard Deviation of property values - /// /// public double? StdDeviation { get; set; } } @@ -239,8 +235,8 @@ private class Statistics // Generic/Numeric statistics internal double sum = 0.0; + internal double sumPrevious = 0.0; internal double stdDeviation = 0.0; - internal List stdDeviationNumbers = new List(); internal object max = null; internal object min = null; @@ -293,9 +289,6 @@ public SwitchParameter StdDeviation set { _measureStdDeviation = value; - if (value) { - _measureAverage = true; - } } } @@ -563,7 +556,7 @@ private void AnalyzeValue(string propertyName, object objValue) AnalyzeString(strValue, stat); } - if (_measureAverage || _measureSum) + if (_measureAverage || _measureSum || _measureStdDeviation) { double numValue = 0.0; if (!LanguagePrimitives.TryConvertTo(objValue, out numValue)) @@ -746,10 +739,20 @@ private void AnalyzeString(string strValue, Statistics stat) /// private void AnalyzeNumber(double numValue, Statistics stat) { - if (_measureSum || _measureAverage) + if (_measureSum || _measureAverage || _measureStdDeviation) + { + stat.sumPrevious = stat.sum; stat.sum += numValue; + } if (_measureStdDeviation) - stat.stdDeviationNumbers.Add(numValue); + { + double countMinusOne = stat.count - 1; + double avgPrevious = (countMinusOne == 0) ? 0 : (stat.sumPrevious / countMinusOne); + double avg = stat.sum / stat.count; + stat.stdDeviation *= countMinusOne; + stat.stdDeviation += (numValue - avgPrevious) * (numValue - avg); + stat.stdDeviation /= stat.count; + } } /// @@ -842,18 +845,9 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene if (_measureAverage && stat.count > 0) average = stat.sum / stat.count; - if (_measureStdDeviation && _measureAverage && stat.count > 0) + if (_measureStdDeviation) { - var sumOfDerivation = 0.0; - - var averageCasted = (double) average; - foreach (double n in stat.stdDeviationNumbers) - { - var m = n - averageCasted; - sumOfDerivation += m * m; - } - - stdDeviation = Math.Sqrt(sumOfDerivation / (stat.stdDeviationNumbers.Count - 1)); + stdDeviation = stat.stdDeviation; } } @@ -944,7 +938,7 @@ private TextMeasureInfo CreateTextMeasureInfo(Statistics stat) /// /// Whether or not a numeric conversion error occurred. - /// If true, then average/sum will not be output. + /// If true, then average/sum/standard deviation will not be output. /// private bool _nonNumericError = false; From 31c716332ff9d60ebf46702bb90f0682cad2bf66 Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Sun, 11 Mar 2018 20:46:13 -0400 Subject: [PATCH 10/19] Remove unused error AverageSwitchNotSet --- .../resources/MeasureObjectStrings.resx | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx index bccbd5db10c..8211114c335 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/MeasureObjectStrings.resx @@ -126,7 +126,4 @@ Input object "{0}" is not numeric. - - StdDeviation was requested and requires the average to be calculated, however '-Average' was set to $false. - From b6c89dcd5680a09322e4861b4c196feb3ff202cd Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Sun, 11 Mar 2018 23:23:42 -0400 Subject: [PATCH 11/19] Fixed calculation of standard deviation --- .../commands/utility/Measure-Object.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 9134f64620a..2d5c2b91bd5 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -236,7 +236,7 @@ private class Statistics // Generic/Numeric statistics internal double sum = 0.0; internal double sumPrevious = 0.0; - internal double stdDeviation = 0.0; + internal double variance = 0.0; internal object max = null; internal object min = null; @@ -744,14 +744,12 @@ private void AnalyzeNumber(double numValue, Statistics stat) stat.sumPrevious = stat.sum; stat.sum += numValue; } - if (_measureStdDeviation) + if (_measureStdDeviation && stat.count > 1) { double countMinusOne = stat.count - 1; - double avgPrevious = (countMinusOne == 0) ? 0 : (stat.sumPrevious / countMinusOne); - double avg = stat.sum / stat.count; - stat.stdDeviation *= countMinusOne; - stat.stdDeviation += (numValue - avgPrevious) * (numValue - avg); - stat.stdDeviation /= stat.count; + double avgPrevious = stat.sumPrevious / countMinusOne; + stat.variance *= (countMinusOne - 1)/countMinusOne; + stat.variance += (numValue - avgPrevious) * (numValue - avgPrevious) / stat.count; } } @@ -847,7 +845,7 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene if (_measureStdDeviation) { - stdDeviation = stat.stdDeviation; + stdDeviation = Math.Sqrt(stat.variance); } } From 28114aafee01c0c35a8d5a3b6f35c73df62191bd Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Sun, 11 Mar 2018 23:32:00 -0400 Subject: [PATCH 12/19] Added test for standard deviation Asks to calculate the standard deviation for the integers [1, 100] --- .../Measure-Object.Tests.ps1 | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index fc7e934a354..d411a645093 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -3,6 +3,7 @@ Describe "Measure-Object" -Tags "CI" { BeforeAll { $testObject = 1,3,4 + $testObject2 = 1..100 } It "Should be able to be called without error" { @@ -23,22 +24,19 @@ Describe "Measure-Object" -Tags "CI" { [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } + + It "Should calculate Standard Deviation" { + $actual = ($testObject2 | Measure-Object -StdDeviation) + # We check this way since .StdDeviation returns a double value + [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 + } + It "Should calculate Standard Deviation without -Average" { $actual = ($testObject | Measure-Object -StdDeviation) # We check this way since .StdDeviation returns a double value [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } - It 'Should throw if Standard Deviation requested and -Average:$false' { - try { - $Error.Clear() - $testObject | Measure-Object -StdDeviation -Average:$false -ErrorAction Stop - } - catch { - $Error.Count | should be 1 - } - } - It "Should be able to count using the Property switch" { $expected = $(Get-ChildItem $TestDrive).Length $actual = $(Get-ChildItem $TestDrive | Measure-Object -Property Length).Count From b6491e81ff9be78d4a56b4b1418470a07f58cd77 Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Mon, 12 Mar 2018 14:50:05 -0400 Subject: [PATCH 13/19] Update standard deviation tests Checks to make sure that -StdDeviation works with -Sum and -Average --- .../Measure-Object.Tests.ps1 | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index d411a645093..d06b4957a9d 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -19,7 +19,7 @@ Describe "Measure-Object" -Tags "CI" { } It "Should calculate Standard Deviation" { - $actual = ($testObject | Measure-Object -Average -StdDeviation) + $actual = ($testObject | Measure-Object -StdDeviation) # We check this way since .StdDeviation returns a double value [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } @@ -31,12 +31,28 @@ Describe "Measure-Object" -Tags "CI" { [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 } - It "Should calculate Standard Deviation without -Average" { - $actual = ($testObject | Measure-Object -StdDeviation) + It "Should calculate Standard Deviation with -Sum" { + $actual = ($testObject | Measure-Object -Sum -StdDeviation) + # We check this way since .StdDeviation returns a double value + $actual.Sum | Should Be 8 + [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 + } + + It "Should calculate Standard Deviation with -Average" { + $actual = ($testObject | Measure-Object -Average -StdDeviation) # We check this way since .StdDeviation returns a double value + [Math]::abs($actual.Average - 2.66666666666667) | Should -BeLessThan .00000000000001 [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } + It "Should calculate Standard Deviation with -Sum -Average" { + $actual = ($testObject2 | Measure-Object -Sum -Average -StdDeviation) + # We check this way since .StdDeviation returns a double value + $actual.Sum | Should Be 5050 + $actual.Average | Should Be 50.5 + [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 + } + It "Should be able to count using the Property switch" { $expected = $(Get-ChildItem $TestDrive).Length $actual = $(Get-ChildItem $TestDrive | Measure-Object -Property Length).Count From c40ab47473e076bfec9de89829004fa2ed3f86ec Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Mon, 12 Mar 2018 14:53:20 -0400 Subject: [PATCH 14/19] Update formatting and standard deviation calculation Added periods to end of public comments and removed empty value tags as per iSazonov's request. Calculation of standard deviation is now cleaner. --- .../commands/utility/Measure-Object.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 2d5c2b91bd5..57ac3a53db3 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -73,7 +73,7 @@ public GenericMeasureInfo() public double? Minimum { get; set; } /// - /// The Standard Deviation of property values + /// The Standard Deviation of property values. /// public double? StdDeviation { get; set; } } @@ -135,7 +135,7 @@ public GenericObjectMeasureInfo() public object Minimum { get; set; } /// - /// The Standard Deviation of property values + /// The Standard Deviation of property values. /// public double? StdDeviation { get; set; } } @@ -276,10 +276,9 @@ public MeasureObjectCommand() #endregion Common parameters in both sets /// - /// Set to true if Standard Deviation is to be returned + /// Set to true if Standard Deviation is to be returned. /// - /// - [Parameter(ParameterSetName = GenericParameterSet)] + [Parameter(ParameterSetName = GenericParameterSet)] public SwitchParameter StdDeviation { get @@ -746,9 +745,10 @@ private void AnalyzeNumber(double numValue, Statistics stat) } if (_measureStdDeviation && stat.count > 1) { - double countMinusOne = stat.count - 1; - double avgPrevious = stat.sumPrevious / countMinusOne; - stat.variance *= (countMinusOne - 1)/countMinusOne; + // Based off of iterative method of calculating variance on + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm + double avgPrevious = stat.sumPrevious / (stat.count - 1); + stat.variance *= (stat.count - 2.0) / (stat.count - 1); stat.variance += (numValue - avgPrevious) * (numValue - avgPrevious) / stat.count; } } From 1a9d6897e950d076fa6537449918dc56aead6afa Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Wed, 14 Mar 2018 09:48:44 -0400 Subject: [PATCH 15/19] Mention how values for standard deviation tests were calculated --- .../Measure-Object.Tests.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index d06b4957a9d..b6213178473 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -21,6 +21,8 @@ Describe "Measure-Object" -Tags "CI" { It "Should calculate Standard Deviation" { $actual = ($testObject | Measure-Object -StdDeviation) # We check this way since .StdDeviation returns a double value + # 1.52752523165195 was calculated outside powershell using formula from + # http://mathworld.wolfram.com/StandardDeviation.html [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } @@ -28,6 +30,8 @@ Describe "Measure-Object" -Tags "CI" { It "Should calculate Standard Deviation" { $actual = ($testObject2 | Measure-Object -StdDeviation) # We check this way since .StdDeviation returns a double value + # 29.011491975882 was calculated outside powershell using formula from + # http://mathworld.wolfram.com/StandardDeviation.html [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 } @@ -35,6 +39,8 @@ Describe "Measure-Object" -Tags "CI" { $actual = ($testObject | Measure-Object -Sum -StdDeviation) # We check this way since .StdDeviation returns a double value $actual.Sum | Should Be 8 + # 1.52752523165195 was calculated outside powershell using formula from + # http://mathworld.wolfram.com/StandardDeviation.html [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } @@ -42,6 +48,8 @@ Describe "Measure-Object" -Tags "CI" { $actual = ($testObject | Measure-Object -Average -StdDeviation) # We check this way since .StdDeviation returns a double value [Math]::abs($actual.Average - 2.66666666666667) | Should -BeLessThan .00000000000001 + # 1.52752523165195 was calculated outside powershell using formula from + # http://mathworld.wolfram.com/StandardDeviation.html [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } @@ -50,6 +58,8 @@ Describe "Measure-Object" -Tags "CI" { # We check this way since .StdDeviation returns a double value $actual.Sum | Should Be 5050 $actual.Average | Should Be 50.5 + # 29.011491975882 was calculated outside powershell using formula from + # http://mathworld.wolfram.com/StandardDeviation.html [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 } From 1820a410507989e482f03649b5fd65b273f333d1 Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Wed, 14 Mar 2018 09:49:10 -0400 Subject: [PATCH 16/19] Remove standard deviation from GenericMeasureInfo --- .../commands/utility/Measure-Object.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 57ac3a53db3..89917c90f36 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -883,7 +883,6 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene gmi.Count = stat.count; gmi.Sum = sum; gmi.Average = average; - gmi.StdDeviation = stdDeviation; if (null != max) { gmi.Maximum = (double)max; From 23bd24b1fca4f75368db203f7b90077c6ef5e64f Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Wed, 14 Mar 2018 13:32:32 -0400 Subject: [PATCH 17/19] Unindent line --- .../commands/utility/Measure-Object.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 89917c90f36..912606c78e0 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -278,7 +278,7 @@ public MeasureObjectCommand() /// /// Set to true if Standard Deviation is to be returned. /// - [Parameter(ParameterSetName = GenericParameterSet)] + [Parameter(ParameterSetName = GenericParameterSet)] public SwitchParameter StdDeviation { get From df8ac60092a752f60c516e1d20a3fd4e7c4b806e Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Wed, 14 Mar 2018 14:14:55 -0400 Subject: [PATCH 18/19] revert commit 1820a41 Standard Devation wasn't being displayed after commit 1820a410507989e482f03649b5fd65b273f333d1 --- .../commands/utility/Measure-Object.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 912606c78e0..63501feb9b2 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -883,6 +883,7 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene gmi.Count = stat.count; gmi.Sum = sum; gmi.Average = average; + gmi.StdDeviation = stdDeviation; if (null != max) { gmi.Maximum = (double)max; From ea6fe0f6e638cbe0a7f825745dcfdf5abac88928 Mon Sep 17 00:00:00 2001 From: Amaan Marfatia Date: Wed, 14 Mar 2018 21:32:56 -0400 Subject: [PATCH 19/19] Change StdDeviation to StandardDeviation to be consistent --- .../commands/utility/Measure-Object.cs | 30 +++++++++---------- .../Measure-Object.Tests.ps1 | 30 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs index 63501feb9b2..62bebc4d212 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Measure-Object.cs @@ -34,7 +34,7 @@ public sealed class GenericMeasureInfo : MeasureInfo /// public GenericMeasureInfo() { - Average = Sum = Maximum = Minimum = StdDeviation = null; + Average = Sum = Maximum = Minimum = StandardDeviation = null; } /// @@ -75,7 +75,7 @@ public GenericMeasureInfo() /// /// The Standard Deviation of property values. /// - public double? StdDeviation { get; set; } + public double? StandardDeviation { get; set; } } /// @@ -95,7 +95,7 @@ public sealed class GenericObjectMeasureInfo : MeasureInfo /// public GenericObjectMeasureInfo() { - Average = Sum = StdDeviation = null; + Average = Sum = StandardDeviation = null; Maximum = Minimum = null; } @@ -137,7 +137,7 @@ public GenericObjectMeasureInfo() /// /// The Standard Deviation of property values. /// - public double? StdDeviation { get; set; } + public double? StandardDeviation { get; set; } } /// @@ -279,19 +279,19 @@ public MeasureObjectCommand() /// Set to true if Standard Deviation is to be returned. /// [Parameter(ParameterSetName = GenericParameterSet)] - public SwitchParameter StdDeviation + public SwitchParameter StandardDeviation { get { - return _measureStdDeviation; + return _measureStandardDeviation; } set { - _measureStdDeviation = value; + _measureStandardDeviation = value; } } - private bool _measureStdDeviation; + private bool _measureStandardDeviation; /// /// Set to true is Sum is to be returned @@ -555,7 +555,7 @@ private void AnalyzeValue(string propertyName, object objValue) AnalyzeString(strValue, stat); } - if (_measureAverage || _measureSum || _measureStdDeviation) + if (_measureAverage || _measureSum || _measureStandardDeviation) { double numValue = 0.0; if (!LanguagePrimitives.TryConvertTo(objValue, out numValue)) @@ -738,12 +738,12 @@ private void AnalyzeString(string strValue, Statistics stat) /// private void AnalyzeNumber(double numValue, Statistics stat) { - if (_measureSum || _measureAverage || _measureStdDeviation) + if (_measureSum || _measureAverage || _measureStandardDeviation) { stat.sumPrevious = stat.sum; stat.sum += numValue; } - if (_measureStdDeviation && stat.count > 1) + if (_measureStandardDeviation && stat.count > 1) { // Based off of iterative method of calculating variance on // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm @@ -831,7 +831,7 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene { double? sum = null; double? average = null; - double? stdDeviation = null; + double? StandardDeviation = null; object max = null; object min = null; @@ -843,9 +843,9 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene if (_measureAverage && stat.count > 0) average = stat.sum / stat.count; - if (_measureStdDeviation) + if (_measureStandardDeviation) { - stdDeviation = Math.Sqrt(stat.variance); + StandardDeviation = Math.Sqrt(stat.variance); } } @@ -883,7 +883,7 @@ private MeasureInfo CreateGenericMeasureInfo(Statistics stat, bool shouldUseGene gmi.Count = stat.count; gmi.Sum = sum; gmi.Average = average; - gmi.StdDeviation = stdDeviation; + gmi.StandardDeviation = StandardDeviation; if (null != max) { gmi.Maximum = (double)max; diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 index b6213178473..0a5b4af2a93 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Measure-Object.Tests.ps1 @@ -19,48 +19,48 @@ Describe "Measure-Object" -Tags "CI" { } It "Should calculate Standard Deviation" { - $actual = ($testObject | Measure-Object -StdDeviation) - # We check this way since .StdDeviation returns a double value + $actual = ($testObject | Measure-Object -StandardDeviation) + # We check this way since .StandardDeviation returns a double value # 1.52752523165195 was calculated outside powershell using formula from # http://mathworld.wolfram.com/StandardDeviation.html - [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 + [Math]::abs($actual.StandardDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } It "Should calculate Standard Deviation" { - $actual = ($testObject2 | Measure-Object -StdDeviation) - # We check this way since .StdDeviation returns a double value + $actual = ($testObject2 | Measure-Object -StandardDeviation) + # We check this way since .StandardDeviation returns a double value # 29.011491975882 was calculated outside powershell using formula from # http://mathworld.wolfram.com/StandardDeviation.html - [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 + [Math]::abs($actual.StandardDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 } It "Should calculate Standard Deviation with -Sum" { - $actual = ($testObject | Measure-Object -Sum -StdDeviation) - # We check this way since .StdDeviation returns a double value + $actual = ($testObject | Measure-Object -Sum -StandardDeviation) + # We check this way since .StandardDeviation returns a double value $actual.Sum | Should Be 8 # 1.52752523165195 was calculated outside powershell using formula from # http://mathworld.wolfram.com/StandardDeviation.html - [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 + [Math]::abs($actual.StandardDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } It "Should calculate Standard Deviation with -Average" { - $actual = ($testObject | Measure-Object -Average -StdDeviation) - # We check this way since .StdDeviation returns a double value + $actual = ($testObject | Measure-Object -Average -StandardDeviation) + # We check this way since .StandardDeviation returns a double value [Math]::abs($actual.Average - 2.66666666666667) | Should -BeLessThan .00000000000001 # 1.52752523165195 was calculated outside powershell using formula from # http://mathworld.wolfram.com/StandardDeviation.html - [Math]::abs($actual.StdDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 + [Math]::abs($actual.StandardDeviation - 1.52752523165195) | Should -BeLessThan .00000000000001 } It "Should calculate Standard Deviation with -Sum -Average" { - $actual = ($testObject2 | Measure-Object -Sum -Average -StdDeviation) - # We check this way since .StdDeviation returns a double value + $actual = ($testObject2 | Measure-Object -Sum -Average -StandardDeviation) + # We check this way since .StandardDeviation returns a double value $actual.Sum | Should Be 5050 $actual.Average | Should Be 50.5 # 29.011491975882 was calculated outside powershell using formula from # http://mathworld.wolfram.com/StandardDeviation.html - [Math]::abs($actual.StdDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 + [Math]::abs($actual.StandardDeviation - 29.011491975882) | Should -BeLessThan .0000000000001 } It "Should be able to count using the Property switch" {