diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeRestMethodCommand.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeRestMethodCommand.CoreClr.cs index 8be7c98342a..23c9c0a44f7 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeRestMethodCommand.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeRestMethodCommand.CoreClr.cs @@ -45,13 +45,28 @@ internal override void ProcessResponse(HttpResponseMessage response) { // determine the response type RestReturnType returnType = CheckReturnType(response); - // get the response encoding - Encoding encoding = ContentHelper.GetEncoding(response); + + // Try to get the response encoding from the ContentType header. + Encoding encoding = null; + string charSet = response.Content.Headers.ContentType?.CharSet; + if (!string.IsNullOrEmpty(charSet)) + { + // NOTE: Don't use ContentHelper.GetEncoding; it returns a + // default which bypasses checking for a meta charset value. + StreamHelper.TryGetEncoding(charSet, out encoding); + } object obj = null; Exception ex = null; string str = StreamHelper.DecodeStream(responseStream, ref encoding); + // NOTE: Tests use this verbose output to verify the encoding. + WriteVerbose(string.Format + ( + System.Globalization.CultureInfo.InvariantCulture, + "Content encoding: {0}", + string.IsNullOrEmpty(encoding.HeaderName) ? encoding.EncodingName : encoding.HeaderName) + ); bool convertSuccess = false; // On CoreCLR, we need to explicitly load Json.NET diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index cc4f0cacfcd..f4486c8771c 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -450,7 +450,7 @@ internal static string DecodeStream(Stream stream, string characterSet, out Enco return DecodeStream(stream, ref encoding); } - static bool TryGetEncoding(string characterSet, out Encoding encoding) + internal static bool TryGetEncoding(string characterSet, out Encoding encoding) { bool result = false; try diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index fdcaba52243..7074c70cc75 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -296,6 +296,69 @@ function ExecuteWebRequest return $result } +[string] $verboseEncodingPrefix = 'Content encoding: ' +# This function calls Invoke-WebRequest with the given uri and +# parses the verbose output to determine the encoding used for the content. +function ExecuteRestMethod +{ + param + ( + [Parameter(Mandatory)] + [string] + $Uri, + + [switch] $UseBasicParsing + ) + $result = @{Output = $null; Error = $null; Encoding = $null; Content = $null} + $verbosePreferenceSave = $VerbosePreference + $VerbosePreference = 'Continue' + try + { + + $verboseFile = Join-Path $TestDrive -ChildPath ExecuteRestMethod.verbose.txt + $result.Output = Invoke-RestMethod -Uri $Uri -TimeoutSec 5 -UseBasicParsing:$UseBasicParsing.IsPresent -Verbose 4>$verboseFile + $result.Content = $result.Output + + if (Test-Path -Path $verboseFile) + { + $result.Verbose = Get-Content -Path $verboseFile + foreach ($item in $result.Verbose) + { + $line = $item.Trim() + if ($line.StartsWith($verboseEncodingPrefix)) + { + $encodingName = $item.SubString($verboseEncodingPrefix.Length).Trim() + $result.Encoding = [System.Text.Encoding]::GetEncoding($encodingName) + break + } + } + if ($result.Encoding -eq $null) + { + throw "Encoding not found in verbose output. Lines: $($result.Verbose.Count) Content:$($result.Verbose)" + } + } + + if ($result.Verbose -eq $null) + { + throw "No verbose output was found" + } + } + catch + { + $result.Error = $_ | select-object * | Out-String + } + finally + { + $VerbosePreference = $verbosePreferenceSave + if (Test-Path -Path $verboseFile) + { + Remove-Item -Path $verboseFile -ErrorAction SilentlyContinue + } + } + + return $result +} + <# Defines the list of redirect codes to test as well as the expected Method when the redirection is handled. @@ -1678,6 +1741,224 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { #endregion Client Certificate Authentication + #region charset encoding tests + + Context "Invoke-RestMethod Encoding tests with BasicHtmlWebResponseObject response" { + It "Verifies Invoke-RestMethod detects charset meta value when the ContentType header does not define it." { + $output = '' + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-WebRequest detects charset meta value when newlines are encountered in the element." { + $output = @' + + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod detects charset meta value when the attribute value is unquoted." { + $output = '' + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod detects http-equiv charset meta value when the ContentType header does not define it." { + $output = @' + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod detects http-equiv charset meta value newlines are encountered in the element." { + $output = @' + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod ignores meta charset value when Content-Type header defines it." { + $output = '' + # NOTE: meta charset should be ignored + $expectedEncoding = [System.Text.Encoding]::UTF8 + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html; charset=utf-8&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod honors non-utf8 charsets in the Content-Type header" { + $output = '' + # NOTE: meta charset should be ignored + $expectedEncoding = [System.Text.Encoding]::GetEncoding('utf-16') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html; charset=utf-16&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod defaults to iso-8859-1 when an unsupported/invalid charset is declared" { + $output = '' + $expectedEncoding = [System.Text.Encoding]::GetEncoding('iso-8859-1') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod defaults to iso-8859-1 when an unsupported/invalid charset is declared using http-equiv" { + $output = @' + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('iso-8859-1') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html&output=$output" -UseBasicParsing + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + } + + Context "Invoke-RestMethod Encoding tests with HtmlWebResponseObject response" { + # these tests are dependent on https://github.com/PowerShell/PowerShell/issues/2867 + # Currently, all paths return BasicHtmlWebResponseObject + It "Verifies Invoke-RestMethod detects charset meta value when the ContentType header does not define it." -Pending { + $output = '' + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod detects charset meta value when newlines are encountered in the element." -Pending { + $output = @' + + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod ignores meta charset value when Content-Type header defines it." -Pending { + $output = '' + # NOTE: meta charset should be ignored + $expectedEncoding = [System.Text.Encoding]::UTF8 + # Update to test for HtmlWebResponseObject when mshtl dependency has been resolved. + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html; charset=utf-8&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod detects http-equiv charset meta value when the ContentType header does not define it." -Pending { + $output = @' + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod detects http-equiv charset meta value newlines are encountered in the element." -Pending { + $output = @' + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('Unicode') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod honors non-utf8 charsets in the Content-Type header" -Pending { + $output = '' + # NOTE: meta charset should be ignored + $expectedEncoding = [System.Text.Encoding]::GetEncoding('utf-16') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html; charset=utf-16&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod defaults to iso-8859-1 when an unsupported/invalid charset is declared" -Pending { + $output = '' + $expectedEncoding = [System.Text.Encoding]::GetEncoding('iso-8859-1') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + + It "Verifies Invoke-RestMethod defaults to iso-8859-1 when an unsupported/invalid charset is declared using http-equiv" -Pending { + $output = @' + + + + +'@ + $expectedEncoding = [System.Text.Encoding]::GetEncoding('iso-8859-1') + $response = ExecuteRestMethod -Uri "http://localhost:8081/PowerShell?test=response&contenttype=text/html&output=$output" + + $response.Error | Should BeNullOrEmpty + $response.Encoding.EncodingName | Should Be $expectedEncoding.EncodingName + } + } + + #endregion charset encoding tests + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy