diff --git a/Docs/How to document Power Query functions.md b/Docs/How to document Power Query functions.md index a3b3c64..39c2c31 100644 --- a/Docs/How to document Power Query functions.md +++ b/Docs/How to document Power Query functions.md @@ -1,5 +1,11 @@ # How to document Power Query functions +first: + +- +- + + @@ -66,3 +72,16 @@ docs: - [Handling Documentation](https://docs.microsoft.com/en-us/power-query/handlingdocumentation) - [Value Functions and Metadata](https://docs.microsoft.com/en-us/powerquery-m/value-functions#__toc360789751) - [Function-Value Functions](https://docs.microsoft.com/en-us/powerquery-m/function-values) + + +``` +doc tips: + +documentation on the types uses for documentation + https://docs.microsoft.com/en-us/power-query/handlingdocumentation + +from: + https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata +> A note about Documentation.Name: This value should probably match the name of the identifier you assign your function when you define it. For example, if you call your function SomeFunction, then when you set Documentation.Name, you should probably set its value to “SomeFunction”, as well. + +``` \ No newline at end of file diff --git a/ErrorRecord.Grammar.md b/ErrorRecord.Grammar.md new file mode 100644 index 0000000..a489cb8 --- /dev/null +++ b/ErrorRecord.Grammar.md @@ -0,0 +1,149 @@ +- [Inline throw](#inline-throw) +- [Function: `Error.Record()` signature](#function-errorrecord-signature) +- [See Also](#see-also) +- [Examples from the Docs](#examples-from-the-docs) +- [Extra Requirements Specific to Try Catch](#extra-requirements-specific-to-try-catch) +- [Try Expression Types](#try-expression-types) + - [Try catch (e) => ](#try-expression-catch-e--expression) + - [Try catch () => ](#try-expression-catch---expression) + - [Try otherwise ](#try-expression-otherwise-expression) + - [Try ](#try-expression) +- [yaml summary](#yaml-summary) + +## Inline throw + +`error` is the name of a **language keyword** that throws an exception. There's a built in function named `Error.Record` to construct the argument `record` for you +```js +error [ + Reason = "Business Rule Violated", + Message = "Item codes must start with a letter", + Detail = "Non-conforming Item Code: 456" +] +``` + +To get a GUID to easily filter query tracing, you can `Table.AddColumn()` a guid for traces using trace specific rows to specific [Diagnostics.ActivityId\(\) as text](https://docs.microsoft.com/en-us/powerquery-m/diagnostics-activityid) +```js += Diagnostics.ActivityId() +``` + + +## Function: `Error.Record()` signature + +Creates the record to `throw` + +```js +Error.Record( + reason as text, + optional message as nullable text, + optional detail as any, + optional parameters as nullable list) as record +``` +## See Also + + - what's new: [New Power Query M Language Keyword: catch](https://powerquery.microsoft.com/en-us/blog/new-power-query-m-language-keyword-catch/) + - pq docs [Error.Record\(\)](https://docs.microsoft.com/en-us/powerquery-m/error-record) + - [Diagnostics.Trace\(\)](https://docs.microsoft.com/en-us/powerquery-m/diagnostics-trace) + - [Diagnostics.ActivityId\(\) as text](https://docs.microsoft.com/en-us/powerquery-m/diagnostics-activityid) + - what's new: [bengribaudo.com: Structured Error Messages](https://bengribaudo.com/blog/2022/05/24/6753/new-m-feature-structured-error-messages) + - [bengribaudo.com: power-query-m-primer-part-15-error-handling](https://bengribaudo.com/blog/2020/01/15/4883/power-query-m-primer-part-15-error-handling) + - Power Query Formal Language Specs.pdf + + +## Examples from the Docs + +The `Ellipsis` expression is **sugar** for throwing this expression +```js +x =... +``` +is equivalent to +```js +x = error Error.Record("Expression.Error", "Not Implemented") +``` +This expression +```js +x = error Error.Record( + "FileNotFound", "File my.txt not found", "my.txt") +``` +is equivalent to +```js +x = error [ + Reason = "FileNotFound", + Message = "File my.txt not found", + Detail = "my.txt" +] +``` +more examples + +## Extra Requirements Specific to Try Catch + +- The catch **function must be defined inline**, so a reference to a function cannot be used. +- `each` cannot be used to define the `catch-function` +- the function defintion **cannot include any type constraints** + +## Try Expression Types + +### Try catch (e) => + +```js += Table.AddColumn( +Source, + "Clean Standard Rate", + each + try [Standard Rate] + catch (e) => + if e[Message] = "Invalid cell value '#REF!'." then [Special Rate] * 2 + else if e[Message] = "Invalid cell value '#DIV/0!'." then [Special Rate] / 3 + else 0 +) +``` + + +### Try catch () => + +Functionally equivalent to `try otherwise ` + +```js +let + x = try 1 / 0 + catch () => "null" +in + x +``` + +### Try otherwise +```js +let + x = try 1 / 0 + otherwise "null" +in + x +``` +### Try + +```js +let + x = try "A" +in + if x[HasError] then x[Error] else x[Value] +``` + +## yaml summary +```yml +returns: + type: record +reason: + type: text + default: "Expression.Error" + notes: . | + the default value isn't mandatory, may be implementation details and change +message: + type: optional nullable text + +detail: + type: optional detail as any + +parameters: + type: optional nullable list +``` + + diff --git a/WIP/Table.MergeDimensionTable and To.DimensionTable.pq b/WIP/Table.MergeDimensionTable and To.DimensionTable.pq new file mode 100644 index 0000000..6b97333 --- /dev/null +++ b/WIP/Table.MergeDimensionTable and To.DimensionTable.pq @@ -0,0 +1,42 @@ +let + + x1 = + let + Table.MergeDimensionTable = (source as table, dim as table, optional options as nullable record + ) => + let + t_join = Table.NestedJoin( source, {"Phone"}, dim, {"Phone"}, "Result", JoinKind.LeftOuter ), + t_expand = Table.ExpandTableColumn( t_join, "Result", {"Id"}, {"Phone Id"}) + in t_expand + + in + Table.MergeDimensionTable, + + + x2 = let + Source = Table.FromRecords({ + [ Phone = "1234", User = "Bob" ], + [ Phone = "1234", User = "Jen" ], + [ Phone = "2234", User = "Fred" ], + [ Phone = "3234", User = "Penny"], + [ Phone = "4234", User = "Gary" ] + }, type table[Phone = text, User = text], MissingField.Error), + + Table.ToDimensionTable = (source as table, column as text) => + let t0 = Table.Distinct( Table.SelectColumns(source, column, MissingField.Error) ), + col_id = Table.AddIndexColumn(t0, "Id", 0) + in col_id, + + dimPhone = Table.ToDimensionTable( Source, "Phone" ), + dimUser = Table.ToDimensionTable( Source, "User" ), + // t0 = Table.Distinct( Table.SelectColumns( Source, {"Phone"}) ), + // dimTable_Phone = Table.AddIndexColumn(t0, "Index", 0, 1, Int64.Type) + Summary = [ + Source = Source, + dimPhone = dimPhone, + dimUser = dimUser + ] + in + Summary +in + x2 \ No newline at end of file diff --git a/WIP/merge/List.SummaryRecurse-rewrite2022.pq b/WIP/merge/List.SummaryRecurse-rewrite2022.pq new file mode 100644 index 0000000..5597129 --- /dev/null +++ b/WIP/merge/List.SummaryRecurse-rewrite2022.pq @@ -0,0 +1,65 @@ +let + + // Text.FromBinary( Json.FromValue( item , TextEncoding.Utf8 ) , TextEncoding.Utf8 ), + // Generates type name by expecting Text.From to throw, + // Then using the serialized name in the exception + // if that fails, final fallback string is "<␀>" null symbol + // write warning: does not work on type table, ewtc. so maybe I have the wrong param + Value.TypeNameAsText = (source as any) => + let + typeInfo = + try + Text.From(Value.Type(source)) + catch (e) => + e[Message.Parameters]?{0}? + ?? "<#(00002400)>" + in + typeInfo, + + // recursively summarize a list as a string + List.Summary = (source as list, optional options as nullable record) as text => + let + options = Record.Combine({ defaults, options }), + defaults = [Culture = "en-us", Encoding = TextEncoding.Utf8, Prefix = "{ ", Suffix = " }", Delimiter = ", " ], + culture = options[Culture], + + items_asText = List.Transform( + source, + (item) => try Text.From(item, culture) catch (e) => + if Type.Is(Value.Type(item), Record.Type) then // placeholder + Text.FromBinary( Json.FromValue( item , options[Encoding] ) , options[Encoding] ) + else if Type.Is( Value.Type(item), List.Type ) then + let + render = + try @List.Summary( item , culture ) + catch (e2) => + error Error.Record( + "NestedInvokeException", "Depth X and X+1 threw:" + [ Exception = e, InnerException = e2 ] + ) + in + options[Prefix] & render & options[Suffix] + + else if Value.Type(item, Type.Type) then + Value.TypeNameAsText( item ) + else + "<#(00002400)>" ), + combinedText = Text.Combine( items_asText, options[Delimiter]) + in combinedText, + + + tests = [ + t0 = List.Summary( { 0, 3, { 9, 8 } }), // output: 0, 3, 9, 8, + t1 = List.Summary( {DateTime.LocalNow(), 0.345, {0..4}, [a = "b"]} ), + t3 = List.Summary( { + {DateTime.LocalNow(), 0.345, {0..4}, [a = "b"]}, + {DateTime.LocalNow(), 0.345, {99..80}, [F = "b"]} + }) + ] + /* + outputs: + + t3 = { 6/11/2022 11:58:28 AM, 0.345, { 0, 1, 2, 3, 4 }, [...] }, { 6/11/2022 11:58:28 AM, 0.345, { }, [...] } + */ +in + tests \ No newline at end of file diff --git a/WIP/merge/sorta-lib-with-helpers.pq b/WIP/merge/sorta-lib-with-helpers.pq new file mode 100644 index 0000000..5474d3a --- /dev/null +++ b/WIP/merge/sorta-lib-with-helpers.pq @@ -0,0 +1,88 @@ +// lib +let + LibType = let + /* experimenting with a different style of indentation +*/ + Uni = [ + Delim = { "#(000021e2)", "#(0000205e)", "#(00002510)", "#(000021fd)" }, + NullSymbol = "#(00002400)" + ] meta [Text.Serialized = true], + Now = () + as datetime => DateTime.LocalNow(), + + + + Ms = (milliseconds as number) + as duration => + #duration(0, 0, 0, 1) / 1000 * milliseconds, + + // alpha table, to time table add column, to test buffers performance hit + Source = Table.FromList( + { "a".."z" } & {"A".."Z"}, + null, type table[Char = text], + null, ExtraValues.Error ), + + Quote = (source as any) + as text => + "'" & Text.From(source) & "'", + + + // Generates type name by expecting Text.From to throw, + // Then using the serialized name in the exception + // if that fails, final fallback string is "<␀>" null symbol + Value.TypeNameAsText = (source as any) => + let + typeInfo = + try + Text.From(Value.Type(source)) + catch (e) => + e[Message.Parameters]?{0}? + ?? "<#(00002400)>" + in + typeInfo, + + NowMsIter3 = (optional options as nullable record) as text => + let + + options = Record.Combine({ defaults, (options ?? []) }), + defaults = [ + Template0 = "#[ms] ms (of #[sec] #[delim] #[ms] )", + Template = "#[sec] #[delim] #[ms] ms", + Culture = Culture.Current, + Comparer.Ordinal = Comparer.OrdinalIgnoreCase + ], + now = Now(), + fin = Text.Format( + options[Template], + [ + ms = DateTime.ToText( now, [ Culture = options[Culture], Format = "fff"]), + sec = DateTime.ToText( now, [ Culture = options[Culture], Format = "m:s"]), + delim = Uni[Delim]{0} + ], + options[Culture] + ) + in fin , + + exports = [ + Ms = Ms, + Now = Now, + Uni = Uni, + Value.TypeNameAsText = Value.TypeNameAsText, + NowMs.ToText = NowMsIter3, + NowAsText = NowMs.ToText, + Quote = Quote, + SampleTable = Source + ] +in + if false then + { + exports[NowMs.ToText](), + exports[NowMs.ToText]( [ Template = "#[ms] ms (of #[sec] #[delim] #[ms] )" ] ), + exports[NowMs.ToText]( [ Template = "#[sec] #[ms] " ] ) + } + else + exports +in + LibType meta [ + Docs = ... + ] \ No newline at end of file diff --git a/WIP/others/iso-week-number.2021.pq b/WIP/others/iso-week-number.2021.pq new file mode 100644 index 0000000..8ac1afa --- /dev/null +++ b/WIP/others/iso-week-number.2021.pq @@ -0,0 +1,27 @@ +//datacornering.com + +let + Source = List.Dates(DateTime.Date(DateTime.LocalNow()), 365, #duration(1, 0, 0, 0)), + #"Converted to Table" = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error), + #"Renamed Columns" = Table.RenameColumns(#"Converted to Table",{{"Column1", "Date"}}), + #"Changed Type" = Table.TransformColumnTypes(#"Renamed Columns",{{"Date", type date}}), + #"Weekday Number" = Table.AddColumn(#"Changed Type", "Weekday Number", each Date.DayOfWeek([Date], Day.Monday)+1), + + #"ISO Week Number" = Table.AddColumn(#"Weekday Number", "ISO Week Number", each if +Number.RoundDown((Date.DayOfYear([Date])-(Date.DayOfWeek([Date], Day.Monday)+1)+10)/7)=0 + +then +Number.RoundDown((Date.DayOfYear(#date(Date.Year([Date])-1,12,31))-(Date.DayOfWeek(#date(Date.Year([Date])-1,12,31), Day.Monday)+1)+10)/7) + +else if +(Number.RoundDown((Date.DayOfYear([Date])-(Date.DayOfWeek([Date], Day.Monday)+1)+10)/7)=53 +and (Date.DayOfWeek(#date(Date.Year([Date]),12,31), Day.Monday)+1<4)) + +then +1 + +else +Number.RoundDown((Date.DayOfYear([Date])-(Date.DayOfWeek([Date], Day.Monday)+1)+10)/7)) + +in + #"ISO Week Number" \ No newline at end of file diff --git a/WIP/others/iso-week-number.fixed.pq b/WIP/others/iso-week-number.fixed.pq new file mode 100644 index 0000000..dbf7972 --- /dev/null +++ b/WIP/others/iso-week-number.fixed.pq @@ -0,0 +1,30 @@ +// originally: +let + todo = "Cleanup" , + Source = List.Dates(DateTime.Date(DateTime.LocalNow()), 365, #duration(1, 0, 0, 0)), + #"Converted to Table" = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error), + #"Renamed Columns" = Table.RenameColumns(#"Converted to Table", {{"Column1", "Date"}}), + #"Changed Type" = Table.TransformColumnTypes(#"Renamed Columns", {{"Date", type date}}), + #"Weekday Number" = Table.AddColumn(#"Changed Type", "Weekday Number", each Date.DayOfWeek([Date], Day.Monday) + 1), + #"ISO Week Number" = Table.AddColumn( + #"Weekday Number", + "ISO Week Number", + each + if Number.RoundDown((Date.DayOfYear([Date]) - (Date.DayOfWeek([Date], Day.Monday) + 1) + 10) / 7) = 0 then + Number.RoundDown( + ( + Date.DayOfYear(#date(Date.Year([Date]) - 1, 12, 31)) - ( + Date.DayOfWeek(#date(Date.Year([Date]) - 1, 12, 31), Day.Monday) + 1 + ) + 10 + ) / 7 + ) + else if ( + Number.RoundDown((Date.DayOfYear([Date]) - (Date.DayOfWeek([Date], Day.Monday) + 1) + 10) / 7) = 53 + and (Date.DayOfWeek(#date(Date.Year([Date]), 12, 31), Day.Monday) + 1 < 4) + ) then + 1 + else + Number.RoundDown((Date.DayOfYear([Date]) - (Date.DayOfWeek([Date], Day.Monday) + 1) + 10) / 7) + ) +in + todo diff --git a/WIP/test--assert-types.pq b/WIP/test--assert-types.pq new file mode 100644 index 0000000..21db146 --- /dev/null +++ b/WIP/test--assert-types.pq @@ -0,0 +1,15 @@ +FinalResult = 1, + + Assert.TypeIs = (source as any, expected as type) as any => + <# + + primitive type is compatible? + Should.BeOfType( #date(2020, 10, 10), type date ) //# true + Should.BeOfType( #date(2020, 10, 10), type datetime ) // # + + primitive type is exactly of type + #> + + // todo: + ..., + diff --git a/source-dax/Format-Percent-Without-Math-Or-P.dax b/source-dax/Format-Percent-Without-Math-Or-P.dax new file mode 100644 index 0000000..13a6e47 --- /dev/null +++ b/source-dax/Format-Percent-Without-Math-Or-P.dax @@ -0,0 +1,23 @@ +[ Percent Message ] = + /* + Render percentage, because the normal "p" format string + is missing. Instead you can use "%", + which sort of is the culture-invariant version of "p" + + (no multiplication needed) to render percentagel, because the normal "p" format string is missing + explained here: https://docs.microsoft.com/en-us/dax/format-function-dax + + Example + Input Output + 0.0 0.0% + 0.5 50.0% + 1.0 100.0% + + related docs about culture invariants: + https://docs.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo?view=net-6.0#invariant-neutral-and-specific-cultures + https://docs.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.invariantculture?view=net-6.0 + */ + var something = SELECTEDVALUE( Range[Value] ) + var render = FORMAT( something, "##0.0%" ) +return + "Hit Rate: " & render diff --git a/source-pwsh/Nin.PqLib.psd1 b/source-pwsh/Nin.PqLib.psd1 new file mode 100644 index 0000000..a4f431f --- /dev/null +++ b/source-pwsh/Nin.PqLib.psd1 @@ -0,0 +1,139 @@ +# +# Module manifest for module 'Nin.PqLib' +# +# Generated by: Jake +# +# Generated on: 5/29/2022 +# + +@{ + + # Script module or binary module file associated with this manifest. + RootModule = 'Nin.PqLib.psm1' + + # Version number of this module. + ModuleVersion = '0.1.0' + + # Supported PSEditions + # CompatiblePSEditions = @() + + # ID used to uniquely identify this module + GUID = '2b89d25f-edca-46c2-aa26-3b3e77698066' + + # Author of this module + Author = 'Jake Bolton ( ninmonkeys@gmail.com )' + + # Company or vendor of this module + CompanyName = 'Jake Bolton' + + # Copyright statement for this module + Copyright = 'Jake Bolton 2021-2022' + + # Description of the functionality provided by this module + Description = 'Build Process to Merge Library' + + # Minimum version of the PowerShell engine required by this module + # PowerShellVersion = '' 7? + + # Name of the PowerShell host required by this module + # PowerShellHostName = '' + + # Minimum version of the PowerShell host required by this module + # PowerShellHostVersion = '' + + # Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. + # DotNetFrameworkVersion = '' + + # Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. + # ClrVersion = '' + + # Processor architecture (None, X86, Amd64) required by this module + # ProcessorArchitecture = '' + + # Modules that must be imported into the global environment prior to importing this module + RequiredModules = @( + 'Ninmonkey.Console' + # 'Dev.Nin' # ? + # 'Pansies' + # 'ClassExplorer' + # 'Utility' # hard dependency for now + # 'functional' + ) + + # Assemblies that must be loaded prior to importing this module + # RequiredAssemblies = @() + + # Script files (.ps1) that are run in the caller's environment prior to importing this module. + # ScriptsToProcess = @() + + # Type files (.ps1xml) to be loaded when importing this module + TypesToProcess = 'Nin.PqLib.psd1' + + # Format files (.ps1xml) to be loaded when importing this module + # FormatsToProcess = @() + + # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess + # NestedModules = @() + + # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. + FunctionsToExport = @() + + # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. + CmdletsToExport = @() + + # Variables to export from this module + VariablesToExport = @() + + # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. + AliasesToExport = @() + + # DSC resources to export from this module + # DscResourcesToExport = @() + + # List of all modules packaged with this module + # ModuleList = @() + + # List of all files packaged with this module + # FileList = @() + + # Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. + PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + Tags = @('PowerQuery', 'PowerBI', 'Generation') + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + ProjectUri = 'https://github.com/ninmonkey/Ninmonkey.PowerQueryLib' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + # Prerelease string of this module + # Prerelease = '' + + # Flag to indicate whether the module requires explicit user acceptance for install/update/save + # RequireLicenseAcceptance = $false + + # External dependent modules of this module + # ExternalModuleDependencies = @() + + } # End of PSData hashtable + + } # End of PrivateData hashtable + + # HelpInfo URI of this module + HelpInfoURI = 'https://github.com/ninmonkey/Ninmonkey.PowerQueryLib' + + # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. + # DefaultCommandPrefix = '' + +} + diff --git a/source-pwsh/Nin.PqLib.psm1 b/source-pwsh/Nin.PqLib.psm1 new file mode 100644 index 0000000..82de090 --- /dev/null +++ b/source-pwsh/Nin.PqLib.psm1 @@ -0,0 +1,49 @@ +using namespace System.Collections.Generic +#Requires -Version 7 + + + +function Format-AsPQListLiteral { + [cmdletBinding()] + [outputtype('String')] + param( + # future: calculated property as a joiner + [Parameter(Mandatory, ValueFromPipeline)] + [string[]]$InputText, + + [alias('Name')] + [string]$ListName + ) + begin { + $Items = [list[string]]::new() + } + process { + $items.AddRange( $InputText ) + } + + end { + $js_PQLiteralSep = @{ + Separator = ', ' + DoubleQuote = $true + Property = { $_ } + } + + $js_PQListBody = @{ + OutputPrefix = $ListName ? "`$${ListName} = { " : '{ ' + OutputSuffix = ' } ' + Property = { $_ } + } + 0..4 | Format-AsPQListLiteral + + $Items + | Join-String @js_PQLiteralSep + | Join-String @js_PQListBody + } +} + +Export-ModuleMember -Function Format-AsPQListLiteral +<# --- should be seeparate -- #> +# Get-Culture -ListAvailable +# | s -First 10 +# | Join-String -Separator ', ' -DoubleQuote { $_.Name } +# | Join-String -op 'names = { ' -os ' } ' { $_ } \ No newline at end of file diff --git a/source-pwsh/src/Text to PowerQueryLiterals.ps1 b/source-pwsh/src/Text to PowerQueryLiterals.ps1 new file mode 100644 index 0000000..079889f --- /dev/null +++ b/source-pwsh/src/Text to PowerQueryLiterals.ps1 @@ -0,0 +1,53 @@ +class RuneToPqLiteral { + [string]$Source + [int]$Codepoint + [string]$Hex + + hidden [string]$displayStringCodepoint + + RuneToPqLiteral ( [string]$Rune ) { + $first, $rest = $Rune.EnumerateRunes() + if ($rest.count -gt 0) { throw 'nyi IEnumerable list of runes' } + + $This.Source = $Rune + $This.First = $first + $This.Codepoint = $first.Value + $This.Hex = '{0:x}' -f @($This.Codepoint) + $This.DisplayCodeoint = '0x' + $this.Hex + # just do 1 for now + # foreach($Rune in $InputText.EnumerateRunes()) { + + # } + $null -eq 2 + } + + [string] ToString() { + return '' + } + + [int] Length () { + return $this.Source.EnumerateRunes().Value.Count + } + + +} + +function StringToPqLiteral { + param( + [Parameter(ValueFromPipeline)] + $InputText + ) + + process { + $ErrorActionPreference = 'break' + $InputText.EnumerateRunes() | ForEach-Object { + [RuneToPqLiteral]::new($_) + } + $ErrorActionPreference = 'continue' + } +} + + +$sample = ' 🙊🐛🐈' + +StringToPqLiteral $sample \ No newline at end of file diff --git a/source-pwsh/src/__init__.ps1 b/source-pwsh/src/__init__.ps1 new file mode 100644 index 0000000..9ddd932 --- /dev/null +++ b/source-pwsh/src/__init__.ps1 @@ -0,0 +1,64 @@ +using namespace System.Text.StringBuilder +#Requires -Version 7.0 +#Requires -Module Ninmonkey.Console +# Requires -Module Dev.Nin + +Import-Module Ninmonkey.Console, Dev.nin -wa ignore + +$ModuleConfig = @{ + +} +$eaStop = @{ ErrorAction = 'stop' } + +$script:__Paths = @{ + RepoRoot = Get-Item @eaStop "$Env:UserProfile/SkyDrive/Documents/2021/Power BI/My_Github/Ninmonkey.PowerQueryLib" +} +$__paths = @{ + OutputRoot = Get-Item @eaStop Join-path $__paths.RepoRoot '.output' + ExportRoot = Get-Item @eaStop Join-path $__paths.RepoRoot '.export' + PowerQuerySourceRoot = Get-Item @eaStop Join-path $__paths.RepoRoot 'source' +} +function foo { + <# + .synpopsis + what + #> + [CmdletBinding()] + param() + + +} +function Set-PqLibOption { + <# + .synopsis + set options + #> + param( + [Parameter(Mandatory)] + [hashtable]$Options + ) + +} +function Get-PqLibOption { + <# + .synopsis + get options + #> + [CmdletBinding()] + param() + $opt = @{ + PsTypeName = 'Nin.PqLib.Options' + Paths = $script:__Paths + } + return [pscustomobject]$Opt +} + + +$Config = @{ + AutoOpenEditor = $false + AutoRoot = Get-Item -ea stop (Join-Path $PSScriptRoot '..') +} +$Config += @{ + AutoExportRoot = Join-Path $Config.AutoRoot '.output' +} + diff --git a/source/CoerceTo.Table.pq b/source/CoerceTo.Table.pq new file mode 100644 index 0000000..3a78487 --- /dev/null +++ b/source/CoerceTo.Table.pq @@ -0,0 +1,31 @@ +// CoerceToTable +let + CoerceToTable = + (source as any) as any => + // as table => + let + // currently these are redundant as case statements, but I may use multiple later + isRecord = Type.Is(sourceType, Record.Type), + isTable = Type.Is(sourceType, Table.Type), + isText = Type.Is(sourceType, Text.Type), + sourceType = Value.Type(source), + other = + error + Error.Record( + "UnhandledTypeConversion", + "UnhandledType", + sourceType + ) + in + if false then + null + else if isRecord then + Record.ToTable(source) as table + else if isText then + #table(1, {source}) as table + else if isTable then + source as table + else + try other catch (e) => e[Message] +in + CoerceToTable \ No newline at end of file diff --git a/source/CoerceTo.Table.succinct.pq b/source/CoerceTo.Table.succinct.pq new file mode 100644 index 0000000..c93329b --- /dev/null +++ b/source/CoerceTo.Table.succinct.pq @@ -0,0 +1,18 @@ +// CoerceToTable +let + // sugar to convert types to table, else text + CoerceToTable = + (source as any) as any => // as table => + let + sourceType = Value.Type(source) + in + if Type.Is(sourceType, Record.Type) then + Record.ToTable(source) as table + else if Type.Is(sourceType, Table.Type) then + source as table + else if Type.Is(sourceType, Text.Type) then + #table(1, {source}) as table + else + error Error.Record( "UnhandledTypeConversion", "UnhandledType", sourceType ) +in + CoerceToTable \ No newline at end of file diff --git a/source/DateTable_FromDates.backup.pq b/source/DateTable_FromDates.backup.pq new file mode 100644 index 0000000..0da4676 --- /dev/null +++ b/source/DateTable_FromDates.backup.pq @@ -0,0 +1,41 @@ +let + /* + Input: + Table with at least a [Date] column + output: + Adds columns for: [Year], [Month], [Day], [Week of Year], [Date Id] key + + future: + optionally add time when datetime is the source + + */ + DateTable.FromDates_Impl = () => + error Error.Record("NYI", "First write ListAll.Dates.pq"), + old_version_Date.TableFromDates = (source as table) as table => + let + // isDatetime = ..., // if datetime, or datetimezone, + source = + if Table.HasColumns( source, {"Date"} ) + then source + else error Error.Record( + "InvalidArgument", "Expects a table with a column named 'Date'", source + ), + + base =List.ContinuousDates( source[Date] ), + col_Year = Table.AddColumn(base, "Year", (_) as number => Date.Year([Date]), Int64.Type), + col_Month = Table.AddColumn(col_Year, "Month", (_) as number => Date.Month([Date]), Int64.Type), + col_Day = Table.AddColumn(col_Month, "Day", (_) as number => Date.Day([Date]), Int64.Type), + col_WeekOfYear = Table.AddColumn(col_Day, "Week of Year", (_) as number => Date.WeekOfYear([Date]), Int64.Type), + col_Index = Table.AddIndexColumn(col_WeekOfYear, "Date Id", 0, 1, Int64.Type), + verifyDistict = Table.IsDistinct(col_Index, {"Date"}), + Final = + if verifyDistict then col_Index + else error Error.Record( + "TableNotDistinct", "Using column 'date'", col_Index ) + in + Final + +in + Value.ReplaceType( DateTable.FromDates_Impl, DateTable.FromDates_Type1 ) + +a \ No newline at end of file diff --git a/source/DateTable_FromDates.pq b/source/DateTable_FromDates.pq index 7d2e75b..719913a 100644 --- a/source/DateTable_FromDates.pq +++ b/source/DateTable_FromDates.pq @@ -9,7 +9,9 @@ let optionally add time when datetime is the source */ - Date.TableFromDates = (source as table) as table => + DateTable.FromDates_Impl = () => + error Error.Record("NYI", "First write ListAll.Dates.pq"), + old_version_Date.TableFromDates = (source as table) as table => let // isDatetime = ..., // if datetime, or datetimezone, source = @@ -34,4 +36,5 @@ let Final in - Date.TableFromDates + Value.ReplaceType( DateTable.FromDates_Impl, DateTable.FromDates_Type1 ) + diff --git a/source/ErrorRecord.Format.pq b/source/ErrorRecord.Format.pq new file mode 100644 index 0000000..dcbb1e1 --- /dev/null +++ b/source/ErrorRecord.Format.pq @@ -0,0 +1,23 @@ +let + ErrorRecord.Format = (source as record, optional options as nullable record) as text => + let + maybeDetail = source[Detail]? ?? [], + prefix = Text.Combine({ + (source[Reason]? ?? ""), + (source[Message]? ?? "") + }, "#(cr,lf) "), + formatted = Text.Format( + (source[Message.Format]? ?? ""), + (source[Message.Parameters]? ?? null) + ), + combineAll = Text.Combine({ + maybeDetail, prefix, formatted + }, "#(cr,lf)"), + summary = [ + nyi = "func not finished", + combineAll = combineAll ] + in + summary + // combineAll meta [Exception = source] +in + ErrorRecord.Format \ No newline at end of file diff --git a/source/List.AllDates.pq b/source/List.AllDates.pq new file mode 100644 index 0000000..e9ac564 --- /dev/null +++ b/source/List.AllDates.pq @@ -0,0 +1,57 @@ +let + List.AllDates_Impl = (source as list) as table => + /* + Generate all dates between 2 or more dates. (order does not matter) + rewrite/rename: 'List.ContinuousDates.pq' + Source: + input: + a list or table table column of dates + output: + table of continuous [Date]s., for a Date table + future: + auto detect if arg is a table or a list + */ + let + // source = List.Buffer(source), // maybe? + + // future: assert compat with dates, + validArgs = source is list and first is date and last is date, + first = List.Min(source) as date, + last = List.Max(source) as date, + days = { Number.From(first)..Number.From(last) }, + extra = [ ValidArgs = validArgs, first = first, last = last, days = days ], + + Datefrom = DateTime.Date() + in + extra, + // a + + + // let + // first = List.Min(source), + // last = List.Max(source), + // days = { Number.From(first)..Number.From(last) }, + + // baseDates = List.Transform( + // days, each Date.From(_) ), + + // FinalTable = Table.FromList( + // baseDates, Splitter.SplitByNothing(), + // type table[Date = date], null, ExtraValues.Error + // ) + // in + // FinalTable, + + // show_example = false, + // example = + // let + // sample = { #date(2010,1,9), #date(2010,1,3), #date(2010,1,5) }, + // result = List.AllDates_Impl(sample) + // in + // result, + + // FinalResult = + // if show_example then example else List.AllDates_Impl + +in + FinalResult \ No newline at end of file diff --git a/source/Query.Summarize.pq b/source/Query.Summarize.pq new file mode 100644 index 0000000..099a37f --- /dev/null +++ b/source/Query.Summarize.pq @@ -0,0 +1,82 @@ +let + /* + future code + get: Number of Table Rows + test: Table.IsDistinct() + additional metadata of table + which primary keys are set + + generate record of table schema, then ToString() + + */ + options = [ + IgnoreQueries = {"Lib", "Listing of Queries"}, + HideNonTables = false + ], + Query.Summarize = (optional options as nullable record) => + let + options = [ + IgnoreQueries = {"Lib", "Listing Of Queries"}, + HideNonTables = false + ] & options, + + Source = Record.RemoveFields( #sections[Section1], options[IgnoreQueries] ), + base = Record.ToTable( Source ), + renamed_columns = Table.RenameColumns(base, + { {"Name", "Query Name"} }), + + add_QueryType = Table.AddColumn( + renamed_columns, "Query Type", + (row) => Type.ToText( row[Value] ), + type text + ), + optional_removeNonTables = + if not options[HideNonTables] then add_QueryType + else Table.SelectRows( add_QueryType, each ([Query Type] = "Table" or [Value] is table)), + + add_Schema = Table.AddColumn( + optional_removeNonTables, "Schema", + (row) => + if not (row[Value] is table) then null + else + try Table.Schema( row[Value] ) + catch (e) => error Error.Record( + "InnerException", "Table.Schema() failed", + e), + type table + ), + add_Profile = Table.AddColumn( + add_Schema, "Profile", + (row) => + if not (row[Value] is table) then null + else + try Table.Profile( row[Value] ) + catch (e) => error Error.Record( + "InnerException", "Table.Profile() failed", + e), + type table + ), + sort_type = Table.Sort( add_Profile, { + {"Query Type", Order.Descending}} ), + + final = sort_type, + + // converts values to typenames as text + Type.ToText = (typeInfo as any) as text => + let + name = + if typeInfo is null then "Null" + else if typeInfo is type then "Type" + else if typeInfo is binary then "Binary" + else if typeInfo is number then "Number" + else if typeInfo is function then "Function" + else if typeInfo is list then "List" + else if typeInfo is table then "Table" + else if typeInfo is record then "Record" + else "Other" meta [ ValueType = typeInfo ] + in + name + in + final + in + Query.Summarize \ No newline at end of file diff --git a/source/Query.Summary.pq b/source/Query.Summary.pq deleted file mode 100644 index 559f0a4..0000000 --- a/source/Query.Summary.pq +++ /dev/null @@ -1,48 +0,0 @@ -let - - Source = #sections, - Config = [ - IgnoreQueries = {"Query Summary"}, - KeepColumns = { "Query", "Type", "Table Meta" }, - HideNonTables = false - ], - /* - Summarize (all?) queries - future: - - [ ] apply schema, to have a better expansion - */ - Query.Summary = ( - optional ignoredKeys as nullable list - ) as any => //table => - let - source = Record.RemoveFields( - #sections[Section1], - (ignoredKeys ?? {"Query Summary"}) - ), - baseTable = Record.ToTable( source ), - t1 = Table.AddColumn( - baseTable, "Type", - each - Value.Type( _[Value] ), - Type.Type - ), - t2 = Table.AddColumn( - t1, "TypeName", - each - Type.ToText( Value.Type( _[Value] )), - type text - ), - t3 = Table.AddColumn( - t2, "Table Schema", - (row) => - if row[TypeName] = "Table" - then Table.Schema( row[Value] ) - else null, - Table.Type - ), - finalMeta = t3 - in - finalMeta, - Invoke = Query.Summary( Config[IgnoreQueries] ) -in - Invoke \ No newline at end of file diff --git a/source/Sketch-SummarizeError.pq b/source/Sketch-SummarizeError.pq new file mode 100644 index 0000000..20cc0c1 --- /dev/null +++ b/source/Sketch-SummarizeError.pq @@ -0,0 +1,102 @@ +// Sketch_ErrorSummary-i0 +let + Source = OData.Feed("https://services.odata.org/v4/TripPinService/"), + People = Source{[Name="People"]}[Data], + SelectColumns = Table.SelectColumns(People, {"UserName", "FirstName", "LastName"}), + + TestUrls = { + "https://services.odata.org/v4/TripPinService/Me", + "https://services.odata.org/v4/TripPinService/GetPersonWithMostFriends()", + "https://services.odata.org/v4/TripPinService/People" + }, + + queries = List.Transform( TestUrls, + (query) => [ + Query = query, + // Response = try OData.Feed(query) catch (e) => "Failed" & e + Response = try OData.Feed(query) catch (e) => "Failed" & e + ] + ), + queries_summary = Table.FromRecords( queries, type table[Query = text, Response = any] ), + Summary = [ + Source = Source, + People = People, + SelectColumns = SelectColumns, + TestUrls = TestUrls, + queries = queries, + queries_summary = queries_summary + + ], + queries_summary1 = Summary[queries_summary], + #"Added Custom" = Table.AddColumn(queries_summary1, "Custom", each try [Response] catch (e)=>e), + Custom1 = Table.AddColumn(queries_summary1, "Custom", each try [Response] catch (e)=> e ), + Custom = Custom1{1}[Custom], + Detail = Custom[Detail], + ErrSource2 = Detail[Right], + Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ), + Str = [ + NullSymbol = "#(2400)", + NL = "#(cr,lf)" + ], + + Template = [ + MinError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]" + }, Str[NL]), + Min2Error = Text.Combine({ + "Reason: #[Reason]", + "", + "Message: #[Message]" + }, Str[NL]), + BasicError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]", + // "", + "Format: #[Formatted]" + + }, Str[NL]), + + FullError = Text.Combine({ + // "Reason: #[Reason]", + // "Message: #[Message]", + // "Msg.Fmt: #[detail]", + // "Msg.Arg: #[detail]", + "" + }, Str[NL]) + ], + FormatErrorBasic = (source as record, optional options as nullable record) as text => + let + // reason = source[Reason]? ?? Str[NullSymbol], + /* + renderFormat usually equals Message + but is that always true? + */ + template = + options[Template]? ?? Template[BasicError], + renderFormat = + try Text.Format( source[Message.Format], source[Message.Parameters] ) + // catch (e) => e, + catch (e) => e[Message], + + render = Text.Format( + template, + [ + Reason = source[Reason]? ?? Str[NullSymbol], + Message = source[Message]? ?? Str[NullSymbol], + Formatted = renderFormat ?? Str[NullSymbol] + ] + ) + // Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ) + in + render, + + + RenderSummary = [ + Basic_Args = ErrSource2, + Min = FormatErrorBasic(ErrSource2, [Template = Template[MinError]]), + Min2 = FormatErrorBasic(ErrSource2, [Template = Template[Min2Error]]), + Duplicates = FormatErrorBasic(ErrSource2) + ] +in + RenderSummary \ No newline at end of file diff --git a/source/Text.MatchesAny.pq b/source/Text.MatchesAny.pq new file mode 100644 index 0000000..c5e1de3 --- /dev/null +++ b/source/Text.MatchesAny.pq @@ -0,0 +1,63 @@ +let + // does source contain any of these substrings? True when at least one does + // see also: List.MatchesAny: https://docs.microsoft.com/en-us/powerquery-m/list-matchesany + // example using List.Transform() instead, because it's used often + // see more: adding date popups + // https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata#function-parameter-documentation + // does source contain any of these substrings? True when at least one does + // see also: List.MatchesAny: https://docs.microsoft.com/en-us/powerquery-m/list-matchesany + // example using List.Transform() instead, because it's used often + Text.MatchesAny_Type = type function ( + message as ( + type text meta [ + Documentation.FieldCaption = "source", + Documentation.FieldDescription = "Text to Search", + Documentation.SampleValues = {"Hello world", "Bat"} + ] + ), + searchStrings as ( + type list meta [ + Documentation.FieldCaption = "Strings to Search", + Documentation.FieldDescription = "Search source, does it match any substrings? Case-insensitive by default", + Documentation.SampleValues = {{"Hello", "at"}, {"world"}} + ] + ), + // optional + optional options as ( + type nullable record meta [ + Documentation.FieldCaption = "Additional arguments", + Documentation.FieldDescription = "Toggle Case-Sensitive Comparer", + Documentation.SampleValues = { + [Comparer = Comparer.OrdinalIgnoreCase], [Comparer = Comparer.Ordinal] + // null + } + ] + ) + ) as ( + type logical meta [ + Documentation.Name = "Hello - Name", + Documentation.LongDescription = "Hello - Long Description", + Documentation.Examples = { + [ + Description = "Returns a table with 'Hello world' repeated 2 times", + Code = "HelloWorldWithDocs.Contents(""Hello world"", 2)", + Result = "#table({""Column1""}, {{""Hello world""}, {""Hello world""}})" + ], + [ + Description = "Another example, new message, new count!", + Code = "HelloWorldWithDocs.Contents(""Goodbye"", 1)", + Result = "#table({""Column1""}, {{""Goodbye""}})" + ] + } + ] + ), + Text.MatchesAny_Impl = (source as text, searchStrings as list, optional options as nullable record) as logical => + let + options = Record.Combine({defaults, options ?? []}), + defaults = [Comparer = Comparer.OrdinalIgnoreCase, Culture = "en-us"], + matchingList = List.Transform(searchStrings, (string) => Text.Contains(source, string, options[Comparer])) + in + List.AnyTrue(matchingList), + Text.MatchesAny = Value.ReplaceType(Text.MatchesAny_Impl, Text.MatchesAny_Type) +in + Text.MatchesAny diff --git a/source/Text.ReplaceMany.pq b/source/Text.ReplaceMany.pq new file mode 100644 index 0000000..c29296f --- /dev/null +++ b/source/Text.ReplaceMany.pq @@ -0,0 +1,47 @@ +let + Text.ReplaceMany + = ( source as text, mapping as list ) as text => + let + result = List.Accumulate( + mapping, source, + ( state, cur ) => Text.Replace(state, cur{0}, cur{1} )) + in + result, + Text.ReplaceMany.Type = type function( + source as ( + type text meta [ + Documentation.FieldCaption = "source", + Documentation.FieldDescription = "List to search and replace multiple values" + ]), + mapping as (type list meta[ + Documentation.FieldCaption = "List of {oldText, newText} replacement Pairs", + Documentation.FieldDescription = "A list of {oldText, newText} pairs, to search and then replace values" + ])) + as (type text meta [ + Documentation.Name = "Text.ReplaceMany", + Documentation.LongDescription = "A list of {oldText, newText} pairs,


#(cr,lf)#(cr,lf) to search case-sensitive, and then replace values", + /* + + example: + map = { + {"cat hat", "cat in the hat"}, + {"cat", "#(0001f408)"}, + {"bat", "hat"}, + {"THE", "the"} + }, + Text.ReplaceMany( "The cat hat") + + output: + "The 🐈 in the hat" + */ + Documentation.Examples = { + [ Description = "cat to emoji", + Code = "{0..4}", + Result = "none" ], + [ Description = "cat to emoji", + Code = "{0..4}", + Result = "none" ] + } + ]) +in + Value.ReplaceType( Text.ReplaceMany, Text.ReplaceMany.Type ) \ No newline at end of file diff --git a/source/Text.ReplaceMapping.pq b/source/Text.ReplaceMapping.pq deleted file mode 100644 index 1d67b91..0000000 --- a/source/Text.ReplaceMapping.pq +++ /dev/null @@ -1,32 +0,0 @@ -let - Text.ReplaceMapping = - "see if I can find a new trick in the url - - https://blog.crossjoin.co.uk/2014/06/25/using-list-generate-to-make-multiple-replacements-of-words-in-text-in-power-query/ - ", - - Source = "The cat in the hat.", - mapping = { - {"Cat", "#(0001f408)"}, - {" ", "#(cr,lf)" }, - {"hat", "matte" }, - {".", "?" } - }, - // expects a list of pairs of text - Text.ReplaceMany = ( source as text, mapping as list ) as text => - let - result = List.Accumulate( - mapping, source, - ( state, cur ) => Text.Replace( state, cur{0}, cur{1} )) - in - result, - - output = Text.ReplaceMany( Source, mapping ), - Summary = [ - Map = mapping, - Source = Source, - Final = output - - ] -in - Summary diff --git a/source/Type.ToText_simple.pq b/source/Type.ToText_simple.pq new file mode 100644 index 0000000..e379843 --- /dev/null +++ b/source/Type.ToText_simple.pq @@ -0,0 +1,45 @@ +let +/* +dependencies: None +*/ + Type.ToText.Type = type function( + typeInfo as (type any meta [ + Documentation.FieldCaption = "Object to test", + Documentation.FieldDescription = "Value to compare type names against", + Documentation.SampleValues = { + 23.45, "string", DateTime.LocalNow() + //, #table(type table[Text = text],{{"hi world"}}) + } + ]), + optional options as (type record meta [ + Documentation.FieldCaption = "for future formatting", + Documentation.FieldDescription = "for future formatting", + Documentation.SampleValues = { + } + ])) as table meta [ + Documentation.Name = "Get Type Name", + Documentation.LongDescription = "Returns type names as text, for the current value", + Documentation.Examples = {[ + Description = "Getting Type Names:", + Code = "{ Type.ToText(3), Type.ToText( {0..4} ), Type.ToText(null) }", + Result = "{ ""Number"", ""List"", ""None""}" + ]} + ], + Type.ToText_impl = (typeInfo as any, optional options as nullable record) as text => + let + options = Record.Combine([], options), + name = + if typeInfo is null then "Null" + else if typeInfo is type then "Type" + else if typeInfo is binary then "Binary" + else if typeInfo is number then "Number" + else if typeInfo is function then "Function" + else if typeInfo is list then "List" + else if typeInfo is table then "Table" + else if typeInfo is record then "Record" + else "Other" meta [ ValueType = typeInfo ] + in + name, + Type.ToText = Value.ReplaceType( Type.ToText_impl, Type.ToText.Type ) +in + Type.ToText diff --git a/source/Web.SimpleRequest.pq b/source/Web.SimpleRequest.pq new file mode 100644 index 0000000..82fd8d9 --- /dev/null +++ b/source/Web.SimpleRequest.pq @@ -0,0 +1,146 @@ +let + /* + + see also: + WebRequest_simple.pq + + Wrapper for Web.Contents, exposes returns response metadata + for options, see: + + Details on preventing "Refresh Errors", using 'Query' and 'RelativePath': + - Not using Query and Relative path cause refresh errors: + + + - You can opt-in to Skip-Test: + + + - Debugging and tracing the HTTP requests + + update: + - MaybeErrResponse: Quick example of parsing an error result. + - Raw text is returned, this is useful when there's an error + - now response[json] does not throw, when the data isn't json to begin with (false errors) + +for this url + + https://httpbin.org/get?Text=hi-world + +the way you split args is the same as Web.Contents, other than the 2nd parameter being +the [RelativePath] value. you would use: + +Web.SimpleRequest( "https://httpbin.org", "/get", [ + Query = [ + Text = "hi-world" + ] +) + + + todo: + + + + + + + + + + + +- embed M expression -> to comments like + add final [Documentation.Examples] file + + for this url#(cr,lf)#(cr,lf) https://httpbin.org/get?Text=hi-world#(cr,lf)#(cr,lf)the way you split args is the same as Web.Contents, other than the 2nd parameter being#(cr,lf)the [RelativePath] value. you would use:#(cr,lf)#(cr,lf)Web.SimpleRequest( ""https://httpbin.org"", ""/get"", [#(cr,lf) Query = [#(cr,lf) Text = ""hi-world""#(cr,lf) ]#(cr,lf)) + + */ + Web.SimpleRequest.Type = type function ( + baseUrl as ( // future: maybe use url type? + type text meta [ + Documentation.FieldCaption = "Minimum Base Url", + Documentation.FieldDescription = "This includes the url up until the first slash after a domain. The rest goes into query string", + Documentation.SampleValues = {"https://httpbin.org", "www.google.com"} + ] + ), + optional relative_path as ( + type nullable text meta [ + Documentation.FieldCaption = "Url relative path", + Documentation.FieldDescription = "everything after the "".com"" up to the ""?"" " , + Documentation.SampleValues = {"/someAPI/Employee/GetStats", null, ""} + ] + ), + optional options as ( + type nullable record meta [ + Documentation.FieldCaption = "All Web.Contents options", + Documentation.FieldDescription = "See for details, or the examples below", + Documentation.SampleValues = {"/someAPI/Employee/GetStats", null, ""} + ] + ) + ) as ( + type record meta [ + Documentation.Name = "Web - SimpleRequest", + Documentation.LongDescription = "Web.Contents - Makes repeated requests easier.", + Documentation.Examples = { + [ + Description = "", + Code = "", + Result = "" + ] + } + ] + ), + + Web.SimpleRequest = // WebRequest_Simple + ( baseUrl as text, + optional relative_path as nullable text, + optional options as nullable record + ) as record => + let + encoding = options[Encoding]? ?? TextEncoding.Utf8, + headers = options[Headers]?, //or: ?? [ Accept = "application/json" ], + manualStatusHandling = options[ManualStatusHandling]? ?? {400, 404..406, 500}, + merged_options = [ + // is null, blank, or [], more reliable? Some hosts and versions + // can differ on how they may implicitly join query strings + // test: is there a different between null paths verses the fields removed from the record? + Query = options[Query]?, //?? [], + RelativePath = relative_path, + ManualStatusHandling = manualStatusHandling, + Headers = headers + ], + + bytes = Web.Contents( baseUrl, merged_options ), + response = Binary.Buffer( bytes ) , + response_metadata = Value.Metadata( bytes ), + response_text = Text.Combine( Lines.FromBinary( + response,null,null, encoding ), "" ), + // depends on if I want to consume the errors from Json, usually it's + // content, but, sometimes it may be useful. + // Json = try Json.Document(response, encoding) catch (e) => null, + // Json = Json.Document(response, encoding), + // this part is new, not sure if it's 100% logically equivalent to the prev test + Json = + try + Json.Document( response, encoding) + catch (e) => + null meta [Exception = e], // null for simple usability, but preserve + // IsJson = Json <> null, //try false <> Json catch (e) => false, + // IsJson2 = try false <> Json catch (e) => false, + Final = [ + RequestUrl = response_metadata[Content.Uri](), + ResponseText = response_text, + RequestHeaders = response_text[Headers]? ?? null, + RequestOptions = response_text[Request.Options]? ?? [], // [] + + StatusCode = response_metadata[Response.Status]?, + MetaData = response_metadata, + IsJson = Json <> null, + // IsJson2 = IsJson, + Response = response, + Json = Json + + // json = if IsJson then json else null + ] + in + Final + in + Value.ReplaceType( Web.SimpleRequest, Web.SimpleRequest.Type ) \ No newline at end of file diff --git a/source/Web.SimpleRequest.tests.pq b/source/Web.SimpleRequest.tests.pq new file mode 100644 index 0000000..2efd55b --- /dev/null +++ b/source/Web.SimpleRequest.tests.pq @@ -0,0 +1,53 @@ +let + + Web.SimpleRequest = #"WebRequest New", + FinalTable = Table.FromRecords(tests, + type table[ + StatusCode = Int64.Type, + RequestUrl = text, + MetaData = record, + ResponseText = text, + IsJson = logical, + Json = nullable record, + Response = binary + ], + MissingField.Error + ), + tests = { // todo: remove + Web.SimpleRequest("https://httpbin.org", "json"), // expected: json + Web.SimpleRequest("https://www.google.com"), // expected: html + Web.SimpleRequest("https://main.ddrv8e4fbmhdv.amplifyapp.com", "/api/timezonemap"), + Web.SimpleRequest("https://main.ddrv8e4fbmhdv.amplifyapp.com", "/api/timezonemap", + [ManualStatusHandling = {404..406, 500}]), + Web.SimpleRequest("https://httpbin.org", "/headers"), + Web.SimpleRequest("https://httpbin.org", "/status/codes/406"), // expected: 404 + Web.SimpleRequest("https://httpbin.org", "/status/406"), // expected: 406 + Web.SimpleRequest("https://httpbin.org", "/get", [ Text = "hi-world"] ) + }, + // record.c + #"Include Metadata" = Table.AddColumn(FinalTable, "JsonErrorMetadata", + (row) => + let + maybeMeta = Value.Metadata( row[Json] ), + metaCount = Record.FieldCount( maybeMeta ) + in + if metaCount = 0 then null + else maybeMeta, + type record), + + singleMeta = RealFinal{7}[MetaData]? ?? "meta", + superVerbose = true, + VerboseFinal = #"Changed Type", + #"Expanded MetaData" = Table.ExpandRecordColumn(RealFinal, "MetaData", {"Content.Type", "Content.Uri", "Content.Name", "Headers", "Request.Options", "Response.Status"}, {"MetaData.Content.Type", "MetaData.Content.Uri", "MetaData.Content.Name", "MetaData.Headers", "MetaData.Request.Options", "MetaData.Response.Status"}), + #"Changed Type" = Table.TransformColumnTypes(#"Expanded MetaData",{{"StatusCode", Int64.Type}}), + RealFinal = #"Include Metadata", + + + Summary = [ + tests = tests, + FinalTable = FinalTable, + RealFinal = RealFinal, + singleMeta = #"Include Metadata" + ] +in + Summary \ No newline at end of file diff --git a/source/WebRequest_Simple.before_docs.pq b/source/WebRequest_Simple.before_docs.pq new file mode 100644 index 0000000..a69fe1c --- /dev/null +++ b/source/WebRequest_Simple.before_docs.pq @@ -0,0 +1,88 @@ +// WebRequest_Simple +// not clippy🐵 +let + + /* + Wrapper for Web.Contents returns response metadata + for options, see: + + Details on preventing "Refresh Errors", using 'Query' and 'RelativePath': + - Not using Query and Relative path cause refresh errors: + + + - You can opt-in to Skip-Test: + + + - Debugging and tracing the HTTP requests + + update: + - MaybeErrResponse: Quick example of parsing an error result. + - Raw text is returned, this is useful when there's an error + - now response[json] does not throw, when the data isn't json to begin with (false errors) + + + - embed M expression -> to comments like + add final [Documentation.Examples] file + + for this url#(cr,lf)#(cr,lf) https://httpbin.org/get?Text=hi-world#(cr,lf)#(cr,lf)the way you split args is the same as Web.Contents, other than the 2nd parameter being#(cr,lf)the [RelativePath] value. you would use:#(cr,lf)#(cr,lf)Web.SimpleRequest( ""https://httpbin.org"", ""/get"", [#(cr,lf) Query = [#(cr,lf) Text = ""hi-world""#(cr,lf) ]#(cr,lf)) + + */ + WebRequest_Simple + = ( + base_url as text, + optional relative_path as nullable text, + optional options as nullable record + ) + as record => + let + + headers = options[Headers]?, //or: ?? [ Accept = "application/json" ], + + merged_options = [ + Query = options[Query]?, + RelativePath = relative_path, + ManualStatusHandling = options[ManualStatusHandling]? ?? { 400, 404, 406 }, + Headers = headers + ], + + bytes = Web.Contents(base_url, merged_options), + response = Binary.Buffer(bytes), + response_metadata = Value.Metadata( bytes ), + status_code = response_metadata[Response.Status]?, + response_text = Text.Combine( Lines.FromBinary(response,null,null, TextEncoding.Utf8), "" ), + json = Json.Document(response), + IsJsonX = not (try json)[HasError], + Final = [ + request_url = metadata[Content.Uri](), + response_text = response_text, + status_code = status_code, + metadata = response_metadata, + IsJson = IsJsonX, + response = response, + + json = if IsJsonX then json else null + ] + in + Final, + + tests = { + WebRequest_Simple("https://httpbin.org", "json"), // expect: json + WebRequest_Simple("https://www.google.com"), // expect: html + WebRequest_Simple("https://httpbin.org", "/headers"), + WebRequest_Simple("https://httpbin.org", "/status/codes/406"), // exect 404 + WebRequest_Simple("https://httpbin.org", "/status/406"), // exect 406 + WebRequest_Simple("https://httpbin.org", "/get", [ Text = "Hello World"]) + }, + + FinalResults = Table.FromRecords(tests, + type table[ + status_code = Int64.Type, request_url = text, + metadata = record, + response_text = text, + IsJson = logical, json = any, + response = binary + ], + MissingField.Error + ) +in + FinalResults diff --git a/source/WebRequest_Simple.pq b/source/WebRequest_Simple.pq index 9191a37..a69fe1c 100644 --- a/source/WebRequest_Simple.pq +++ b/source/WebRequest_Simple.pq @@ -1,10 +1,11 @@ // WebRequest_Simple +// not clippy🐵 let /* Wrapper for Web.Contents returns response metadata for options, see: - + Details on preventing "Refresh Errors", using 'Query' and 'RelativePath': - Not using Query and Relative path cause refresh errors: @@ -13,12 +14,18 @@ let - Debugging and tracing the HTTP requests - + update: - MaybeErrResponse: Quick example of parsing an error result. - Raw text is returned, this is useful when there's an error - now response[json] does not throw, when the data isn't json to begin with (false errors) + + - embed M expression -> to comments like + add final [Documentation.Examples] file + + for this url#(cr,lf)#(cr,lf) https://httpbin.org/get?Text=hi-world#(cr,lf)#(cr,lf)the way you split args is the same as Web.Contents, other than the 2nd parameter being#(cr,lf)the [RelativePath] value. you would use:#(cr,lf)#(cr,lf)Web.SimpleRequest( ""https://httpbin.org"", ""/get"", [#(cr,lf) Query = [#(cr,lf) Text = ""hi-world""#(cr,lf) ]#(cr,lf)) + */ WebRequest_Simple = ( diff --git a/source/alias/default_alias_list.pq b/source/alias/default_alias_list.pq index cdbe623..022aec3 100644 --- a/source/alias/default_alias_list.pq +++ b/source/alias/default_alias_list.pq @@ -2,17 +2,21 @@ let mapping = [ Join-String = Text.Combine, t = Value.Type, + mdt = Inspect.Meta, + // value.me = Value, - random = { - ..,... /// list of all funcs related to random - } - findFunc = ..., // filter functions in current session with pattern + // random = { + // ..,... /// list of all funcs related to random + // }, + // findFunc = ..., // filter functions in current session with pattern - fmt_Hex = Number_ToHexString + // fmt_Hex = Number_ToHexString ] meta [ pqLibAlias = "Inspect.MetaOfType", IsAlias = true ] +in + mapping diff --git a/source/sketch-error-summary.md b/source/sketch-error-summary.md new file mode 100644 index 0000000..b0cd36d --- /dev/null +++ b/source/sketch-error-summary.md @@ -0,0 +1,310 @@ +- [base](#base) +- [cleaner?](#cleaner) +- [copy ?](#copy--) +## base + +## cleaner? + +```pq +let + Source = OData.Feed("https://services.odata.org/v4/TripPinService/"), + People = Source{[Name="People"]}[Data], + SelectColumns = Table.SelectColumns(People, {"UserName", "FirstName", "LastName"}), + + TestUrls = { + "https://services.odata.org/v4/TripPinService/Me", + "https://services.odata.org/v4/TripPinService/GetPersonWithMostFriends()", + "https://services.odata.org/v4/TripPinService/People" + }, + + queries = List.Transform( TestUrls, + (query) => [ + Query = query, + Response = try OData.Feed(query) catch (e) => e + ] + ), + queries_summary = Table.FromRecords( queries, type table[Query = text, Response = any] ), + Str = [ + NullSymbol = "#(2400)", + NL = "#(cr,lf)" + ], + + Template = [ + BasicError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]" + // "Format: #[Formatted]" + + }, Str[NL]), + + FullError = Text.Combine({ + // "Reason: #[Reason]", + // "Message: #[Message]", + // "Msg.Fmt: #[detail]", + // "Msg.Arg: #[detail]", + "" + }, Str[NL]) + ], + FormatErrorBasic = (source as record) as text => + let + // reason = source[Reason]? ?? Str[NullSymbol], + renderFormat = + try Text.Format( source[Message.Format], source[Message.Parameters] ) + // catch (e) => e, + catch (e) => e[Message], + + render = Text.Format( + Template[BasicError], + [ + /* future keys: + Reason, Message, Detail, Message.Format, Message.Parameters + */ + Reason = source[Reason]? ?? Str[NullSymbol], + Message = source[Message]? ?? Str[NullSymbol], + Formatted = renderFormat ?? Str[NullSymbol] + ] + ) + // Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ) + in + render, + + RenderSummary = [ + Basic_Args = ErrSource2, + Basic = FormatErrorBasic(ErrSource2) + ], + Summary = [ + Source = Source, + People = People, + SelectColumns = SelectColumns, + TestUrls = TestUrls, + queries = queries, + queries_summary = queries_summary + + ], + queries_summary1 = Summary[queries_summary], + #"Added Custom" = Table.AddColumn(queries_summary1, + "DataSourceErrorMessage", + (row) => + let + IsError = row[Response][Reason]? = "DataSource.Error", + render = FormatErrorBasic( row[Response] ), + fin = if IsError then render else "" + in + fin, + + type text + ) + +in + #"Added Custom" +``` + +```pq +let + Source = OData.Feed("https://services.odata.org/v4/TripPinService/"), + People = Source{[Name="People"]}[Data], + SelectColumns = Table.SelectColumns(People, {"UserName", "FirstName", "LastName"}), + + TestUrls = { + "https://services.odata.org/v4/TripPinService/Me", + "https://services.odata.org/v4/TripPinService/GetPersonWithMostFriends()", + "https://services.odata.org/v4/TripPinService/People" + }, + + queries = List.Transform( TestUrls, + (query) => [ + Query = query, + // Response = try OData.Feed(query) catch (e) => "Failed" & e + Response = try OData.Feed(query) catch (e) => "Failed" & e + ] + ), + queries_summary = Table.FromRecords( queries, type table[Query = text, Response = any] ), + Summary = [ + Source = Source, + People = People, + SelectColumns = SelectColumns, + TestUrls = TestUrls, + queries = queries, + queries_summary = queries_summary + + ], + queries_summary1 = Summary[queries_summary], + #"Added Custom" = Table.AddColumn(queries_summary1, "Custom", each try [Response] catch (e)=>e), + Custom1 = Table.AddColumn(queries_summary1, "Custom", each try [Response] catch (e)=> e ), + Custom = Custom1{1}[Custom], + Detail = Custom[Detail], + ErrSource2 = Detail[Right], + Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ), + Str = [ + NullSymbol = "#(2400)", + NL = "#(cr,lf)" + ], + + Template = [ + MinError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]" + }, Str[NL]), + Min2Error = Text.Combine({ + "Reason: #[Reason]", + "", + "Message: #[Message]" + }, Str[NL]), + BasicError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]", + // "", + "Format: #[Formatted]" + + }, Str[NL]), + + FullError = Text.Combine({ + // "Reason: #[Reason]", + // "Message: #[Message]", + // "Msg.Fmt: #[detail]", + // "Msg.Arg: #[detail]", + "" + }, Str[NL]) + ], + FormatErrorBasic = (source as record, optional options as nullable record) as text => + let + // reason = source[Reason]? ?? Str[NullSymbol], + /* + renderFormat usually equals Message + but is that always true? + */ + template = + options[Template]? ?? Template[BasicError], + renderFormat = + try Text.Format( source[Message.Format], source[Message.Parameters] ) + // catch (e) => e, + catch (e) => e[Message], + + render = Text.Format( + template, + [ + Reason = source[Reason]? ?? Str[NullSymbol], + Message = source[Message]? ?? Str[NullSymbol], + Formatted = renderFormat ?? Str[NullSymbol] + ] + ) + // Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ) + in + render, + + + RenderSummary = [ + Basic_Args = ErrSource2, + Min = FormatErrorBasic(ErrSource2, [Template = Template[MinError]]), + Min2 = FormatErrorBasic(ErrSource2, [Template = Template[Min2Error]]), + Duplicates = FormatErrorBasic(ErrSource2) + ] +in + RenderSummary +``` + +## copy ? + +```pq +let + Source = OData.Feed("https://services.odata.org/v4/TripPinService/"), + People = Source{[Name="People"]}[Data], + SelectColumns = Table.SelectColumns(People, {"UserName", "FirstName", "LastName"}), + + TestUrls = { + "https://services.odata.org/v4/TripPinService/Me", + "https://services.odata.org/v4/TripPinService/GetPersonWithMostFriends()", + "https://services.odata.org/v4/TripPinService/People" + }, + + queries = List.Transform( TestUrls, + (query) => [ + Query = query, + // Response = try OData.Feed(query) catch (e) => "Failed" & e + Response = try OData.Feed(query) catch (e) => "Failed" & e + ] + ), + queries_summary = Table.FromRecords( queries, type table[Query = text, Response = any] ), + Summary = [ + Source = Source, + People = People, + SelectColumns = SelectColumns, + TestUrls = TestUrls, + queries = queries, + queries_summary = queries_summary + + ], + queries_summary1 = Summary[queries_summary], + #"Added Custom" = Table.AddColumn(queries_summary1, "Custom", each try [Response] catch (e)=>e), + Custom1 = Table.AddColumn(queries_summary1, "Custom", each try [Response] catch (e)=> e ), + Custom = Custom1{1}[Custom], + Detail = Custom[Detail], + ErrSource2 = Detail[Right], + Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ), + Str = [ + NullSymbol = "#(2400)", + NL = "#(cr,lf)" + ], + + Template = [ + MinError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]" + }, Str[NL]), + Min2Error = Text.Combine({ + "Reason: #[Reason]", + "", + "Message: #[Message]" + }, Str[NL]), + BasicError = Text.Combine({ + "Reason: #[Reason]", + "Message: #[Message]", + // "", + "Format: #[Formatted]" + + }, Str[NL]), + + FullError = Text.Combine({ + // "Reason: #[Reason]", + // "Message: #[Message]", + // "Msg.Fmt: #[detail]", + // "Msg.Arg: #[detail]", + "" + }, Str[NL]) + ], + FormatErrorBasic = (source as record, optional options as nullable record) as text => + let + // reason = source[Reason]? ?? Str[NullSymbol], + /* + renderFormat usually equals Message + but is that always true? + */ + template = + options[Template]? ?? Template[BasicError], + renderFormat = + try Text.Format( source[Message.Format], source[Message.Parameters] ) + // catch (e) => e, + catch (e) => e[Message], + + render = Text.Format( + template, + [ + Reason = source[Reason]? ?? Str[NullSymbol], + Message = source[Message]? ?? Str[NullSymbol], + Formatted = renderFormat ?? Str[NullSymbol] + ] + ) + // Custom2 = (source as record) as any => Text.Format( source[Message.Format], source[Message.Parameters] ) + in + render, + + + RenderSummary = [ + Basic_Args = ErrSource2, + Min = FormatErrorBasic(ErrSource2, [Template = Template[MinError]]), + Min2 = FormatErrorBasic(ErrSource2, [Template = Template[Min2Error]]), + Duplicates = FormatErrorBasic(ErrSource2) + ] +in + RenderSummary +``` \ No newline at end of file diff --git a/source/test/summarize-schema.pq b/source/test/summarize-schema.pq new file mode 100644 index 0000000..cf929ab --- /dev/null +++ b/source/test/summarize-schema.pq @@ -0,0 +1,44 @@ +let + Source = AllRecords, + Schema = Table.Schema( Source ), + + #"Drop Most Properties" = Table.ReorderColumns( + Schema,{"Name", "TypeName", "Kind", "IsNullable", "Position", "NumericPrecisionBase", + "NumericPrecision", "NumericScale", "DateTimePrecision", "MaxLength", "IsVariableLength", + "NativeTypeName", "NativeDefaultExpression", "NativeExpression", "Description", "IsWritable", "FieldCaption"}), + + // summarizes column types and name as a string + SummarizeSchemaColumn = + (column as record, + optional options as nullable record) as any => + let + options = Record.Combine({default, options ?? []}), + default = [ + Template = "[#[AscribedTypeName]]: #[ColName]#[Nullability] = #[TypeName]#[Nullability]" + ], + render = Text.Format( + options[Template], + [ + AscribedTypeName = column[TypeName], + TypeName = column[Kind], + ColName = column[Name], + Nullability = if column[IsNullable] then "?" else "" + ] + ) + in + render, + + #"Removed Other Columns" = Table.SelectColumns( + #"Drop Most Properties",{"Name", "TypeName", "Kind", "IsNullable", "Position"}), + + #"Summarized Cols" = Table.AddColumn( + #"Removed Other Columns", + "Summary", + (row as record) as any => + try SummarizeSchemaColumn( row ) catch (e) => e, + type text + ), + Summary = #"Summarized Cols", + #"Reordered Columns" = Table.ReorderColumns(Summary,{"Summary", "Name", "TypeName", "Kind", "IsNullable", "Position"}) +in + #"Reordered Columns" \ No newline at end of file diff --git a/src-new-text-replacemany-documented.2022-07.pq b/src-new-text-replacemany-documented.2022-07.pq new file mode 100644 index 0000000..53d26ea --- /dev/null +++ b/src-new-text-replacemany-documented.2022-07.pq @@ -0,0 +1,43 @@ +let +/* +doc tips: + +documentation on the types uses for documentation + https://docs.microsoft.com/en-us/power-query/handlingdocumentation + +from: + https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata +> A note about Documentation.Name: This value should probably match the name of the identifier you assign your function when you define it. For example, if you call your function SomeFunction, then when you set Documentation.Name, you should probably set its value to “SomeFunction”, as well. + +*/ + + + Text.ReplaceMany + = ( source as text, mapping as list ) as text => + let + result = List.Accumulate( + mapping, source, + ( state, cur ) => Text.Replace(state, cur{0}, cur{1} )) + in + result, + Text.ReplaceMany.Type = type function( + source as ( + type text meta [ + Documentation.FieldCaption = "source", + Documentation.FieldDescription = "List to search and replace multiple values" + ]), + mapping as (type list meta[ + Documentation.FieldCaption = "List of {oldText, newText} replacement Pairs", + Documentation.FieldDescription = "A list of {oldText, newText} pairs, to search and then replace values" + ])) + as (type text meta [ + Documentation.Name = "Text.ReplaceMany", + Documentation.LongDescription = "A list of {oldText, newText} pairs,


#(cr,lf)#(cr,lf) to search and then replace values", + Documentation.Examples = {[ + Description = "cat to emoji", + Code = "{0..4}", + Result = "none" + ]} + ]) +in + Value.ReplaceType( Text.ReplaceMany, Text.ReplaceMany.Type ) \ No newline at end of file diff --git a/testing/PowerQuery-documentation-grammar.md b/testing/PowerQuery-documentation-grammar.md new file mode 100644 index 0000000..ff0e8dc --- /dev/null +++ b/testing/PowerQuery-documentation-grammar.md @@ -0,0 +1,63 @@ +## Important + +- https://docs.microsoft.com/en-us/power-query/handlingdocumentation +- Intro to docs metadata: https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata + +## Related: +- https://bengribaudo.com/blog/2020/06/02/5259/power-query-m-primer-part18-type-system-iii-custom-types +- https://bengribaudo.com/blog/2020/09/03/5408/power-query-m-primer-part19-type-system-iv-ascription-conformance-and-equalitys-strange-behaviors + + +## Partial lexical grammar of PQ documentation + +- Todo: Script that mines the names of all metadata on built-in connectors +- this is to detect any missing `Documentation.` keys for function metadata +Abbreviation +`Documentation.Examples` as `D.Examples` + +### Basic pseudo lexical grammar + +```yml +type Function: + | D.Examples + | D.LongDescription + | D.Name + +type Parameter: + | D.AllowedValues + | D.FieldCaption + | D.FieldDescription + | D.SampleValues + | Formatting.IsCode + | Formatting.IsMultiLine +``` + +### expanded with types + +```yml +type Function: + | D.Examples + ListOf: + type [ Description, Code, Result ] + | D.LongDescription + text + | D.Name + text + +type Parameter: + | D.AllowedValues + ListOf: + type any + + | D.FieldCaption + text + | D.FieldDescription + text + | D.SampleValues + ListOf: + type any + | Formatting.IsCode + logical + | Formatting.IsMultiLine + logical +``` \ No newline at end of file diff --git a/testing/syntax-highlight-stress-other.pq b/testing/syntax-highlight-stress-other.pq new file mode 100644 index 0000000..e038c02 --- /dev/null +++ b/testing/syntax-highlight-stress-other.pq @@ -0,0 +1,5 @@ +let + q =3, // e=4, + T.Wfd = List.Count = q = e = 3 +in + [ q=q, lc = List.Count , j =false][[q],[lc]] \ No newline at end of file diff --git a/testing/syntax-highlight-stress-test.pq b/testing/syntax-highlight-stress-test.pq new file mode 100644 index 0000000..c088863 --- /dev/null +++ b/testing/syntax-highlight-stress-test.pq @@ -0,0 +1,44 @@ +/* +this code is terrible on purpose. +it's meant to stress test syntax highlighting +It is syntactically valid query + Foo.bar( , ) + Table.List( Tbar.Casdf(...) ) + ) +see more: + */ +let + T.Wfd = List.Count = q = e = 3, + l = TA.A.Wfd(), + l3 = T.Wfd(), + l7 = T.Aa(), + l9 = T.Aaf(), + l2 = TA.AWfd(), + T.L.P = List.Count, + j = T.L(...), q = T.L.P( ), + q1 = #"List."( each 0, each _ < 1, each _ ), + z = table.list(df), + #"TA.A.Wfd" = "#(cr,lf)#""Table.List(""", + #" Table.List(bar Table.List({3,45,5}))" = 345, + a = Table.List, + b = Table(), + now = #duration(0,0,0,1), t = #table( type table[ Id = Int64.Type, Nested = Table.Type], {} ), + r = Table.Combine({null, #"Some.Stepame"( ... ) }), + r2 = Table.Combine( + {null, #"Some.Stepame"( ... ) } + ), + r3 = {null, #"Some.Stepame"( ... ) }, // #"Table.List"#".List", + #"[ T.Bar(" = [ fd = List.Bar( List.Transform( {9..4}, each _ + 3 )), When = DateTime.FixedLocalNow() ], + // c = List.Generate(...) ], + zed = List.Transform( source, (item) => item * 3 ) + Bar.FromText( " Table.List(" & ".." ), + /* + Text.Type, TA.A.Wfd, + Text.Type + */ + q3 = foo.bar(sdf), + #"q35" = Bcat( { 234 } ), + z2 = Bcat(), + g = Table.Bar(dsf) + + +in [g=Table.RowCount( #table(null, {{null}}))] \ No newline at end of file diff --git a/todo.snippets.pq.md b/todo.snippets.pq.md new file mode 100644 index 0000000..1b6ac8d --- /dev/null +++ b/todo.snippets.pq.md @@ -0,0 +1,24 @@ +snippets to write first +snippets + +- [ ] new buffer +- [ ] export env to json, with version numbers +- [ ] table join types +- [ ] documentation helper + +## textmate grammar + + +todo: powerquery + +snippets + new buffer + table join types + documentation helper + +update the grammar + https://github.com/microsoft/vscode-powerquery/issues/66#issuecomment-739428136 + + https://github.com/microsoft/vscode-powerquery/tree/master/syntaxes + + original repo: https://github.com/microsoft/powerquery-language \ No newline at end of file diff --git "a/\360\237\220\222 mini-workspace.PowerQueryLib.code-workspace" "b/\360\237\220\222 -c- 2022- mini-workspace.PowerQueryLib.code-workspace" similarity index 86% rename from "\360\237\220\222 mini-workspace.PowerQueryLib.code-workspace" rename to "\360\237\220\222 -c- 2022- mini-workspace.PowerQueryLib.code-workspace" index 8eb90c0..d91beed 100644 --- "a/\360\237\220\222 mini-workspace.PowerQueryLib.code-workspace" +++ "b/\360\237\220\222 -c- 2022- mini-workspace.PowerQueryLib.code-workspace" @@ -2,12 +2,16 @@ "folders": [ { "path": ".", - "name": "//PowerQueryLib🐒" + "name": "c : 2022 : //PowerQueryLib🐒" }, { "name": "vscode-pq-parser", "path": "..\\..\\..\\..\\..\\2022-fam-share\\logs\\vscode-pq-parser" - } + }, + { + "path": "C:/Users/cppmo_000/SkyDrive/Documents/2022/Power-BI/My_Github/ninMonkQuery-examples", + "name": "c : 2022 : //ninMonkQuery-examples🐒" + }, ], "settings": { "explorer.fileNesting.enabled": true, @@ -66,5 +70,6 @@ } ], "search.searchEditor.defaultNumberOfContextLines": null, + "powershell.cwd": "c:\\Users\\cppmo_000\\SkyDrive\\Documents\\2022\\Power-BI\\My_Github\\Ninmonkey.PowerQueryLib", } } \ No newline at end of file