From 27ffdc29a8d1d2d8df2b9caee121b44238588170 Mon Sep 17 00:00:00 2001 From: sverdel Date: Mon, 17 Aug 2020 09:18:10 +0500 Subject: [PATCH] add histogram to grpc --- Benchmark.NetCore/Benchmark.NetCore.csproj | 1 + Benchmark.NetCore/GrpcExporterBenchmarks.cs | 62 +++++++++++++++++++ Benchmark.NetCore/Program.cs | 3 +- .../GrpcMetricsMiddlewareExtensions.cs | 5 ++ .../GrpcMiddlewareExporterOptions.cs | 2 + .../GrpcRequestCountMiddleware.cs | 14 ++++- .../GrpcRequestDurationMiddleware.cs | 50 +++++++++++++++ .../GrpcRequestDurationOptions.cs | 10 +++ .../GrpcRequestLabelNames.cs | 8 +++ .../GrpcRequestMiddlewareBase.cs | 6 ++ .../HttpResponseExtensions.cs | 28 +++++++++ 11 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 Benchmark.NetCore/GrpcExporterBenchmarks.cs create mode 100644 Prometheus.AspNetCore.Grpc/GrpcRequestDurationMiddleware.cs create mode 100644 Prometheus.AspNetCore.Grpc/GrpcRequestDurationOptions.cs create mode 100644 Prometheus.AspNetCore.Grpc/HttpResponseExtensions.cs diff --git a/Benchmark.NetCore/Benchmark.NetCore.csproj b/Benchmark.NetCore/Benchmark.NetCore.csproj index dc948687..d688dc24 100644 --- a/Benchmark.NetCore/Benchmark.NetCore.csproj +++ b/Benchmark.NetCore/Benchmark.NetCore.csproj @@ -28,6 +28,7 @@ + diff --git a/Benchmark.NetCore/GrpcExporterBenchmarks.cs b/Benchmark.NetCore/GrpcExporterBenchmarks.cs new file mode 100644 index 00000000..dde8ecf1 --- /dev/null +++ b/Benchmark.NetCore/GrpcExporterBenchmarks.cs @@ -0,0 +1,62 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.AspNetCore.Http; +using Prometheus; +using System.Threading.Tasks; +using Grpc.AspNetCore.Server; +using Grpc.Core; + +namespace Benchmark.NetCore +{ + [MemoryDiagnoser] + public class GrpcExporterBenchmarks + { + private CollectorRegistry _registry; + private MetricFactory _factory; + private GrpcRequestCountMiddleware _countMiddleware; + private GrpcRequestDurationMiddleware _durationMiddleware; + private DefaultHttpContext _ctx; + + [Params(1000, 10000)] + public int RequestCount { get; set; } + + [GlobalSetup] + public void Setup() + { + _ctx = new DefaultHttpContext(); + _ctx.SetEndpoint(new Endpoint( + ctx => Task.CompletedTask, + new EndpointMetadataCollection(new GrpcMethodMetadata(typeof(int), + new Method(MethodType.Unary, + "test", + "test", + new Marshaller(o => new byte[0], c => null), + new Marshaller(o => new byte[0], c => null)))), + "test")); + _registry = Metrics.NewCustomRegistry(); + _factory = Metrics.WithCustomRegistry(_registry); + + _countMiddleware = new GrpcRequestCountMiddleware(next => Task.CompletedTask, new GrpcRequestCountOptions + { + Counter = _factory.CreateCounter("count", "help") + }); + _durationMiddleware = new GrpcRequestDurationMiddleware(next => Task.CompletedTask, new GrpcRequestDurationOptions + { + Histogram = _factory.CreateHistogram("duration", "help") + }); + } + + [Benchmark] + public async Task GrpcRequestCount() + { + for (var i = 0; i < RequestCount; i++) + await _countMiddleware.Invoke(_ctx); + } + + [Benchmark] + public async Task GrpcRequestDuration() + { + for (var i = 0; i < RequestCount; i++) + await _durationMiddleware.Invoke(_ctx); + } + } +} \ No newline at end of file diff --git a/Benchmark.NetCore/Program.cs b/Benchmark.NetCore/Program.cs index 9d618768..a0a7e5f3 100644 --- a/Benchmark.NetCore/Program.cs +++ b/Benchmark.NetCore/Program.cs @@ -11,7 +11,8 @@ private static void Main(string[] args) //BenchmarkRunner.Run(); //BenchmarkRunner.Run(); //BenchmarkRunner.Run(); - BenchmarkRunner.Run(); + //BenchmarkRunner.Run(); + BenchmarkRunner.Run(); } } } diff --git a/Prometheus.AspNetCore.Grpc/GrpcMetricsMiddlewareExtensions.cs b/Prometheus.AspNetCore.Grpc/GrpcMetricsMiddlewareExtensions.cs index 895066a1..1e93e247 100644 --- a/Prometheus.AspNetCore.Grpc/GrpcMetricsMiddlewareExtensions.cs +++ b/Prometheus.AspNetCore.Grpc/GrpcMetricsMiddlewareExtensions.cs @@ -30,6 +30,11 @@ public static IApplicationBuilder UseGrpcMetrics(this IApplicationBuilder app, app.UseMiddleware(options.RequestCount); } + if (options.RequestDuration.Enabled) + { + app.UseMiddleware(options.RequestDuration); + } + return app; } } diff --git a/Prometheus.AspNetCore.Grpc/GrpcMiddlewareExporterOptions.cs b/Prometheus.AspNetCore.Grpc/GrpcMiddlewareExporterOptions.cs index f7d65578..15ddf441 100644 --- a/Prometheus.AspNetCore.Grpc/GrpcMiddlewareExporterOptions.cs +++ b/Prometheus.AspNetCore.Grpc/GrpcMiddlewareExporterOptions.cs @@ -3,5 +3,7 @@ public sealed class GrpcMiddlewareExporterOptions { public GrpcRequestCountOptions RequestCount { get; set; } = new GrpcRequestCountOptions(); + + public GrpcRequestDurationOptions RequestDuration { get; set; } = new GrpcRequestDurationOptions(); } } diff --git a/Prometheus.AspNetCore.Grpc/GrpcRequestCountMiddleware.cs b/Prometheus.AspNetCore.Grpc/GrpcRequestCountMiddleware.cs index 11211fa0..39769934 100644 --- a/Prometheus.AspNetCore.Grpc/GrpcRequestCountMiddleware.cs +++ b/Prometheus.AspNetCore.Grpc/GrpcRequestCountMiddleware.cs @@ -19,9 +19,17 @@ public GrpcRequestCountMiddleware(RequestDelegate next, GrpcRequestCountOptions? public async Task Invoke(HttpContext context) { - CreateChild(context)?.Inc(); - - await _next(context); + try + { + await _next(context); + } + finally + { + // We need to record metrics after inner handler execution because routing data in + // ASP.NET Core 2 is only available *after* executing the next request delegate. + // So we would not have the right labels if we tried to create the child early on. + CreateChild(context)?.Inc(); + } } protected override string[] DefaultLabels => GrpcRequestLabelNames.All; diff --git a/Prometheus.AspNetCore.Grpc/GrpcRequestDurationMiddleware.cs b/Prometheus.AspNetCore.Grpc/GrpcRequestDurationMiddleware.cs new file mode 100644 index 00000000..ab19050c --- /dev/null +++ b/Prometheus.AspNetCore.Grpc/GrpcRequestDurationMiddleware.cs @@ -0,0 +1,50 @@ +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Prometheus.HttpMetrics; + +namespace Prometheus +{ + internal sealed class GrpcRequestDurationMiddleware : GrpcRequestMiddlewareBase, IHistogram> + { + private readonly RequestDelegate _next; + + public GrpcRequestDurationMiddleware(RequestDelegate next, GrpcRequestDurationOptions? options) + : base(options, options?.Histogram) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + } + + public async Task Invoke(HttpContext context) + { + var stopWatch = Stopwatch.StartNew(); + + // We need to write this out in long form instead of using a timer because routing data in + // ASP.NET Core 2 is only available *after* executing the next request delegate. + // So we would not have the right labels if we tried to create the child early on. + try + { + await _next(context); + } + finally + { + stopWatch.Stop(); + + CreateChild(context)?.Observe(stopWatch.Elapsed.TotalSeconds); + } + } + + protected override string[] DefaultLabels => GrpcRequestLabelNames.NoStatusSpecific; + + protected override ICollector CreateMetricInstance(string[] labelNames) => MetricFactory.CreateHistogram( + "grpc_request_duration_seconds", + "The duration of gRPC requests processed by an ASP.NET Core application.", + new HistogramConfiguration + { + // 1 ms to 32K ms buckets + Buckets = Histogram.ExponentialBuckets(0.001, 2, 16), + LabelNames = labelNames + }); + } +} \ No newline at end of file diff --git a/Prometheus.AspNetCore.Grpc/GrpcRequestDurationOptions.cs b/Prometheus.AspNetCore.Grpc/GrpcRequestDurationOptions.cs new file mode 100644 index 00000000..6fa3f11a --- /dev/null +++ b/Prometheus.AspNetCore.Grpc/GrpcRequestDurationOptions.cs @@ -0,0 +1,10 @@ +namespace Prometheus +{ + public sealed class GrpcRequestDurationOptions : GrpcMetricsOptionsBase + { + /// + /// Set this to use a custom metric instead of the default. + /// + public ICollector? Histogram { get; set; } + } +} \ No newline at end of file diff --git a/Prometheus.AspNetCore.Grpc/GrpcRequestLabelNames.cs b/Prometheus.AspNetCore.Grpc/GrpcRequestLabelNames.cs index ac17708e..fb01b9e0 100644 --- a/Prometheus.AspNetCore.Grpc/GrpcRequestLabelNames.cs +++ b/Prometheus.AspNetCore.Grpc/GrpcRequestLabelNames.cs @@ -7,8 +7,16 @@ public static class GrpcRequestLabelNames { public const string Service = "service"; public const string Method = "method"; + public const string Status = "status"; public static readonly string[] All = + { + Service, + Method, + Status, + }; + + public static readonly string[] NoStatusSpecific = { Service, Method, diff --git a/Prometheus.AspNetCore.Grpc/GrpcRequestMiddlewareBase.cs b/Prometheus.AspNetCore.Grpc/GrpcRequestMiddlewareBase.cs index 380841da..569bf7eb 100644 --- a/Prometheus.AspNetCore.Grpc/GrpcRequestMiddlewareBase.cs +++ b/Prometheus.AspNetCore.Grpc/GrpcRequestMiddlewareBase.cs @@ -1,7 +1,9 @@ using Microsoft.AspNetCore.Http; using System; using System.Linq; +using System.Threading; using Grpc.AspNetCore.Server; +using Grpc.Core; namespace Prometheus { @@ -78,6 +80,10 @@ protected TChild CreateChild(HttpContext context, GrpcMethodMetadata metadata) case GrpcRequestLabelNames.Method: labelValues[i] = metadata.Method.Name; break; + case GrpcRequestLabelNames.Status: + labelValues[i] = context.Response?.GetStatusCode().ToString() ?? StatusCode.OK.ToString(); + break; + default: // Should never reach this point because we validate in ctor. throw new NotSupportedException($"Unexpected label name on {_metric.Name}: {_metric.LabelNames[i]}"); diff --git a/Prometheus.AspNetCore.Grpc/HttpResponseExtensions.cs b/Prometheus.AspNetCore.Grpc/HttpResponseExtensions.cs new file mode 100644 index 00000000..a3ab6b1c --- /dev/null +++ b/Prometheus.AspNetCore.Grpc/HttpResponseExtensions.cs @@ -0,0 +1,28 @@ +using System.Linq; +using Grpc.Core; +using Microsoft.AspNetCore.Http; + +namespace Prometheus +{ + internal static class HttpResponseExtensions + { + private const string _grpcStatus = "grpc-status"; + + public static StatusCode GetStatusCode(this HttpResponse response) + { + var headerExists = response.Headers.TryGetValue(_grpcStatus, out var header); + + if (!headerExists && response.StatusCode == StatusCodes.Status200OK) + { + return StatusCode.OK; + } + + if (header.Any() && int.TryParse(header.FirstOrDefault(), out var status)) + { + return (StatusCode)status; + } + + return StatusCode.OK; + } + } +} \ No newline at end of file