From b225c3cd600ff29a8ea17b94f2c2bac877a91660 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:32:23 -0600 Subject: [PATCH 1/8] WIP: envelope v1, shared transport, single-receiver processor, routing, middleware, dead-letter queue --- .../AzureStorage.QueueService.csproj | 1 + .../DeadLetterQueue.cs | 230 +++++++++ .../DelegateHandler.cs | 74 +++ .../ILargeMessageStore.cs | 49 ++ .../IOwnedReceiver.cs | 18 +- src/AzureStorage.QueueService/IQueueSender.cs | 34 +- .../InMemoryQueueStore.cs | 16 +- .../InMemoryStorageQueue.cs | 136 ------ .../JsonQueueMessageSerializer.cs | 31 +- src/AzureStorage.QueueService/LeaseRenewal.cs | 20 +- .../LoggerMessageExtensions.cs | 51 +- src/AzureStorage.QueueService/MessageCodec.cs | 302 ++++++++++++ .../MessageEnvelope.cs | 368 ++++++++++++++ .../PeekedQueueMessage.cs | 20 + .../QueueClientBuilderExtensions.cs | 281 +++++++++-- .../QueueClientFactory.cs | 73 ++- .../QueueClientSettings.cs | 94 +++- .../QueueClientSettingsBuilder.cs | 107 ++++- .../QueueClientSettingsRegistry.cs | 16 + .../QueueDepthMonitor.cs | 77 +++ src/AzureStorage.QueueService/QueueMessage.cs | 325 ++++++++++--- .../QueueMessageInfo.cs | 18 +- .../QueueMessageMiddleware.cs | 73 +++ .../QueueProcessor.cs | 447 +++++++++++++++--- .../QueueProcessorController.cs | 142 ++++++ .../QueueProcessorOptions.cs | 102 +++- .../QueueTransport.cs | 220 +++++++++ src/AzureStorage.QueueService/Routing.cs | 164 +++++++ .../SdkQueueClientFactory.cs | 2 +- .../ServiceCollectionExtensions.cs | 40 +- src/AzureStorage.QueueService/StorageQueue.cs | 414 +++++++++++----- .../StorageQueueCommon.cs | 83 ---- .../MeterProviderBuilderExtensions.cs | 14 + .../Telemetry/QueueServiceDiagnostics.cs | 29 +- .../Telemetry/QueueServiceMetrics.cs | 156 +++++- .../Telemetry/TraceContextEnvelope.cs | 176 ------- .../IntegrationTests.cs | 8 +- .../QueueProcessorTests.cs | 7 +- .../StorageQueueTests.cs | 32 +- .../TraceContextEnvelopeTests.cs | 50 -- 40 files changed, 3658 insertions(+), 842 deletions(-) create mode 100644 src/AzureStorage.QueueService/DeadLetterQueue.cs create mode 100644 src/AzureStorage.QueueService/DelegateHandler.cs create mode 100644 src/AzureStorage.QueueService/ILargeMessageStore.cs delete mode 100644 src/AzureStorage.QueueService/InMemoryStorageQueue.cs create mode 100644 src/AzureStorage.QueueService/MessageCodec.cs create mode 100644 src/AzureStorage.QueueService/MessageEnvelope.cs create mode 100644 src/AzureStorage.QueueService/QueueDepthMonitor.cs create mode 100644 src/AzureStorage.QueueService/QueueMessageMiddleware.cs create mode 100644 src/AzureStorage.QueueService/QueueProcessorController.cs create mode 100644 src/AzureStorage.QueueService/QueueTransport.cs create mode 100644 src/AzureStorage.QueueService/Routing.cs delete mode 100644 src/AzureStorage.QueueService/StorageQueueCommon.cs delete mode 100644 src/AzureStorage.QueueService/Telemetry/TraceContextEnvelope.cs delete mode 100644 tests/AzureStorage.QueueService.Tests/TraceContextEnvelopeTests.cs diff --git a/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj b/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj index 9544592..a6aa590 100644 --- a/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj +++ b/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj @@ -31,6 +31,7 @@ + diff --git a/src/AzureStorage.QueueService/DeadLetterQueue.cs b/src/AzureStorage.QueueService/DeadLetterQueue.cs new file mode 100644 index 0000000..1989396 --- /dev/null +++ b/src/AzureStorage.QueueService/DeadLetterQueue.cs @@ -0,0 +1,230 @@ +using Azure; +using Azure.Storage.Queues.Models; + +namespace AzureStorage.QueueService; + +/// +/// A queue's poison queue: see what was dead-lettered and why, and move messages back to be processed again. Inject it +/// for the default client, or with [FromKeyedServices("name")] for a named one, or call +/// . +/// +public interface IDeadLetterQueue +{ + /// The poison queue's name. + string QueueName { get; } + + /// The queue whose messages land here, and where sends them back to. + string SourceQueueName { get; } + + /// Looks at up to (1 to 32) dead-lettered messages without receiving them. + ValueTask> PeekAsync(int maxMessages = 32, CancellationToken cancellationToken = default); + + /// + /// Moves up to messages back to , those for which + /// returns true (all, without one), and returns how many were moved. Each goes back as it + /// was sent, without the dead-letter record, and gets a fresh dequeue count; its time to live starts again at the + /// service default of 7 days. Messages the filter skips stay on the poison queue. + /// + /// + /// Each message is copied and then deleted, so a failure between the two can leave it on both queues; the handler + /// sees it at least once, as with any delivery. Run one replay at a time per poison queue. + /// + ValueTask ReplayAsync(int maxMessages = int.MaxValue, Func? filter = null, CancellationToken cancellationToken = default); + + /// The service's approximate count of messages on the poison queue; 0 when it does not exist yet. + ValueTask GetApproximateCountAsync(CancellationToken cancellationToken = default); +} + +/// A message on a poison queue, with the record of why it was dead-lettered. +public sealed class DeadLetteredMessage +{ + private readonly IQueueMessageSerializer _serializer; + + internal DeadLetteredMessage(QueueMessageInfo info, string sourceQueueName, DeadLetterInfo? record, IReadOnlyDictionary headers, BinaryData rawBody, DeserializationException? error, IQueueMessageSerializer serializer) + { + MessageId = info.MessageId; + DeadLetteredOn = record?.At ?? info.InsertedOn; + SourceQueueName = record?.Queue ?? sourceQueueName; + OriginalMessageId = record?.MessageId; + Reason = record?.Reason; + Category = record?.Category; + DequeueCount = record?.DequeueCount ?? 0; + Headers = headers; + RawBody = rawBody; + Error = error; + _serializer = serializer; + } + + /// The message's id on the poison queue. + public string MessageId { get; } + + /// The id it had on the source queue, when the dead-letter record has it. + public string? OriginalMessageId { get; } + + /// The queue it was dead-lettered from. + public string SourceQueueName { get; } + + /// Why it was dead-lettered, as logged at the time. + public string? Reason { get; } + + /// One of , when the record has it. + public string? Category { get; } + + /// How many times it had been delivered when it was dead-lettered. + public long DequeueCount { get; } + + /// When it was dead-lettered. + public DateTimeOffset? DeadLetteredOn { get; } + + /// Headers the sender attached. + public IReadOnlyDictionary Headers { get; } + + /// + /// The serialized message as the sender's serializer wrote it, unwrapped, decompressed and, for a body kept in Blob + /// storage, downloaded. A message that cannot be read that way (see ) is here as it arrived, or + /// empty when its body is in Blob storage and could not be downloaded. + /// + public BinaryData RawBody { get; } + + /// Set when the body could not be unwrapped or downloaded. + public DeserializationException? Error { get; } + + /// Reads the body as with the source client's serializer. + /// The body could not be read as . + public T? GetBody() => Error is not null ? throw Error : _serializer.Deserialize(RawBody); +} + +internal sealed class DeadLetterQueue : IDeadLetterQueue +{ + // Long enough to look at a batch and move it back; skipped messages are made visible again at the end. + private static readonly TimeSpan ScanVisibility = TimeSpan.FromMinutes(5); + + private readonly StorageQueue _source; + + public DeadLetterQueue(StorageQueue source) + { + _source = source; + } + + public string QueueName => _source.PoisonQueue().Name; + + public string SourceQueueName => _source.QueueName; + + public async ValueTask> PeekAsync(int maxMessages = 32, CancellationToken cancellationToken = default) + { + StorageQueue.ValidateMaxMessages(maxMessages); + var poison = _source.PoisonQueue(); + + IReadOnlyList peeked; + try + { + peeked = await poison.PeekAsync(maxMessages, cancellationToken); + } + catch (RequestFailedException e) when (e.ErrorCode == QueueErrorCode.QueueNotFound) + { + return []; + } + + var messages = new List(peeked.Count); + foreach (var m in peeked) messages.Add(await ReadAsync(m, cancellationToken)); + return messages; + } + + public async ValueTask ReplayAsync(int maxMessages = int.MaxValue, Func? filter = null, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(maxMessages); + var poison = _source.PoisonQueue(); + + var replayed = 0; + // Skipped messages, by id, with their latest receipt: made visible again at the end. + var skipped = new Dictionary(StringComparer.Ordinal); + try + { + while (replayed < maxMessages) + { + IReadOnlyList batch; + try + { + batch = await poison.ReceiveAsync(StorageQueue.ServiceMaxMessages, ScanVisibility, cancellationToken); + } + catch (RequestFailedException e) when (e.ErrorCode == QueueErrorCode.QueueNotFound) + { + break; + } + + // Stop when the queue is empty, or when everything left is something already looked at and skipped + // (a slow scan outlives ScanVisibility and sees those again). + var anythingNew = false; + foreach (var m in batch) + { + if (skipped.ContainsKey(m.Info.MessageId)) + { + skipped[m.Info.MessageId] = m.Info.PopReceipt; + continue; + } + + anythingNew = true; + if (replayed >= maxMessages || (filter is not null && !filter(await ReadAsync(m, cancellationToken)))) + { + skipped[m.Info.MessageId] = m.Info.PopReceipt; + continue; + } + + await _source.SendWireAsync(WithoutDeadLetterRecord(m.Body), cancellationToken); + await poison.DeleteAsync(m.Info.MessageId, m.Info.PopReceipt, cancellationToken); + replayed++; + } + + if (!anythingNew) break; + } + } + finally + { + foreach (var (id, receipt) in skipped) + { + try + { + await poison.UpdateVisibilityAsync(id, receipt, TimeSpan.Zero, CancellationToken.None); + } + catch (RequestFailedException) + { + // Its scan lease lapsed and someone else has it, or it is gone; either way it is not hidden by us. + } + } + } + + return replayed; + } + + public async ValueTask GetApproximateCountAsync(CancellationToken cancellationToken = default) + { + try + { + return (await _source.PoisonQueue().GetPropertiesAsync(cancellationToken)).ApproximateMessagesCount; + } + catch (RequestFailedException e) when (e.ErrorCode == QueueErrorCode.QueueNotFound) + { + return 0; + } + } + + private async ValueTask ReadAsync(TransportMessage m, CancellationToken cancellationToken) + { + // Read like any received message (envelope, decompression, a body kept in Blob storage), but never settled + // through this object: replay works on the transport directly. + var message = MessageCodec.Received(m, _source); + var error = message.Error ?? await _source.ResolvePayloadAsync(message.Lease, cancellationToken); + var record = message.Lease.Envelope?.DeadLetter; + return new DeadLetteredMessage(m.Info, _source.QueueName, record, message.Headers, message.RawBody, error, _source.Codec.Serializer); + } + + /// The message as it was before it was dead-lettered: the envelope without its record, or just the body when nothing else is left. + internal static BinaryData WithoutDeadLetterRecord(BinaryData wire) + { + if (MessageEnvelope.TryRead(wire, out var envelope, out _) != EnvelopeKind.Envelope || envelope!.DeadLetter is null) return wire; + + var copy = envelope.Clone(); + copy.DeadLetter = null; + return copy.CarriesOnlyBody ? copy.Body! : copy.Write(); + } +} diff --git a/src/AzureStorage.QueueService/DelegateHandler.cs b/src/AzureStorage.QueueService/DelegateHandler.cs new file mode 100644 index 0000000..b84b66e --- /dev/null +++ b/src/AzureStorage.QueueService/DelegateHandler.cs @@ -0,0 +1,74 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; + +namespace AzureStorage.QueueService; + +/// +/// How a lambda handler's parameters are filled, worked out once at registration: the message body, the message, the +/// cancellation token, or a service from the message's scope. +/// +internal sealed class DelegateHandlerBinding +{ + private readonly Delegate _handler; + private readonly Func, IServiceProvider, CancellationToken, object?>[] _arguments; + + public DelegateHandlerBinding(Delegate handler) + { + _handler = handler; + var method = handler.Method; + + if (method.ReturnType != typeof(void) && !typeof(Task).IsAssignableFrom(method.ReturnType) && method.ReturnType != typeof(ValueTask)) + throw new ArgumentException($"A handler delegate must return void, Task or ValueTask; this one returns {method.ReturnType.Name}.", nameof(handler)); + + _arguments = method.GetParameters().Select(Bind).ToArray(); + Name = method.Name.Contains('<') ? $"lambda handler for {typeof(TMessage).Name}" : method.Name; + } + + /// What logs call the handler: the method's name, or a description for a lambda. + public string Name { get; } + + private static Func, IServiceProvider, CancellationToken, object?> Bind(ParameterInfo parameter) + { + var type = parameter.ParameterType; + if (type == typeof(TMessage)) return static (message, _, _) => message.Body; + if (type == typeof(QueueMessage) || type == typeof(IQueueMessage)) return static (message, _, _) => message; + if (type == typeof(CancellationToken)) return static (_, _, token) => token; + if (type == typeof(IServiceProvider)) return static (_, services, _) => services; + + if (parameter.GetCustomAttribute() is { } keyed) + { + return parameter.HasDefaultValue + ? (_, services, _) => services.GetKeyedService(type, keyed.Key) ?? parameter.DefaultValue + : (_, services, _) => services.GetRequiredKeyedService(type, keyed.Key); + } + + return parameter.HasDefaultValue + ? (_, services, _) => services.GetService(type) ?? parameter.DefaultValue + : (_, services, _) => services.GetRequiredService(type); + } + + public async ValueTask InvokeAsync(QueueMessage message, IServiceProvider services, CancellationToken cancellationToken) + { + var arguments = new object?[_arguments.Length]; + for (var i = 0; i < arguments.Length; i++) arguments[i] = _arguments[i](message, services, cancellationToken); + + // DoNotWrapExceptions: the handler's own exception reaches the processor, not a TargetInvocationException. + var result = _handler.Method.Invoke(_handler.Target, BindingFlags.DoNotWrapExceptions, binder: null, arguments, culture: null); + switch (result) + { + case Task task: + await task; + break; + case ValueTask valueTask: + await valueTask; + break; + } + } +} + +/// The handler behind AddProcessor<TMessage>(Delegate), created per message in its scope. +internal sealed class DelegateHandler(DelegateHandlerBinding binding, IServiceProvider services) : IQueueMessageHandler +{ + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) => + binding.InvokeAsync(message, services, cancellationToken); +} diff --git a/src/AzureStorage.QueueService/ILargeMessageStore.cs b/src/AzureStorage.QueueService/ILargeMessageStore.cs new file mode 100644 index 0000000..a35a5ad --- /dev/null +++ b/src/AzureStorage.QueueService/ILargeMessageStore.cs @@ -0,0 +1,49 @@ +namespace AzureStorage.QueueService; + +/// +/// Keeps message bodies that are too big for a queue message somewhere else, leaving a reference on the queue (the +/// claim-check pattern). The AzureStorage.QueueService.Blobs package implements it over Blob storage; attach one +/// to a client with UseLargeMessageStore or that package's UseBlobStorageForLargeMessages. +/// +/// +/// A client with a store sends a body to it when the message would otherwise be over the queue's 64 KiB limit (or over +/// ), and reads it back, inside the delivery and after lease +/// renewal has started, when a message refers to one. The body is deleted once the message is completed, and kept when +/// it is dead-lettered so a replay still has it. Every client that reads the queue needs the same store. +/// +public interface ILargeMessageStore +{ + /// Stores for a message going to and returns the reference to put on the queue. + ValueTask UploadAsync(string queueName, BinaryData body, CancellationToken cancellationToken); + + /// Reads back a body stored by . + ValueTask DownloadAsync(string reference, CancellationToken cancellationToken); + + /// Deletes a stored body. Deleting one that is already gone is not an error. + ValueTask DeleteAsync(string reference, CancellationToken cancellationToken); +} + +/// +/// Thrown before sending a message the service would reject for its size: over 64 KiB as it would go on the queue, +/// after any envelope and Base64 encoding. Attach an to send bodies that big. +/// +public sealed class QueueMessageTooLargeException : InvalidOperationException +{ + /// Creates the exception for a message of bytes against a limit of . + public QueueMessageTooLargeException(string queueName, int size, int limit, string detail) + : base($"Queue '{queueName}': the message is {size:N0} bytes as it would go on the queue, over the service's limit of {limit:N0} bytes. {detail}") + { + QueueName = queueName; + Size = size; + Limit = limit; + } + + /// The queue the message was for. + public string QueueName { get; } + + /// The message's size as it would have gone on the queue, in bytes. + public int Size { get; } + + /// The service's limit, in bytes. + public int Limit { get; } +} diff --git a/src/AzureStorage.QueueService/IOwnedReceiver.cs b/src/AzureStorage.QueueService/IOwnedReceiver.cs index 63ccd7b..4263137 100644 --- a/src/AzureStorage.QueueService/IOwnedReceiver.cs +++ b/src/AzureStorage.QueueService/IOwnedReceiver.cs @@ -1,15 +1,21 @@ namespace AzureStorage.QueueService; /// -/// The processor's receive path. A message comes back with its identity and raw body but without deserialization, -/// so the processor can arm lease renewal first and deserialize inside the delivery boundary. The public -/// contract is unchanged; this is implemented by -/// and used when the processor's queue supports it. +/// The processor's receive path. Messages come back with their identity and raw body but without deserialization, so +/// the processor can arm lease renewal first and do the slow parts (downloading a body kept in Blob storage, +/// deserializing) inside the delivery boundary. The public contract is +/// unchanged; this is implemented by and used when the processor's queue supports it. /// internal interface IOwnedReceiver { - /// Receives one message, or null when the queue is empty. is set only for a decoding failure. - ValueTask?> ReceiveUndeserializedAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken); + /// + /// Receives up to messages. is set only for one + /// that cannot be decoded or unwrapped. A failure is thrown without being logged: the processor logs it. + /// + ValueTask>> ReceiveForProcessingAsync(int maxMessages, TimeSpan visibilityTimeout, CancellationToken cancellationToken); + + /// Brings in a body kept outside the queue. Returns the problem, or null; never throws except for cancellation. + ValueTask ResolvePayloadAsync(MessageLease message, CancellationToken cancellationToken); /// Fills in or . Never throws. void Deserialize(QueueMessage message); diff --git a/src/AzureStorage.QueueService/IQueueSender.cs b/src/AzureStorage.QueueService/IQueueSender.cs index 082fcf4..26eefbd 100644 --- a/src/AzureStorage.QueueService/IQueueSender.cs +++ b/src/AzureStorage.QueueService/IQueueSender.cs @@ -7,10 +7,38 @@ namespace AzureStorage.QueueService; /// public interface IQueueSender : INamedQueue { + /// Serializes and puts it on the queue. + /// asks for a delay or time to live the service would reject. + /// The message would be over the service's 64 KiB limit and the client has no large message store. /// The service rejected the send. ValueTask SendAsync(T message, SendOptions? options = null, CancellationToken cancellationToken = default); } -/// How long the message stays hidden after it is sent; by default it is visible immediately. -/// How long the message lives before the service discards it; the service default is 7 days. Use to never expire. -public sealed record SendOptions(TimeSpan? VisibilityTimeout = null, TimeSpan? TimeToLive = null); +/// How one message is sent. +/// +/// How long the message stays hidden after it is sent (0 to 7 days, and less than ); by +/// default it is visible immediately. is another name for the same thing. +/// +/// How long the message lives before the service discards it (at least 1 second); the service default is 7 days. Use to never expire. +public sealed record SendOptions(TimeSpan? VisibilityTimeout = null, TimeSpan? TimeToLive = null) +{ + /// How long to wait before the message can be received: under the name scheduling code expects. + public TimeSpan? Delay + { + get => VisibilityTimeout; + init => VisibilityTimeout = value; + } + + /// + /// Name and value pairs that travel with the message, read back as : a correlation + /// id, a tenant, a content type. Headers put the message in the library's envelope, which only receivers on 3.1 or + /// later read, so see the README before sending them to a queue older code also reads. + /// + public IReadOnlyDictionary? Headers { get; init; } + + /// + /// The message's type, sent as the header, which a routed processor + /// uses to choose the handler. Overrides for this message. + /// + public string? MessageType { get; init; } +} diff --git a/src/AzureStorage.QueueService/InMemoryQueueStore.cs b/src/AzureStorage.QueueService/InMemoryQueueStore.cs index 993edda..f3da28d 100644 --- a/src/AzureStorage.QueueService/InMemoryQueueStore.cs +++ b/src/AzureStorage.QueueService/InMemoryQueueStore.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using Azure; using Azure.Storage.Queues.Models; -using Microsoft.Extensions.Logging; namespace AzureStorage.QueueService; @@ -23,15 +22,8 @@ public InMemoryQueueStore(TimeProvider time) // Every queue exists: a test should not have to provision what an emulator or deployment script would. public InMemoryQueue GetQueue(string queueName) => _queues.GetOrAdd(queueName, name => new InMemoryQueue(name, _time)); - /// A client over the queue names. The settings must have passed . - public IStorageQueue CreateClient(QueueClientSettings settings, IQueueMessageSerializer serializer, ILoggerFactory loggerFactory) => - new InMemoryStorageQueue( - GetQueue(settings.InMemoryQueueName!), - () => GetQueue(settings.InMemoryPoisonQueueName!), - serializer, - loggerFactory, - settings.VisibilityTimeout, - settings.PropagateTraceContext); + /// The clock the queues run on: the registered , or the system clock. + public TimeProvider Time => _time; } /// @@ -42,7 +34,7 @@ internal sealed class InMemoryQueue { private static readonly TimeSpan MaxVisibility = QueueClientSettings.MaxVisibility; private static readonly TimeSpan DefaultTimeToLive = TimeSpan.FromDays(7); - private static readonly TimeSpan InfiniteTimeToLive = StorageQueueCommon.InfiniteTimeToLive; + private static readonly TimeSpan InfiniteTimeToLive = StorageQueue.InfiniteTimeToLive; private readonly object _lock = new(); private readonly List _messages = new(); @@ -59,7 +51,7 @@ public InMemoryQueue(string name, TimeProvider time) /// /// Takes the time to live as the service does: -1 second for never. The client converts .NET's -1 ms sentinel first - /// (), as the real client does before calling the service. + /// (), as the real client does before calling the service. /// public SendReceipt Send(BinaryData body, TimeSpan? visibilityTimeout, TimeSpan? timeToLive) { diff --git a/src/AzureStorage.QueueService/InMemoryStorageQueue.cs b/src/AzureStorage.QueueService/InMemoryStorageQueue.cs deleted file mode 100644 index 810d721..0000000 --- a/src/AzureStorage.QueueService/InMemoryStorageQueue.cs +++ /dev/null @@ -1,136 +0,0 @@ -using System.Diagnostics; -using Azure.Storage.Queues.Models; -using Microsoft.Extensions.Logging; - -namespace AzureStorage.QueueService; - -/// -/// A client over an : what every hands out -/// once AddInMemoryQueues is registered. Mirrors , including the processor's -/// path and the trace-context envelope, so a processor under test takes the same route -/// it takes in production. -/// -internal sealed class InMemoryStorageQueue : IStorageQueue, IQueueMessageOperations, IOwnedReceiver -{ - private static readonly TimeSpan ServiceDefaultVisibility = TimeSpan.FromSeconds(30); - - private readonly InMemoryQueue _queue; - private readonly Lazy _poisonQueue; - private readonly IQueueMessageSerializer _serializer; - private readonly ILogger _logger; - private readonly TimeSpan? _defaultVisibilityTimeout; - private readonly bool _propagateTraceContext; - - public InMemoryStorageQueue( - InMemoryQueue queue, - Func poisonQueue, - IQueueMessageSerializer serializer, - ILoggerFactory loggerFactory, - TimeSpan? defaultVisibilityTimeout, - bool propagateTraceContext) - { - _queue = queue; - _poisonQueue = new Lazy(poisonQueue); - _serializer = serializer; - _logger = loggerFactory.CreateLogger(); - _defaultVisibilityTimeout = defaultVisibilityTimeout; - _propagateTraceContext = propagateTraceContext; - } - - public string QueueName => _queue.Name; - - public ValueTask SendAsync(T message, SendOptions? options = null, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - // No send span of its own, so the caller's current span is the one carried, as StorageQueue does when its - // send span is turned off. - var body = StorageQueueCommon.Encode(_serializer, message, _propagateTraceContext, Activity.Current, out _); - return new(_queue.Send(body, options?.VisibilityTimeout, StorageQueueCommon.ServiceTimeToLive(options?.TimeToLive))); - } - - public ValueTask>> ReceiveAsync(int maxMessages = 1, TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default) - { - var messages = ReceiveUndeserialized(maxMessages, visibilityTimeout, cancellationToken); - foreach (var message in messages) Deserialize(message); - return new(messages); - } - - ValueTask?> IOwnedReceiver.ReceiveUndeserializedAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken) => - new(ReceiveUndeserialized(1, visibilityTimeout, cancellationToken).FirstOrDefault()); - - public void Deserialize(QueueMessage message) => StorageQueueCommon.Deserialize(_serializer, message); - - private List> ReceiveUndeserialized(int maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) - { - StorageQueueCommon.ValidateMaxMessages(maxMessages); - cancellationToken.ThrowIfCancellationRequested(); - - return _queue.Receive(maxMessages, visibilityTimeout ?? _defaultVisibilityTimeout ?? ServiceDefaultVisibility) - .Select(m => StorageQueueCommon.Received(new QueueMessageInfo(m.MessageId, m.PopReceipt, m.DequeueCount, m.InsertedOn, m.ExpiresOn, m.NextVisibleOn), m.Body, this)) - .ToList(); - } - - public ValueTask>> PeekAsync(int maxMessages = 1, CancellationToken cancellationToken = default) - { - StorageQueueCommon.ValidateMaxMessages(maxMessages); - cancellationToken.ThrowIfCancellationRequested(); - - IReadOnlyList> peeked = _queue.Peek(maxMessages) - .Select(m => - { - var raw = StorageQueueCommon.Peeked(m.Body); - var (body, error) = StorageQueueCommon.TryDeserialize(_serializer, raw); - return new PeekedQueueMessage(m.MessageId, m.DequeueCount, m.InsertedOn, m.ExpiresOn, raw, body, error); - }) - .ToList(); - return new(peeked); - } - - public ValueTask CreateIfNotExistsAsync(IDictionary? metadata = null, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - _queue.CreateIfNotExists(metadata); - return ValueTask.CompletedTask; - } - - public ValueTask ClearAsync(CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - _queue.Clear(); - return ValueTask.CompletedTask; - } - - public ValueTask GetPropertiesAsync(CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - return new(_queue.GetProperties()); - } - - ValueTask IQueueMessageOperations.DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - _queue.Delete(messageId, popReceipt); - return ValueTask.CompletedTask; - } - - ValueTask IQueueMessageOperations.UpdateVisibilityAsync(string messageId, string popReceipt, TimeSpan visibilityTimeout, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - return new(_queue.UpdateVisibility(messageId, popReceipt, visibilityTimeout)); - } - - ValueTask IQueueMessageOperations.DeadLetterAsync(string messageId, string popReceipt, long dequeueCount, BinaryData rawBody, string? reason, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - var poison = _poisonQueue.Value; - - // The same two steps, in the same order, as StorageQueue: copy the body as it arrived (envelope and all), then - // delete with the receipt. A stale receipt therefore leaves a duplicate in the poison queue here too, which is - // the at-least-once trade. - poison.Send(rawBody, visibilityTimeout: null, timeToLive: null); - _queue.Delete(messageId, popReceipt); - - _logger.LogDeadLettered(QueueName, messageId, poison.Name, dequeueCount, reason ?? "(none)"); - return ValueTask.CompletedTask; - } -} diff --git a/src/AzureStorage.QueueService/JsonQueueMessageSerializer.cs b/src/AzureStorage.QueueService/JsonQueueMessageSerializer.cs index 69f5371..bc8237c 100644 --- a/src/AzureStorage.QueueService/JsonQueueMessageSerializer.cs +++ b/src/AzureStorage.QueueService/JsonQueueMessageSerializer.cs @@ -1,27 +1,52 @@ +using System.Diagnostics.CodeAnalysis; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; namespace AzureStorage.QueueService; internal sealed class JsonQueueMessageSerializer : IQueueMessageSerializer { + /// + /// The library's defaults: System.Text.Json's, except that property names are read case-insensitively, so a camelCase + /// message from JavaScript, Azure Functions or Event Grid fills a PascalCase .NET type instead of arriving with empty + /// fields. Writing is unchanged (PascalCase). + /// + public static readonly JsonSerializerOptions DefaultOptions = new() { PropertyNameCaseInsensitive = true }; + private readonly JsonSerializerOptions _options; public JsonQueueMessageSerializer(JsonSerializerOptions? options = null) { - _options = options ?? new JsonSerializerOptions(); + _options = options ?? DefaultOptions; } - public BinaryData Serialize(T value) => BinaryData.FromObjectAsJson(value, _options); + public BinaryData Serialize(T value) => new(JsonSerializer.SerializeToUtf8Bytes(value, TypeInfo())); public T? Deserialize(BinaryData data) { try { - return JsonSerializer.Deserialize(data, _options); + return JsonSerializer.Deserialize(data.ToMemory().Span, TypeInfo()); } catch (Exception e) { throw new DeserializationException($"Could not deserialize the queue message as {typeof(T).Name}: {e.Message}", e); } } + + // Metadata from the options' resolver: source-generated when the application gave one (UseJsonSerializer(context)), + // which is what trimming and native AOT need. Without one, reflection, unless the application turned reflection-based + // serialization off (as native AOT does), in which case GetTypeInfo explains what to do. + [UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Reflection is used only when the application left it enabled; JsonSerializer.IsReflectionEnabledByDefault is false under native AOT and trimming.")] + [UnconditionalSuppressMessage("AOT", "IL3050", Justification = "As above.")] + private JsonTypeInfo TypeInfo() + { + if (_options.TypeInfoResolver is null && JsonSerializer.IsReflectionEnabledByDefault) + { + // Options without a resolver get the reflection-based default; making them read-only fills it in. + _options.MakeReadOnly(populateMissingResolver: true); + } + + return (JsonTypeInfo)_options.GetTypeInfo(typeof(T)); + } } diff --git a/src/AzureStorage.QueueService/LeaseRenewal.cs b/src/AzureStorage.QueueService/LeaseRenewal.cs index ac7f90f..723678e 100644 --- a/src/AzureStorage.QueueService/LeaseRenewal.cs +++ b/src/AzureStorage.QueueService/LeaseRenewal.cs @@ -16,17 +16,19 @@ internal sealed class LeaseRenewal : IAsyncDisposable private readonly Task _loop; private bool _disposed; - private LeaseRenewal(Func renewAsync, Func isSettled, TimeSpan interval, TimeProvider time, CancellationToken stopAll, Action onFailed) + private LeaseRenewal(Func renewAsync, Func isSettled, TimeSpan interval, TimeProvider time, CancellationToken stopAll, Action onFailed, TimeSpan? maxDuration, Action? onLimitReached) { _stopWaiting = CancellationTokenSource.CreateLinkedTokenSource(stopAll); - _loop = RunAsync(renewAsync, isSettled, interval, time, _stopWaiting.Token, onFailed); + _loop = RunAsync(renewAsync, isSettled, interval, time, _stopWaiting.Token, onFailed, maxDuration, onLimitReached); } /// Renews the lease once. Must not observe a cancellation token: an accepted renewal changes the receipt, so the response must be read. /// True once the message no longer needs renewing. /// The processor-wide signal that ends every renewal, for example at shutdown. - public static LeaseRenewal Start(Func renewAsync, Func isSettled, TimeSpan interval, TimeProvider time, CancellationToken stopAll, Action onFailed) => - new(renewAsync, isSettled, interval, time, stopAll, onFailed); + /// Stop renewing once this long has passed since the start; null for no limit. + /// Called once when stops the renewals. + public static LeaseRenewal Start(Func renewAsync, Func isSettled, TimeSpan interval, TimeProvider time, CancellationToken stopAll, Action onFailed, TimeSpan? maxDuration = null, Action? onLimitReached = null) => + new(renewAsync, isSettled, interval, time, stopAll, onFailed, maxDuration, onLimitReached); /// For lifetime tests only: the linked source whose disposal unregisters from the parent. internal CancellationTokenSource LinkedSource => _stopWaiting; @@ -46,14 +48,22 @@ public async ValueTask DisposeAsync() _stopWaiting.Dispose(); } - private static async Task RunAsync(Func renewAsync, Func isSettled, TimeSpan interval, TimeProvider time, CancellationToken stopWaiting, Action onFailed) + private static async Task RunAsync(Func renewAsync, Func isSettled, TimeSpan interval, TimeProvider time, CancellationToken stopWaiting, Action onFailed, TimeSpan? maxDuration, Action? onLimitReached) { + var started = time.GetTimestamp(); try { while (true) { await Task.Delay(interval, time, stopWaiting); if (isSettled()) return; + if (maxDuration is { } max && time.GetElapsedTime(started) >= max) + { + // The lease lapses within one more interval and the message is delivered again, hung handler or not. + onLimitReached?.Invoke(); + return; + } + await renewAsync(); } } diff --git a/src/AzureStorage.QueueService/LoggerMessageExtensions.cs b/src/AzureStorage.QueueService/LoggerMessageExtensions.cs index 581bda3..856ea61 100644 --- a/src/AzureStorage.QueueService/LoggerMessageExtensions.cs +++ b/src/AzureStorage.QueueService/LoggerMessageExtensions.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; namespace AzureStorage.QueueService; @@ -9,10 +9,10 @@ internal static partial class LoggerMessageExtensions [LoggerMessage(26, LogLevel.Warning, "Queue '{QueueName}': the message handler threw for message {MessageId} after settling it ({Settlement}); the settlement stands.")] public static partial void LogHandlerFailedAfterSettling(this ILogger logger, string? queueName, string messageId, MessageSettlement settlement, Exception exception); - [LoggerMessage(3, LogLevel.Information, "Queue '{QueueName}': processed message {MessageId} successfully; removing it from the queue.")] + [LoggerMessage(3, LogLevel.Debug, "Queue '{QueueName}': processed message {MessageId} successfully; removing it from the queue.")] public static partial void LogMessageProcessed(this ILogger logger, string? queueName, string messageId); - [LoggerMessage(4, LogLevel.Information, "Queue '{QueueName}': processing {MessageCount} message(s).")] + [LoggerMessage(4, LogLevel.Debug, "Queue '{QueueName}': processing {MessageCount} message(s).")] public static partial void LogMessagesReceived(this ILogger logger, string? queueName, int messageCount); [LoggerMessage(5, LogLevel.Error, "Queue '{QueueName}': there was a problem sending the message.")] @@ -21,8 +21,39 @@ internal static partial class LoggerMessageExtensions [LoggerMessage(6, LogLevel.Error, "Queue '{QueueName}': failed to receive messages.")] public static partial void LogReceiveFailed(this ILogger logger, string? queueName, Exception exception); - [LoggerMessage(7, LogLevel.Error, "Queue '{QueueName}': the message handler threw for message {MessageId} (dequeue count {DequeueCount}). The message was not completed and will be delivered again.")] - public static partial void LogHandlerFailed(this ILogger logger, string? queueName, string messageId, long dequeueCount, Exception exception); + // Warning while retries remain, Error on the last attempt: a message that will be retried is not yet a failure. + [LoggerMessage(EventId = 7, Message = "Queue '{QueueName}': the message handler threw for message {MessageId} (dequeue count {DequeueCount}). The message was not completed and will be delivered again after {RetryDelay}.")] + public static partial void LogHandlerFailed(this ILogger logger, LogLevel level, string? queueName, string messageId, long dequeueCount, TimeSpan retryDelay, Exception exception); + + [LoggerMessage(27, LogLevel.Error, "Queue '{QueueName}': the message handler threw for message {MessageId} on its last attempt (dequeue count {DequeueCount}). It will be moved to the poison queue on its next delivery.")] + public static partial void LogHandlerFailedLastAttempt(this ILogger logger, string? queueName, string messageId, long dequeueCount, Exception exception); + + [LoggerMessage(28, LogLevel.Warning, "Queue '{QueueName}': the message handler threw {ExceptionType} for message {MessageId}; the processor's error policy says {Action}.")] + public static partial void LogHandlerFailedPolicy(this ILogger logger, string? queueName, string messageId, string exceptionType, HandlerErrorAction action, Exception exception); + + [LoggerMessage(29, LogLevel.Error, "Processor {Processor}: the OnHandlerError callback threw for message {MessageId}; retrying the message instead.")] + public static partial void LogProcessorErrorPolicyFailed(this ILogger logger, string processor, string messageId, Exception exception); + + [LoggerMessage(30, LogLevel.Warning, "Queue '{QueueName}': could not delete the stored body {Reference} of message {MessageId}. It is orphaned; a lifecycle rule on the container cleans up anything left behind.")] + public static partial void LogLargeMessageDeleteFailed(this ILogger logger, string? queueName, string? messageId, string reference, Exception exception); + + [LoggerMessage(31, LogLevel.Warning, "Processor {Processor}: the handler for message {MessageId} has been running for {Elapsed}, longer than LongRunningHandlerWarning ({Threshold}). It may be hung; its lease is still being renewed{RenewalNote}.")] + public static partial void LogProcessorHandlerLongRunning(this ILogger logger, string processor, string messageId, TimeSpan elapsed, TimeSpan threshold, string renewalNote); + + [LoggerMessage(32, LogLevel.Warning, "Processor {Processor}: stopped renewing the lease on message {MessageId} after MaxRenewalDuration ({MaxRenewalDuration}). It becomes visible to other consumers when the lease lapses, even if this handler is still running.")] + public static partial void LogProcessorRenewalLimitReached(this ILogger logger, string processor, string messageId, TimeSpan maxRenewalDuration); + + [LoggerMessage(33, LogLevel.Information, "Processor {Processor}: paused; no new messages will be received until it is resumed.")] + public static partial void LogProcessorPaused(this ILogger logger, string processor); + + [LoggerMessage(34, LogLevel.Information, "Processor {Processor}: resumed.")] + public static partial void LogProcessorResumed(this ILogger logger, string processor); + + [LoggerMessage(35, LogLevel.Error, "Processor {Processor}: queue '{QueueName}' does not exist; trying again in {Backoff}. Create it, or set CreateIfNotExists on the client.")] + public static partial void LogProcessorQueueNotFound(this ILogger logger, string processor, string queueName, TimeSpan backoff); + + [LoggerMessage(36, LogLevel.Warning, "Queue '{QueueName}': could not read the queue's approximate message count for the depth metric.")] + public static partial void LogDepthReadFailed(this ILogger logger, string queueName, Exception exception); [LoggerMessage(8, LogLevel.Warning, "Queue '{QueueName}': message {MessageId} was handled successfully but could not be deleted. The handler probably ran longer than the visibility timeout ({VisibilityTimeout}), so the message may be delivered again.")] public static partial void LogDeleteFailed(this ILogger logger, string? queueName, string messageId, TimeSpan visibilityTimeout, Exception exception); @@ -72,6 +103,16 @@ internal static partial class LoggerMessageExtensions [LoggerMessage(25, LogLevel.Critical, "Processor {Processor}: a worker failed unexpectedly and will restart in {Backoff}. This is a bug in the library; please report it.")] public static partial void LogProcessorWorkerFailed(this ILogger logger, string processor, TimeSpan backoff, Exception exception); + [LoggerMessage(37, LogLevel.Information, "Processor {Processor}: handler threw for message {MessageId} and the processor's error policy completed it.")] + public static partial void LogProcessorCompletedByPolicy(this ILogger logger, string processor, string messageId); + + private static readonly Func DeliveryScope = + LoggerMessage.DefineScope("Queue {QueueName}, message {MessageId}, dequeue count {DequeueCount}"); + + /// A scope for everything logged while one message is delivered, the handler's own logs included. + public static IDisposable? BeginDeliveryScope(this ILogger logger, string queueName, string messageId, long dequeueCount) => + DeliveryScope(logger, queueName, messageId, dequeueCount); + [LoggerMessage(45, LogLevel.Warning, "{Client}: could not make a private copy of its QueueClientOptions, so the Base64 decoding handler is added to the caller's options while the SDK client is built, then removed. A QueueClient the application builds from the same options object at that moment may pick it up. Logged once.")] public static partial void LogClientOptionsCopyFailed(this ILogger logger, string client, Exception exception); } diff --git a/src/AzureStorage.QueueService/MessageCodec.cs b/src/AzureStorage.QueueService/MessageCodec.cs new file mode 100644 index 0000000..5544cb0 --- /dev/null +++ b/src/AzureStorage.QueueService/MessageCodec.cs @@ -0,0 +1,302 @@ +using System.Diagnostics; +using System.IO.Compression; + +namespace AzureStorage.QueueService; + +/// Names of headers the library itself reads or writes. Any other name is the application's. +public static class QueueMessageHeaders +{ + /// + /// The message's type, which a routed processor uses to choose the handler. Set by , + /// or on every send by . + /// + public const string MessageType = "message-type"; + + /// A conventional name for a correlation id; the library does not read it. + public const string CorrelationId = "correlation-id"; + + /// A conventional name for the body's content type; the library does not read it. + public const string ContentType = "content-type"; +} + +/// A message ready to send: the bytes for the queue, and the stored body to delete if the send fails. +internal readonly record struct EncodedMessage(BinaryData Wire, int SerializedSize, string? BlobReference); + +/// +/// Turns one client's messages into what goes on the queue and back: serialization, the envelope (trace context, +/// headers), compression, the size limit and large bodies kept elsewhere. Shared by the SDK and in-memory queues +/// through , so a test runs exactly the production path. +/// +internal sealed class MessageCodec +{ + /// The service's limit on a message as it goes on the queue. + public const int MaxMessageBytes = 64 * 1024; + + private readonly bool _propagateTraceContext; + private readonly int? _compressionThreshold; + private readonly bool _sendMessageType; + private readonly int? _largeMessageThreshold; + + public MessageCodec( + IQueueMessageSerializer serializer, + bool propagateTraceContext = false, + int? compressionThreshold = null, + bool sendMessageType = false, + ILargeMessageStore? largeMessageStore = null, + int? largeMessageThreshold = null) + { + Serializer = serializer; + _propagateTraceContext = propagateTraceContext; + _compressionThreshold = compressionThreshold; + _sendMessageType = sendMessageType; + LargeMessageStore = largeMessageStore; + _largeMessageThreshold = largeMessageThreshold; + } + + public IQueueMessageSerializer Serializer { get; } + + public ILargeMessageStore? LargeMessageStore { get; } + + /// How big bytes are against the limit once the SDK has encoded them. + public static int WireSize(int bytes, bool base64Encoded) => base64Encoded ? (int)Math.Min(int.MaxValue, (bytes + 2L) / 3 * 4) : bytes; + + // ------------------------------------------------------------------------------------------------------------ + // Sending + // ------------------------------------------------------------------------------------------------------------ + + /// + /// Serializes and wraps it in an envelope when anything has to travel with it: the trace + /// context of (when propagation is on), headers, a compressed body, or a reference to a + /// body kept in the large message store. Throws rather than send what the + /// service would reject. + /// + public async ValueTask EncodeAsync(string queueName, bool base64Encoded, T message, SendOptions? options, Activity? creation, CancellationToken cancellationToken) + { + BinaryData payload = Serializer.Serialize(message); + var serializedSize = payload.ToMemory().Length; + + var envelope = new MessageEnvelope(); + var needsEnvelope = false; + + if (_propagateTraceContext && creation is not null) needsEnvelope |= envelope.SetTraceContext(creation); + + if (Headers(options) is { } headers) + { + envelope.Headers = headers; + needsEnvelope = true; + } + + var body = payload; + // The library's own serializer always writes exactly one JSON value, so there is no need to parse it to find out. + var knownJson = Serializer is JsonQueueMessageSerializer; + if (_compressionThreshold is { } threshold && serializedSize >= threshold && Gzip(payload) is { } compressed) + { + body = compressed; + envelope.Encoding = MessageEnvelope.GzipEncoding; + needsEnvelope = true; + } + + envelope.SetBody(body, knownJson && envelope.Encoding is null); + + string? blob = null; + if (LargeMessageStore is { } store && _largeMessageThreshold is { } large && body.ToMemory().Length >= large) + { + blob = await store.UploadAsync(queueName, body, cancellationToken); + envelope.BlobReference = blob; + needsEnvelope = true; + } + + var wire = needsEnvelope ? envelope.Write() : payload; + var size = WireSize(wire.ToMemory().Length, base64Encoded); + if (size <= MaxMessageBytes) return new EncodedMessage(wire, serializedSize, blob); + + if (LargeMessageStore is { } fallback && blob is null) + { + blob = await fallback.UploadAsync(queueName, body, cancellationToken); + envelope.BlobReference = blob; + wire = envelope.Write(); + size = WireSize(wire.ToMemory().Length, base64Encoded); + if (size <= MaxMessageBytes) return new EncodedMessage(wire, serializedSize, blob); + } + + var detail = blob is not null + ? "Even with the body in the large message store, the headers and trace context are too big." + : $"The serializer wrote {serializedSize:N0} bytes{(needsEnvelope ? ", and the library's envelope adds to that" : "")}{(base64Encoded ? "; Base64 encoding makes it a third bigger" : "")}. Send less, turn on compression with UseCompression(), or keep large bodies in Blob storage with UseBlobStorageForLargeMessages() from the AzureStorage.QueueService.Blobs package."; + throw new QueueMessageTooLargeException(queueName, size, MaxMessageBytes, detail); + } + + private IReadOnlyDictionary? Headers(SendOptions? options) + { + var messageType = options?.MessageType ?? (_sendMessageType ? typeof(T).Name : null); + if (options?.Headers is not { Count: > 0 } given && messageType is null) return null; + + var headers = new Dictionary(StringComparer.Ordinal); + if (options?.Headers is { } source) + { + foreach (var (name, value) in source) + { + if (string.IsNullOrEmpty(name)) throw new ArgumentException("A header name cannot be empty.", nameof(options)); + headers[name] = value ?? throw new ArgumentException($"Header '{name}' has no value.", nameof(options)); + } + } + + if (messageType is not null) headers[QueueMessageHeaders.MessageType] = messageType; + return headers; + } + + // ------------------------------------------------------------------------------------------------------------ + // Receiving + // ------------------------------------------------------------------------------------------------------------ + + /// + /// A received message, not yet deserialized, with the envelope (if any) taken apart. Never throws: one bad message + /// must not fail the batch it arrived in, so what cannot be read comes back with + /// set. Unwrapped whatever this client's own settings, which govern sending only: a receiver reads what any sender + /// put on the queue. + /// + public static QueueMessage Received(TransportMessage received, IQueueMessageOperations operations) + { + var info = received.Info; + var wire = received.Body; + + if (received.Undecodable) + return new QueueMessage(info, wire, default, new DeserializationException($"Message {info.MessageId} is not valid Base64, so the SDK could not decode it."), operations); + + switch (MessageEnvelope.TryRead(wire, out var envelope, out var problem)) + { + case EnvelopeKind.Plain: + return new QueueMessage(info, wire, default, null, operations); + + case EnvelopeKind.Unreadable: + return new QueueMessage(info, wire, default, new DeserializationException($"Message {info.MessageId} is wrapped in the library's envelope, but {problem}."), operations); + } + + var message = new QueueMessage(info, envelope!.Body ?? BinaryData.Empty, default, null, operations); + var lease = message.Lease; + lease.WireBody = wire; + lease.Envelope = envelope; + lease.CreationContext = envelope.CreationContext; + lease.Headers = envelope.Headers ?? info.Headers; + + if (envelope.BlobReference is not null) + { + // Read inside the delivery, after lease renewal has started: a download can be slow. + lease.PendingBlob = envelope.BlobReference; + return message; + } + + if (envelope.Encoding is not null) + { + var (body, error) = Decode(envelope.Body!, envelope.Encoding, info.MessageId); + if (error is not null) message.SetBody(default, error); + else lease.RawBody = body!; + } + + return message; + } + + /// + /// Brings in a body kept in the large message store, if the message has one. Returns the problem, or null; never + /// throws except for cancellation. A message whose body is gone or cannot be read is an unreadable message, to be + /// dead-lettered; its stored body is left alone. + /// + public async ValueTask ResolvePayloadAsync(MessageLease lease, Action? onDeleteFailed, CancellationToken cancellationToken) + { + if (lease.PendingBlob is not { } reference) return null; + + if (LargeMessageStore is not { } store) + return new DeserializationException($"Message {lease.MessageId}'s body is kept outside the queue ({reference}), and this client has no large message store to read it from. Attach the same store the sender uses, for example with UseBlobStorageForLargeMessages()."); + + BinaryData stored; + try + { + stored = await store.DownloadAsync(reference, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + return new DeserializationException($"Message {lease.MessageId}'s body could not be read from the large message store ({reference}): {e.Message}", e); + } + + var (body, error) = lease.Envelope?.Encoding is { } encoding ? Decode(stored, encoding, lease.MessageId) : (stored, null); + if (error is not null) return error; + + lease.RawBody = body!; + lease.PendingBlob = null; + // Deleted once the message is gone from the queue for good; kept through retries and dead-lettering. + lease.AfterComplete = async () => + { + try + { + await store.DeleteAsync(reference, CancellationToken.None); + } + catch (Exception e) + { + onDeleteFailed?.Invoke(reference, e); + } + }; + return null; + } + + /// Fills in or . Never throws. + public void Deserialize(QueueMessage message) + { + if (message.Error is not null) return; // a decoding failure; there is nothing to deserialize + var (body, error) = TryDeserialize(message.RawBody); + message.SetBody(body, error); + } + + public (T? Body, DeserializationException? Error) TryDeserialize(BinaryData raw) => TryDeserialize(Serializer, raw); + + public static (T? Body, DeserializationException? Error) TryDeserialize(IQueueMessageSerializer serializer, BinaryData raw) + { + try + { + return (serializer.Deserialize(raw), null); + } + catch (DeserializationException e) + { + return (default, e); + } + catch (Exception e) + { + // A custom serializer that throws something else still must not take the receive down. + return (default, new DeserializationException($"Could not deserialize the queue message as {typeof(T).Name}: {e.Message}", e)); + } + } + + // ------------------------------------------------------------------------------------------------------------ + // Compression + // ------------------------------------------------------------------------------------------------------------ + + /// The body gzipped, or null when that would not make the message smaller. + private static BinaryData? Gzip(BinaryData payload) + { + var source = payload.ToMemory(); + using var output = new MemoryStream(source.Length / 2 + 64); + using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true)) + gzip.Write(source.Span); + + // Compressed bytes travel Base64-encoded (body64), a third bigger, so compression has to beat that to be worth it. + return output.Length * 4 / 3 < source.Length ? new BinaryData(output.GetBuffer().AsMemory(0, (int)output.Length)) : null; + } + + private static (BinaryData? Body, DeserializationException? Error) Decode(BinaryData encoded, string encoding, string messageId) + { + try + { + using var input = encoded.ToStream(); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var output = new MemoryStream(); + gzip.CopyTo(output); + return (new BinaryData(output.GetBuffer().AsMemory(0, (int)output.Length)), null); + } + catch (Exception e) when (e is InvalidDataException or IOException) + { + return (null, new DeserializationException($"Message {messageId}'s body is marked '{encoding}' but could not be decompressed: {e.Message}", e)); + } + } +} diff --git a/src/AzureStorage.QueueService/MessageEnvelope.cs b/src/AzureStorage.QueueService/MessageEnvelope.cs new file mode 100644 index 0000000..b2c5250 --- /dev/null +++ b/src/AzureStorage.QueueService/MessageEnvelope.cs @@ -0,0 +1,368 @@ +using System.Buffers; +using System.Diagnostics; +using System.Text.Json; + +namespace AzureStorage.QueueService; + +/// What found on the queue. +internal enum EnvelopeKind +{ + /// An ordinary message: the body is the serializer's output as it is. + Plain, + + /// An envelope this version can read. + Envelope, + + /// + /// An envelope this version cannot read: written by a newer version of the library, or damaged. The message is + /// handed over with set so it is dead-lettered intact, and can be replayed once + /// the reader is upgraded, rather than being read as something it is not. + /// + Unreadable, +} + +/// Why and from where a message was dead-lettered; kept in the envelope on the poison queue. +internal sealed record DeadLetterInfo(string? Reason, string? Category, string? MessageId, long DequeueCount, string? Queue, DateTimeOffset? At); + +/// +/// The library's wrapper around a message body. Storage queue messages have no properties or headers, so anything +/// that has to travel with a message (the sender's trace context, application headers, how the body is encoded, +/// where a large body was put, why a message was dead-lettered) travels in the body, inside this JSON object. +/// +/// +/// Wire format, version 1. The marker comes first; every other property is optional, in any order: +/// +/// {"$azqs-envelope":1, +/// "traceparent":"00-<trace id>-<span id>-<flags>", "tracestate":"...", +/// "headers":{"name":"value"}, +/// "encoding":"gzip", +/// "deadLetter":{"reason":"...","category":"handler","messageId":"...","dequeueCount":5,"queue":"orders","at":"2026-10-16T12:00:00+00:00"}, +/// "body":<the serialized message> or "body64":"<Base64>" or "blob":"<where the body is stored>"} +/// +/// +/// Exactly one of body (the serializer's output verbatim, when it is JSON), body64 (any other bytes, +/// Base64-encoded) and blob (a reference to a body kept outside the queue) is present. encoding applies to +/// the bytes of body64 or the blob, never to body. +/// +/// +/// Compatibility rules, so that a message written by a later 3.x is never silently misread by an earlier one: a +/// property this reader does not know is skipped, so later versions can add optional properties without changing the +/// version; anything that changes how the body must be read (a new encoding, a new kind of body) either uses a value +/// this reader rejects or increments the version. A version this reader does not know, or an envelope that breaks the +/// rules above, is : the message gets an error and is dead-lettered unchanged. +/// Detection needs the marker to be the first property; its hyphen means no C# property name can produce it by accident. +/// +/// +internal sealed class MessageEnvelope +{ + public const int Version = 1; + public const string GzipEncoding = "gzip"; + + private static ReadOnlySpan MarkerUtf8 => "$azqs-envelope"u8; + private static ReadOnlySpan TraceParentUtf8 => "traceparent"u8; + private static ReadOnlySpan TraceStateUtf8 => "tracestate"u8; + private static ReadOnlySpan HeadersUtf8 => "headers"u8; + private static ReadOnlySpan EncodingUtf8 => "encoding"u8; + private static ReadOnlySpan DeadLetterUtf8 => "deadLetter"u8; + private static ReadOnlySpan BodyUtf8 => "body"u8; + private static ReadOnlySpan Body64Utf8 => "body64"u8; + private static ReadOnlySpan BlobUtf8 => "blob"u8; + + // The writer nests the payload one level inside the envelope, so reading allows deeper than the check on write + // (64, System.Text.Json's default); an envelope this library wrote can never fail to unwrap on depth. + private static readonly JsonReaderOptions ReadOptions = new() { MaxDepth = 256 }; + + public string? TraceParent { get; set; } + + public string? TraceState { get; set; } + + public IReadOnlyDictionary? Headers { get; set; } + + /// How the bytes of (when not JSON) or the blob are encoded; null for as-is. + public string? Encoding { get; set; } + + public DeadLetterInfo? DeadLetter { get; set; } + + /// The body: JSON when , otherwise bytes. Null when the body is in a blob. + public BinaryData? Body { get; set; } + + public bool BodyIsJson { get; set; } + + /// Where the body is stored when it is not on the queue; see . + public string? BlobReference { get; set; } + + /// The sender's trace context, when there is one and it parses. + public ActivityContext? CreationContext => + TraceParent is not null && ActivityContext.TryParse(TraceParent, TraceState, isRemote: true, out var context) ? context : null; + + /// + /// True when nothing but a body is left: the message can go on the queue as the body alone, the way it would have + /// been sent with no features on. + /// + public bool CarriesOnlyBody => + TraceParent is null && Headers is not { Count: > 0 } && Encoding is null && DeadLetter is null && BlobReference is null && Body is not null; + + public MessageEnvelope Clone() => (MessageEnvelope)MemberwiseClone(); + + /// + /// Sets and from ; returns false when it + /// has no W3C identity to carry (the legacy hierarchical id format). + /// + public bool SetTraceContext(Activity activity) + { + if (activity.IdFormat != ActivityIdFormat.W3C) return false; + + var context = activity.Context; + TraceParent = $"00-{context.TraceId.ToHexString()}-{context.SpanId.ToHexString()}-{(byte)context.TraceFlags:x2}"; + TraceState = string.IsNullOrEmpty(activity.TraceStateString) ? null : activity.TraceStateString; + return true; + } + + /// Sets the body, as JSON when it is exactly one JSON value (or the caller already knows it is), otherwise as bytes. + public void SetBody(BinaryData body, bool knownJson = false) + { + Body = body; + BodyIsJson = knownJson || IsSingleJsonValue(body.ToMemory().Span); + BlobReference = null; + } + + public BinaryData Write() + { + var body = Body?.ToMemory() ?? ReadOnlyMemory.Empty; + var buffer = new ArrayBufferWriter(body.Length + 256); + using (var writer = new Utf8JsonWriter(buffer)) + { + writer.WriteStartObject(); + writer.WriteNumber(MarkerUtf8, Version); + if (TraceParent is not null) writer.WriteString(TraceParentUtf8, TraceParent); + if (TraceState is not null) writer.WriteString(TraceStateUtf8, TraceState); + + if (Headers is { Count: > 0 } headers) + { + writer.WriteStartObject(HeadersUtf8); + foreach (var (name, value) in headers) writer.WriteString(name, value); + writer.WriteEndObject(); + } + + if (Encoding is not null) writer.WriteString(EncodingUtf8, Encoding); + + if (DeadLetter is { } deadLetter) + { + writer.WriteStartObject(DeadLetterUtf8); + if (deadLetter.Reason is not null) writer.WriteString("reason"u8, deadLetter.Reason); + if (deadLetter.Category is not null) writer.WriteString("category"u8, deadLetter.Category); + if (deadLetter.MessageId is not null) writer.WriteString("messageId"u8, deadLetter.MessageId); + writer.WriteNumber("dequeueCount"u8, deadLetter.DequeueCount); + if (deadLetter.Queue is not null) writer.WriteString("queue"u8, deadLetter.Queue); + if (deadLetter.At is { } at) writer.WriteString("at"u8, at); + writer.WriteEndObject(); + } + + if (BlobReference is not null) + { + writer.WriteString(BlobUtf8, BlobReference); + } + else if (BodyIsJson) + { + writer.WritePropertyName(BodyUtf8); + writer.WriteRawValue(body.Span, skipInputValidation: true); + } + else + { + writer.WriteBase64String(Body64Utf8, body.Span); + } + + writer.WriteEndObject(); + } + + return new BinaryData(buffer.WrittenMemory); + } + + /// + /// Looks at a message as it came off the queue. Never throws: it runs inside every receive, so one malformed message + /// must not fail the batch it arrived in. says what is wrong with an unreadable envelope. + /// + public static EnvelopeKind TryRead(BinaryData wire, out MessageEnvelope? envelope, out string? problem) + { + envelope = null; + problem = null; + + var memory = wire.ToMemory(); + if (!StartsWithObject(memory.Span)) return EnvelopeKind.Plain; // the cheap test first: most plain payloads stop here + + var reader = new Utf8JsonReader(memory.Span, ReadOptions); + try + { + // Up to and including the marker, anything unexpected means an ordinary message that happens to start with '{'. + if (!reader.Read() || reader.TokenType != JsonTokenType.StartObject) return EnvelopeKind.Plain; + if (!reader.Read() || reader.TokenType != JsonTokenType.PropertyName || !reader.ValueTextEquals(MarkerUtf8)) return EnvelopeKind.Plain; + } + catch (JsonException) + { + return EnvelopeKind.Plain; + } + + // From here on it is ours, so anything wrong is an unreadable envelope, never a plain message: read as a plain + // message, the envelope's JSON would deserialize into an object with empty fields and be completed. + try + { + if (!reader.Read() || reader.TokenType != JsonTokenType.Number || !reader.TryGetInt32(out var version) || version < 1) + return Unreadable("its version marker is not a positive whole number", out problem); + if (version > Version) + return Unreadable($"it was written in envelope version {version} by a newer version of AzureStorage.QueueService, and this one reads version {Version}. Upgrade the application, then replay the message from the poison queue", out problem); + + var result = new MessageEnvelope(); + if (!TryReadProperties(ref reader, memory, result, out problem)) return EnvelopeKind.Unreadable; + + envelope = result; + return EnvelopeKind.Envelope; + } + catch (Exception e) when (e is JsonException or InvalidOperationException or FormatException) + { + // Invalid JSON (JsonException), a string that is not valid UTF-8 or a value of the wrong type + // (InvalidOperationException), or a malformed date (FormatException). + return Unreadable($"it is not valid: {e.Message}", out problem); + } + } + + private static EnvelopeKind Unreadable(string why, out string? problem) + { + problem = why; + return EnvelopeKind.Unreadable; + } + + private static bool TryReadProperties(ref Utf8JsonReader reader, ReadOnlyMemory memory, MessageEnvelope result, out string? problem) + { + problem = null; + var seen = new HashSet(StringComparer.Ordinal); + var bodies = 0; + + while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) + { + var name = reader.GetString()!; + if (!seen.Add(name)) return Fail($"it has the property '{name}' twice", out problem); + + if (reader.ValueTextEquals(TraceParentUtf8)) result.TraceParent = ReadString(ref reader, name); + else if (reader.ValueTextEquals(TraceStateUtf8)) result.TraceState = ReadString(ref reader, name); + else if (reader.ValueTextEquals(HeadersUtf8)) result.Headers = ReadHeaders(ref reader); + else if (reader.ValueTextEquals(EncodingUtf8)) result.Encoding = ReadString(ref reader, name); + else if (reader.ValueTextEquals(DeadLetterUtf8)) result.DeadLetter = ReadDeadLetter(ref reader); + else if (reader.ValueTextEquals(BodyUtf8)) + { + bodies++; + reader.Read(); + var start = (int)reader.TokenStartIndex; + reader.Skip(); + result.Body = new BinaryData(memory[start..(int)reader.BytesConsumed]); + result.BodyIsJson = true; + } + else if (reader.ValueTextEquals(Body64Utf8)) + { + bodies++; + reader.Read(); + if (reader.TokenType != JsonTokenType.String || !reader.TryGetBytesFromBase64(out var bytes)) return Fail("its body64 is not a Base64 string", out problem); + result.Body = new BinaryData(bytes); + } + else if (reader.ValueTextEquals(BlobUtf8)) + { + bodies++; + result.BlobReference = ReadString(ref reader, name); + if (string.IsNullOrEmpty(result.BlobReference)) return Fail("its blob reference is empty", out problem); + } + else + { + // Added by a later version as optional: skip it. + reader.Read(); + reader.Skip(); + } + } + + if (reader.TokenType != JsonTokenType.EndObject) return Fail("it is not a complete JSON object", out problem); + if (reader.Read()) return Fail("something follows the JSON object", out problem); + if (bodies != 1) return Fail(bodies == 0 ? "it has no body" : "it has more than one body", out problem); + + if (result.Encoding is { } encoding) + { + if (result.BodyIsJson) return Fail("it has an encoding but a JSON body", out problem); + if (encoding != GzipEncoding) return Fail($"its body is encoded as '{encoding}', which this version cannot decode. Upgrade the application, then replay the message from the poison queue", out problem); + } + + return true; + } + + private static bool Fail(string why, out string? problem) + { + problem = why; + return false; + } + + private static string? ReadString(ref Utf8JsonReader reader, string name) + { + reader.Read(); + if (reader.TokenType == JsonTokenType.Null) return null; + if (reader.TokenType != JsonTokenType.String) throw new InvalidOperationException($"'{name}' must be a string."); + return reader.GetString(); + } + + private static Dictionary ReadHeaders(ref Utf8JsonReader reader) + { + reader.Read(); + if (reader.TokenType != JsonTokenType.StartObject) throw new InvalidOperationException("'headers' must be an object."); + + var headers = new Dictionary(StringComparer.Ordinal); + while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) + { + var name = reader.GetString()!; + headers[name] = ReadString(ref reader, name) ?? throw new InvalidOperationException($"Header '{name}' must be a string."); + } + + return headers; + } + + private static DeadLetterInfo ReadDeadLetter(ref Utf8JsonReader reader) + { + reader.Read(); + if (reader.TokenType != JsonTokenType.StartObject) throw new InvalidOperationException("'deadLetter' must be an object."); + + string? reason = null, category = null, messageId = null, queue = null; + long dequeueCount = 0; + DateTimeOffset? at = null; + while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) + { + if (reader.ValueTextEquals("reason"u8)) reason = ReadString(ref reader, "reason"); + else if (reader.ValueTextEquals("category"u8)) category = ReadString(ref reader, "category"); + else if (reader.ValueTextEquals("messageId"u8)) messageId = ReadString(ref reader, "messageId"); + else if (reader.ValueTextEquals("queue"u8)) queue = ReadString(ref reader, "queue"); + else if (reader.ValueTextEquals("dequeueCount"u8)) { reader.Read(); dequeueCount = reader.GetInt64(); } + else if (reader.ValueTextEquals("at"u8)) { reader.Read(); at = reader.GetDateTimeOffset(); } + else { reader.Read(); reader.Skip(); } + } + + return new DeadLetterInfo(reason, category, messageId, dequeueCount, queue, at); + } + + private static bool StartsWithObject(ReadOnlySpan bytes) + { + foreach (var b in bytes) + { + if (b is (byte)' ' or (byte)'\t' or (byte)'\r' or (byte)'\n') continue; + return b == (byte)'{'; + } + return false; + } + + private static bool IsSingleJsonValue(ReadOnlySpan bytes) + { + try + { + var reader = new Utf8JsonReader(bytes); + if (!reader.Read()) return false; + reader.Skip(); + return !reader.Read(); + } + catch (JsonException) + { + return false; + } + } +} diff --git a/src/AzureStorage.QueueService/PeekedQueueMessage.cs b/src/AzureStorage.QueueService/PeekedQueueMessage.cs index 3567a4a..963a5e4 100644 --- a/src/AzureStorage.QueueService/PeekedQueueMessage.cs +++ b/src/AzureStorage.QueueService/PeekedQueueMessage.cs @@ -3,8 +3,12 @@ namespace AzureStorage.QueueService; /// /// A message looked at without receiving it. It has no pop receipt, so there is nothing to complete or abandon. /// +/// The type the body is read as. public sealed class PeekedQueueMessage { + private static readonly IReadOnlyDictionary NoHeaders = new Dictionary(); + + /// Public so a test can build one. public PeekedQueueMessage(string messageId, long dequeueCount, DateTimeOffset? insertedOn, DateTimeOffset? expiresOn, BinaryData rawBody, T? body, DeserializationException? error) { MessageId = messageId; @@ -16,17 +20,33 @@ public PeekedQueueMessage(string messageId, long dequeueCount, DateTimeOffset? i Error = error; } + /// The service's id for the message. public string MessageId { get; } + /// How many times the message has been received. public long DequeueCount { get; } + /// When the message was put on the queue. public DateTimeOffset? InsertedOn { get; } + /// When the service discards the message. public DateTimeOffset? ExpiresOn { get; } + /// The serialized message, unwrapped from the library's envelope if it was in one. public BinaryData RawBody { get; } + /// The deserialized body; null when it could not be read (see ) or was JSON null. public T? Body { get; } + /// Set when the body could not be decoded or deserialized. public DeserializationException? Error { get; } + + /// Headers the sender attached; empty when there were none. + public IReadOnlyDictionary Headers + { + get => _headers ?? NoHeaders; + init => _headers = value; + } + + private readonly IReadOnlyDictionary? _headers; } diff --git a/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs b/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs index e410b67..db27ee0 100644 --- a/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs +++ b/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs @@ -1,6 +1,7 @@ using System.Text.Json; using Azure.Messaging; using AzureStorage.QueueService.Telemetry; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; @@ -9,6 +10,7 @@ namespace AzureStorage.QueueService; +/// What can be attached to one registered client: settings, a serializer, middleware and processors. public static class QueueClientBuilderExtensions { /// @@ -33,7 +35,9 @@ public static IQueueClientBuilder UseSerializer(this IQueueClientBuilder builder } /// - /// Serializes this client's messages with System.Text.Json using . + /// Serializes this client's messages with System.Text.Json using . For trimmed or native + /// AOT applications, give the options a source-generated TypeInfoResolver, or use the + /// overload. /// public static IQueueClientBuilder UseJsonSerializer(this IQueueClientBuilder builder, JsonSerializerOptions options) { @@ -41,12 +45,87 @@ public static IQueueClientBuilder UseJsonSerializer(this IQueueClientBuilder bui return builder.UseSerializer(new JsonQueueMessageSerializer(options)); } + /// + /// Serializes this client's messages with System.Text.Json using source-generated metadata, for example + /// UseJsonSerializer(MyJsonContext.Default), which is what a trimmed or native AOT application needs. Property + /// names are read case-insensitively, as with the default serializer. + /// + public static IQueueClientBuilder UseJsonSerializer(this IQueueClientBuilder builder, System.Text.Json.Serialization.Metadata.IJsonTypeInfoResolver typeInfoResolver) + { + ArgumentNullException.ThrowIfNull(typeInfoResolver); + return builder.UseSerializer(new JsonQueueMessageSerializer(new JsonSerializerOptions(JsonQueueMessageSerializer.DefaultOptions) { TypeInfoResolver = typeInfoResolver })); + } + + /// + /// Gzips message bodies of at least bytes before sending (see + /// ). Every receiver of the queue must be on 3.1 or later. + /// + public static IQueueClientBuilder UseCompression(this IQueueClientBuilder builder, int thresholdBytes = 1024) + { + ArgumentOutOfRangeException.ThrowIfNegative(thresholdBytes); + return builder.Configure(settings => settings.CompressionThreshold = thresholdBytes); + } + + /// + /// Keeps bodies too big for a queue message in , leaving a reference on the queue (see + /// ). With , bodies at least that big go to the store + /// even when they would fit. + /// + public static IQueueClientBuilder UseLargeMessageStore(this IQueueClientBuilder builder, ILargeMessageStore store, int? thresholdBytes = null) + { + ArgumentNullException.ThrowIfNull(store); + return builder.Configure(settings => + { + settings.LargeMessageStore = store; + settings.LargeMessageThreshold = thresholdBytes; + }); + } + + /// + /// As , with the store built from the + /// application's services the first time the client is used. + /// + public static IQueueClientBuilder UseLargeMessageStore(this IQueueClientBuilder builder, Func createStore, int? thresholdBytes = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(createStore); + + builder.Services.AddKeyedSingleton(QueueClientFactory.LargeMessageStoreKey(builder.Name), (provider, _) => createStore(provider)); + return builder.Configure(settings => settings.LargeMessageThreshold = thresholdBytes); + } + + /// + /// Runs around every message this client's processors deliver, resolved from the + /// message's scope (registered as scoped unless you registered it yourself). See . + /// + public static IQueueClientBuilder UseMiddleware(this IQueueClientBuilder builder) + where TMiddleware : class, IQueueMessageMiddleware + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Services.TryAddScoped(); + builder.Services.Configure(builder.Name, o => o.Middleware.Add(static provider => provider.GetRequiredService())); + return builder; + } + + /// Runs around every message this client's processors deliver. See . + public static IQueueClientBuilder Use(this IQueueClientBuilder builder, Func middleware) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(middleware); + + var instance = new DelegateMiddleware(middleware); + builder.Services.Configure(builder.Name, o => o.Middleware.Add(_ => instance)); + return builder; + } + /// /// Runs a hosted processor for this client: the library receives messages of type /// and hands each one to a new resolved from its own dependency-injection scope. /// The processor owns the polling loop, backoff, visibility renewal, retries, the poison queue, cancellation and /// shutdown; see for the knobs. A processor whose client is misconfigured fails - /// host startup on purpose; runtime failures never stop the host. + /// host startup on purpose; runtime failures never stop the host. Registering the same handler on the same client + /// again adds to its options; it does not start a second processor. /// public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, Action? configure = null) where THandler : class, IQueueMessageHandler @@ -54,14 +133,55 @@ public static IQueueClientBuilder AddProcessor(this IQueueCl ArgumentNullException.ThrowIfNull(builder); builder.Services.TryAddScoped(); - return builder.AddHostedProcessor(typeof(THandler).Name, configure, createHandler: null); + return builder.AddHostedProcessor(OptionsName(builder), configure, new ProcessorHooks(HandlerName: typeof(THandler).Name)); + } + + /// + /// As , with the + /// options bound from (for example configuration.GetSection("Queues:Orders:Processor")) + /// and then , so MaxConcurrentMessages and the rest can differ by environment. + /// + public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, IConfiguration configuration, Action? configure = null) + where THandler : class, IQueueMessageHandler + { + ArgumentNullException.ThrowIfNull(configuration); + return builder.AddProcessor(options => + { + configuration.Bind(options); + configure?.Invoke(options); + }); + } + + /// + /// Runs a hosted processor whose handler is a delegate, with its parameters filled per message: a parameter of type + /// gets the body; or the + /// message; the shutdown token; anything else is resolved from the message's + /// dependency-injection scope ([FromKeyedServices] works, and a parameter with a default value is optional). + /// It must return , or nothing. Everything else is + /// . + /// + /// + /// queues.AddClient("orders", config.GetSection("Queues:Orders")) + /// .AddProcessor<Order>(async (Order order, OrdersDb db, CancellationToken ct) => await db.Fulfil(order, ct)); + /// + public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, Delegate handler, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(handler); + + var binding = new DelegateHandlerBinding(handler); + // Each delegate is its own processor, so each gets its own options name. + var optionsName = $"{builder.Name}|delegate:{ServiceCollectionExtensions.GetOrAddRegistry(builder.Services).NextProcessorNumber()}|{typeof(TMessage).FullName}"; + return builder.AddHostedProcessor(optionsName, configure, new ProcessorHooks>( + CreateHandler: provider => new DelegateHandler(binding, provider), + HandlerName: binding.Name)); } /// /// Runs a hosted processor for a queue of CloudEvents, such as an Event Grid subscription that delivers to a /// storage queue with the CloudEvents schema. Each message is read as a , its data is read /// as with this client's serializer, and both go to a new - /// resolved from its own dependency-injection scope. Everything else is : + /// resolved from its own dependency-injection scope. Everything else is : /// the same options, settlement, retries, poison queue, lease renewal and shutdown. An event whose data cannot be /// read as counts as an unreadable body (see /// ). @@ -71,58 +191,137 @@ public static IQueueClientBuilder AddCloudEventProcessor(this I { ArgumentNullException.ThrowIfNull(builder); - var clientName = builder.Name; - builder.Services.TryAddScoped(); - return builder.AddHostedProcessor>( - typeof(THandler).Name, - configure, - createHandler: provider => new CloudEventHandlerAdapter(provider.GetRequiredService()), - createAfterDeserialize: provider => - { - // The serializer the client itself uses, so the data is read with the same naming policy as everything - // else on this queue. Looked up on first use rather than now: by then the processor's StartAsync has - // built and validated the client, so a misconfigured client fails startup there, the way it always has. - var serializer = new Lazy(() => - provider.GetRequiredService>().Get(clientName).Serializer - ?? provider.GetRequiredService()); - return message => CloudEventHandlerAdapter.ReadData(message, serializer.Value); - }); + var serializer = ClientSerializer(builder.Name); + return builder.AddHostedProcessor(OptionsName>(builder), configure, provider => new ProcessorHooks>( + CreateHandler: scope => new CloudEventHandlerAdapter(scope.GetRequiredService()), + HandlerName: typeof(THandler).Name, + AfterDeserialize: ReadWithClientSerializer(provider, serializer, CloudEventHandlerAdapter.ReadData))); + } + + /// As , with the options bound from first. + public static IQueueClientBuilder AddCloudEventProcessor(this IQueueClientBuilder builder, IConfiguration configuration, Action? configure = null) + where THandler : class, ICloudEventHandler + { + ArgumentNullException.ThrowIfNull(configuration); + return builder.AddCloudEventProcessor(options => + { + configuration.Bind(options); + configure?.Invoke(options); + }); + } + + /// + /// Runs one hosted processor for a queue that carries several message types, choosing each message's handler by its + /// header (see ). A message with no type, + /// or a type nothing is routed for, is dead-lettered. Two ordinary processors on one queue would take each other's + /// messages and fail to read them; this is the way to share a queue. + /// + public static IQueueClientBuilder AddRoutedProcessor(this IQueueClientBuilder builder, Action routes, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(routes); + + var table = new QueueMessageRoutes(builder.Services); + routes(table); + if (table.Routes.Count == 0) throw new InvalidOperationException($"{Describe(builder)}: AddRoutedProcessor needs at least one route."); + + var serializer = ClientSerializer(builder.Name); + return builder.AddHostedProcessor(OptionsName(builder), configure, provider => new ProcessorHooks( + CreateHandler: scope => new RoutingHandler(scope), + HandlerName: "routed processor", + ReadBody: ReadWithClientSerializer(provider, serializer, (message, s) => RoutingHandler.Read(message, table.Routes, s)))); + } + + /// + /// Runs one hosted processor for a queue of CloudEvents of several types, choosing each event's handler by its + /// type (see ). An event of a type nothing is routed for is dead-lettered. + /// + public static IQueueClientBuilder AddCloudEventRouter(this IQueueClientBuilder builder, Action routes, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(routes); + + var table = new CloudEventRoutes(builder.Services); + routes(table); + if (table.Routes.Count == 0) throw new InvalidOperationException($"{Describe(builder)}: AddCloudEventRouter needs at least one route."); + + var serializer = ClientSerializer(builder.Name); + return builder.AddHostedProcessor(OptionsName(builder), configure, provider => new ProcessorHooks( + CreateHandler: scope => new CloudEventRoutingHandler(scope), + HandlerName: "CloudEvent router", + AfterDeserialize: ReadWithClientSerializer(provider, serializer, (message, s) => CloudEventRoutingHandler.Read(message, table.Routes, s)))); } + private static string Describe(IQueueClientBuilder builder) => builder.Name.Length == 0 ? "The default queue client" : $"Queue client '{builder.Name}'"; + // Options are named per client + handler so the same handler type can serve two queues with different settings. // A CloudEvent processor is named by its adapter type, so it never shares options with an AddProcessor of the same class. private static string OptionsName(IQueueClientBuilder builder) => $"{builder.Name}|{typeof(THandler).FullName}"; - private static IQueueClientBuilder AddHostedProcessor( - this IQueueClientBuilder builder, - string handlerName, - Action? configure, - Func? createHandler, - Func>>? createAfterDeserialize = null) + // The serializer the client itself uses, so a wrapper reads with the same naming policy as everything else on this + // queue. Looked up on first use rather than now: by then the processor's StartAsync has built and validated the + // client, so a misconfigured client fails startup there, the way it always has. + private static Func ClientSerializer(string clientName) => provider => + provider.GetRequiredService>().Get(clientName).Serializer + ?? provider.GetRequiredService(); + + private static Action> ReadWithClientSerializer(IServiceProvider provider, Func serializer, Action, IQueueMessageSerializer> read) + { + var lazy = new Lazy(() => serializer(provider)); + return message => read(message, lazy.Value); + } + + private static IQueueClientBuilder AddHostedProcessor(this IQueueClientBuilder builder, string optionsName, Action? configure, ProcessorHooks hooks) + where THandler : class, IQueueMessageHandler => + builder.AddHostedProcessor(optionsName, configure, _ => hooks); + + private static IQueueClientBuilder AddHostedProcessor(this IQueueClientBuilder builder, string optionsName, Action? configure, Func> createHooks) where THandler : class, IQueueMessageHandler { - var optionsName = OptionsName(builder); var clientName = builder.Name; - builder.Services.AddOptions(optionsName).Configure(configure ?? (_ => { })); + // The same handler on the same client again: its options are merged above, and the one processor runs with them. + if (!ServiceCollectionExtensions.GetOrAddRegistry(builder.Services).AddProcessor(optionsName)) return builder; + // Not AddHostedService: that de-duplicates by implementation type, and two processors of the same type for // different queues must both run. - builder.Services.AddSingleton(provider => new QueueProcessor( - clientName, - optionsName, - provider.GetRequiredService(), - provider.GetRequiredService(), - provider.GetRequiredService>(), - provider.GetRequiredService(), - provider.GetRequiredService(), - provider.GetService() ?? TimeProvider.System, - provider.GetService(), - createHandler, - handlerName, - createAfterDeserialize?.Invoke(provider))); + builder.Services.AddSingleton(provider => + { + var middleware = provider.GetRequiredService>().Get(clientName).Middleware.ToList(); + return new QueueProcessor( + clientName, + optionsName, + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetService() ?? TimeProvider.System, + provider.GetService(), + createHooks(provider) with { Middleware = middleware }, + provider.GetService()); + }); return builder; } } + +/// Poison queues from the factory, for code that does not inject . +public static class QueueClientFactoryExtensions +{ + /// The poison queue of the client registered as . + /// No client with that name was registered, or its settings are incomplete. + public static IDeadLetterQueue GetDeadLetterQueue(this IQueueClientFactory factory, string clientName) + { + ArgumentNullException.ThrowIfNull(clientName); + return factory is QueueClientFactory library + ? library.GetDeadLetterQueue(clientName) + : throw new NotSupportedException("Dead-letter queues come from the library's own IQueueClientFactory."); + } + + /// The poison queue of the default client. + /// No default client was registered, or its settings are incomplete. + public static IDeadLetterQueue GetDeadLetterQueue(this IQueueClientFactory factory) => factory.GetDeadLetterQueue(string.Empty); +} diff --git a/src/AzureStorage.QueueService/QueueClientFactory.cs b/src/AzureStorage.QueueService/QueueClientFactory.cs index cac71bf..703965c 100644 --- a/src/AzureStorage.QueueService/QueueClientFactory.cs +++ b/src/AzureStorage.QueueService/QueueClientFactory.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Azure.Storage.Queues; using AzureStorage.QueueService.Telemetry; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -9,7 +10,7 @@ namespace AzureStorage.QueueService; internal sealed class QueueClientFactory : IQueueClientFactory { // Lazy so two threads asking for the same client for the first time get one instance, not two. - private readonly ConcurrentDictionary> _queues = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary> _queues = new(StringComparer.Ordinal); private readonly QueueClientSettingsRegistry _registry; private readonly IOptionsFactory _settings; private readonly ISdkQueueClientFactory _sdkClients; @@ -18,8 +19,11 @@ internal sealed class QueueClientFactory : IQueueClientFactory private readonly QueueServiceTelemetrySettings _telemetry; private readonly InMemoryQueueStore? _inMemory; private readonly QueueServiceMetrics _metrics; + private readonly IServiceProvider? _services; + private readonly TimeProvider _time; /// Registered only by AddInMemoryQueues; when present, every client is an in-memory queue. + /// For a large message store registered with UseLargeMessageStore(provider => ...). public QueueClientFactory( QueueClientSettingsRegistry registry, IOptionsFactory settings, @@ -28,7 +32,9 @@ public QueueClientFactory( IQueueMessageSerializer serializer, IOptions telemetrySettingsOptions, QueueServiceMetrics metrics, - InMemoryQueueStore? inMemory = null) + InMemoryQueueStore? inMemory = null, + IServiceProvider? services = null, + TimeProvider? time = null) { _registry = registry; _settings = settings; @@ -38,12 +44,17 @@ public QueueClientFactory( _telemetry = telemetrySettingsOptions.Value; _inMemory = inMemory; _metrics = metrics; + _services = services; + _time = time ?? TimeProvider.System; } public IStorageQueue GetQueue(string clientName) { ArgumentNullException.ThrowIfNull(clientName); + // The default client asked for by its registered name, the empty string, as the processor controller names it. + if (clientName.Length == 0) return GetQueue(); + if (!_registry.Contains(clientName)) { var registered = _registry.ClientNames.Count == 0 @@ -62,56 +73,78 @@ public IStorageQueue GetQueue() { if (!_registry.HasDefaultClient) { + var named = _registry.ClientNames.Count == 0 ? "" : $" Registered named clients: {string.Join(", ", _registry.ClientNames.Select(name => $"'{name}'"))}; inject one with [FromKeyedServices(\"name\")]."; throw new InvalidOperationException( "No default queue client has been registered. Register one with " + - "services.AddAzureStorageQueueClient(x => x.AddDefaultClient(...)), or ask for a named client by name."); + "services.AddAzureStorageQueueClient(x => x.AddDefaultClient(...)), or ask for a named client by name." + named); } return GetOrCreate(Options.DefaultName); } - private IStorageQueue GetOrCreate(string clientName) => - _queues.GetOrAdd(clientName, name => new Lazy(() => Create(name), LazyThreadSafetyMode.ExecutionAndPublication)).Value; + /// The poison queue of the client registered as ; empty for the default client. + internal IDeadLetterQueue GetDeadLetterQueue(string clientName) => + ((StorageQueue)(clientName.Length == 0 ? GetQueue() : GetQueue(clientName))).DeadLetterQueue; + + private StorageQueue GetOrCreate(string clientName) => + _queues.GetOrAdd(clientName, name => new Lazy(() => Create(name), LazyThreadSafetyMode.ExecutionAndPublication)).Value; - private IStorageQueue Create(string clientName) + private StorageQueue Create(string clientName) { // A snapshot: the settings a queue was built with are the settings it keeps. var settings = _settings.Create(clientName); var description = clientName == Options.DefaultName ? "The default queue client" : $"Queue client '{clientName}'"; + var base64 = settings.ClientOptions?.MessageEncoding == QueueMessageEncoding.Base64; + + var codec = new MessageCodec( + settings.Serializer ?? _serializer, + settings.PropagateTraceContext, + settings.CompressionThreshold, + settings.SendMessageType, + settings.LargeMessageStore ?? _services?.GetKeyedService(LargeMessageStoreKey(clientName)), + settings.LargeMessageThreshold); if (_inMemory is not null) { // The check ValidateOnStart runs in this mode: the policy without the connection. if (settings.ValidateInMemory() is { } problem) throw new InvalidOperationException($"{description} {problem}"); - return _inMemory.CreateClient(settings, settings.Serializer ?? _serializer, _loggerFactory); + + var queueName = settings.InMemoryQueueName!; + var poisonQueueName = settings.InMemoryPoisonQueueName!; + return new StorageQueue( + new InMemoryQueueTransport(_inMemory.GetQueue(queueName), base64), + () => new InMemoryQueueTransport(_inMemory.GetQueue(poisonQueueName), base64), + codec, + _loggerFactory, + _telemetry, + _metrics, + QueueOptions(settings, _inMemory.Time)); } // The same check ValidateOnStart runs, so first use can never fail for something startup accepted. if (settings.Validate() is { } error) throw new InvalidOperationException($"{description} {error}"); var sdkClient = _sdkClients.CreateClient(description, settings); - var poisonQueueName = settings.EffectivePoisonQueueName!; + var poisonName = settings.EffectivePoisonQueueName!; - QueueClient PoisonQueue() => _sdkClients.CreateClient($"{description} (poison queue '{poisonQueueName}')", ForSiblingQueue(settings, poisonQueueName)); + IQueueTransport PoisonQueue() => new SdkQueueTransport( + _sdkClients.CreateClient($"{description} (poison queue '{poisonName}')", ForSiblingQueue(settings, poisonName)), base64); - return new StorageQueue( - sdkClient, - settings.Serializer ?? _serializer, - _loggerFactory, - _telemetry, - _metrics, - settings.VisibilityTimeout, - settings.CreateIfNotExists, - PoisonQueue, - settings.PropagateTraceContext); + return new StorageQueue(new SdkQueueTransport(sdkClient, base64), PoisonQueue, codec, _loggerFactory, _telemetry, _metrics, QueueOptions(settings, _time)); } + private static StorageQueueOptions QueueOptions(QueueClientSettings settings, TimeProvider time) => + new(settings.VisibilityTimeout, settings.CreateIfNotExists, settings.PoisonTimeToLive, time); + + /// The DI key a client's large message store is registered under by UseLargeMessageStore(provider => ...). + internal static string LargeMessageStoreKey(string clientName) => $"AzureStorage.QueueService.LargeMessageStore:{clientName}"; + // The same account and credentials, pointed at another queue. A destination only: no poison policy of its own. internal static QueueClientSettings ForSiblingQueue(QueueClientSettings settings, string queueName) => new() { ConnectionString = settings.ConnectionString, QueueName = settings.UsesConnectionString ? queueName : null, - EndpointUri = settings.EndpointUri is null ? null : SiblingQueueUri(settings.EndpointUri, queueName), + EndpointUri = settings.QueueUri is { } queueUri ? SiblingQueueUri(queueUri, queueName) : null, TokenCredential = settings.TokenCredential, ClientOptions = settings.ClientOptions, }; diff --git a/src/AzureStorage.QueueService/QueueClientSettings.cs b/src/AzureStorage.QueueService/QueueClientSettings.cs index 6e26eed..84977e8 100644 --- a/src/AzureStorage.QueueService/QueueClientSettings.cs +++ b/src/AzureStorage.QueueService/QueueClientSettings.cs @@ -1,10 +1,12 @@ -using System.Diagnostics.CodeAnalysis; using Azure.Core; using Azure.Storage.Queues; namespace AzureStorage.QueueService; -[ExcludeFromCodeCoverage] +/// +/// One client's settings: how to connect, which queue, and how messages are sent and dead-lettered. Bound from +/// configuration with AddClient(name, IConfiguration) or set in code with AddClient(name, settings => ...). +/// public class QueueClientSettings { /// @@ -13,6 +15,10 @@ public class QueueClientSettings /// public string? ConnectionString { get; set; } + /// + /// The queue. Needed with , and with an that is the storage + /// account's endpoint rather than the queue's. + /// public string? QueueName { get; set; } /// @@ -21,10 +27,13 @@ public class QueueClientSettings public bool CreateIfNotExists { get; set; } /// - /// The full queue URI, including the queue name. Use this together with . + /// The queue's URI (https://account.queue.core.windows.net/orders), or the storage account's queue endpoint + /// (https://account.queue.core.windows.net/) together with . Use it with + /// . /// public Uri? EndpointUri { get; set; } + /// The identity to connect with when using ; see also UseTokenCredential. public TokenCredential? TokenCredential { get; set; } /// @@ -65,21 +74,77 @@ public class QueueClientSettings /// public bool PropagateTraceContext { get; set; } + /// + /// How long a dead-lettered message stays on the poison queue before the service discards it; null (the default) or + /// keeps it until someone deletes or replays it. Before 3.1 poison messages + /// took the service default of 7 days and could disappear unseen. + /// + public TimeSpan? PoisonTimeToLive { get; set; } + + /// + /// Gzip message bodies of at least this many bytes (as the serializer writes them) before sending; null (the + /// default) never compresses. A compressed message travels in the library's envelope, so the same caution as + /// applies: every receiver of the queue must be on 3.1 or later. Compression is + /// skipped for a message it would not make smaller. Set with UseCompression() on the client builder. + /// + public int? CompressionThreshold { get; set; } + + /// + /// Send each message's type (typeof(T).Name) as the header, for a + /// routed processor on the receiving side. Like any header, it puts the message in the library's envelope. + /// + public bool SendMessageType { get; set; } + + /// + /// Where bodies too big for a queue message go; set with UseLargeMessageStore, or + /// UseBlobStorageForLargeMessages from the AzureStorage.QueueService.Blobs package. Every client that + /// reads the queue needs the same store. + /// + public ILargeMessageStore? LargeMessageStore { get; set; } + + /// + /// With a , send bodies of at least this many bytes to it even when they would fit on + /// the queue; null (the default) sends only what would otherwise be over the 64 KiB limit. + /// + public int? LargeMessageThreshold { get; set; } + internal static readonly TimeSpan MinVisibility = TimeSpan.FromSeconds(1); internal static readonly TimeSpan MaxVisibility = TimeSpan.FromDays(7); /// True when the SDK client is built from the connection string; otherwise from the endpoint URI. internal bool UsesConnectionString => !string.IsNullOrEmpty(ConnectionString); - /// The queue named by the last path segment of , if any. - internal string? EndpointQueueName => - EndpointUri?.AbsolutePath.TrimEnd('/').Split('/').Last() is { Length: > 0 } segment ? segment : null; + /// + /// The queue named in 's path, if any. Read the way the SDK reads it, so an emulator's + /// path-style URI (http://127.0.0.1:10001/devstoreaccount1/orders) names the queue after the account. + /// + internal string? EndpointQueueName => QueueInUri(EndpointUri); + + private static string? QueueInUri(Uri? uri) + { + if (uri is null) return null; + try + { + return new QueueUriBuilder(uri).QueueName?.Trim('/') is { Length: > 0 } name ? name : null; + } + catch (Exception e) when (e is FormatException or ArgumentException) + { + return null; + } + } /// /// The queue the SDK client will actually target, following the same precedence as construction: the connection - /// string path uses ; the endpoint path uses the queue in the URI and ignores . + /// string path uses ; the endpoint path uses the queue in the URI, or + /// when the URI is the account's endpoint. /// - internal string? EffectiveQueueName => UsesConnectionString ? QueueName : EndpointQueueName; + internal string? EffectiveQueueName => UsesConnectionString ? QueueName : EndpointQueueName ?? (string.IsNullOrEmpty(QueueName) ? null : QueueName); + + /// The queue's own URI: when it names the queue, or the account endpoint with added. + internal Uri? QueueUri => + EndpointUri is null || EndpointQueueName is not null || string.IsNullOrEmpty(QueueName) + ? EndpointUri + : new QueueUriBuilder(EndpointUri) { QueueName = QueueName }.ToUri(); /// /// Where dead-lettered messages go: , or the queue name plus -poison. An empty @@ -122,7 +187,9 @@ public class QueueClientSettings private string? ValidateEndpointQueue() { if (EndpointQueueName is null) - return "has an EndpointUri with no queue name in its path."; + return string.IsNullOrEmpty(QueueName) + ? "has an EndpointUri with no queue name in its path, and no QueueName. Put the queue in the URI, or set QueueName when EndpointUri is the storage account's endpoint." + : null; // With endpoint authentication the SDK takes the queue from the URI, so a different QueueName would be // silently ignored and every check based on it would be checking the wrong queue. @@ -158,6 +225,15 @@ public class QueueClientSettings if (VisibilityTimeout is { } visibility && (visibility < MinVisibility || visibility > MaxVisibility)) return $"has a VisibilityTimeout of {visibility}; the service accepts 1 second to 7 days."; + if (PoisonTimeToLive is { } poisonTtl && poisonTtl != Timeout.InfiniteTimeSpan && poisonTtl < TimeSpan.FromSeconds(1)) + return $"has a PoisonTimeToLive of {poisonTtl}; use at least 1 second, or Timeout.InfiniteTimeSpan (or nothing) for never."; + + if (CompressionThreshold is < 0) + return $"has a CompressionThreshold of {CompressionThreshold}; it is a size in bytes and cannot be negative."; + + if (LargeMessageThreshold is { } large && large < 0) + return $"has a LargeMessageThreshold of {large}; it is a size in bytes and cannot be negative."; + var poison = PoisonQueueNameFor(queueName); if (poison is not null && QueueNameRules.Check(poison) is { } problem) { diff --git a/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs b/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs index bf50fbe..e6eeee3 100644 --- a/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs +++ b/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs @@ -1,4 +1,4 @@ -using Azure.Core; +using Azure.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -24,8 +24,8 @@ internal QueueClientSettingsBuilder(IServiceCollection services, QueueClientSett /// /// Registers a named client. Get it from , or inject - /// , or with - /// [FromKeyedServices("name")]. Registering the same name again adds to its configuration. + /// , , or + /// with [FromKeyedServices("name")]. Registering the same name again adds to its configuration. /// public IQueueClientBuilder AddClient(string clientName, Action configure) { @@ -41,6 +41,8 @@ public IQueueClientBuilder AddClient(string clientName, Action(key)); Services.TryAddKeyedSingleton(clientName, static (provider, key) => provider.GetRequiredKeyedService(key)); + Services.TryAddKeyedSingleton(clientName, static (provider, key) => + provider.GetRequiredService().GetDeadLetterQueue((string)key!)); if (Registry.ValidateOnStart) AddStartupValidation(clientName); @@ -59,7 +61,7 @@ public IQueueClientBuilder AddClient(string clientName, IConfiguration configura /// /// Registers the default client. Get it from , or inject - /// , or directly. + /// , , or directly. /// public IQueueClientBuilder AddDefaultClient(Action configure) { @@ -68,9 +70,8 @@ public IQueueClientBuilder AddDefaultClient(Action configur Registry.HasDefaultClient = true; Services.Configure(Options.DefaultName, configure); - Services.TryAddSingleton(static provider => provider.GetRequiredService().GetQueue()); - Services.TryAddSingleton(static provider => provider.GetRequiredService()); - Services.TryAddSingleton(static provider => provider.GetRequiredService()); + // The unkeyed IStorageQueue, IQueueSender, IQueueReceiver and IDeadLetterQueue are registered by + // AddAzureStorageQueueClient, default client or not, so that injecting one without a default client says so. if (Registry.ValidateOnStart) AddStartupValidation(Options.DefaultName); @@ -119,22 +120,104 @@ public QueueClientSettingsBuilder UseTokenCredential(TokenCredential credential) // The post-configurer reads the credential from the registry when settings are built, so a later call replaces // it rather than queueing behind the first. Registry.SharedTokenCredential = credential; - Services.TryAddEnumerable(ServiceDescriptor.Singleton, SharedTokenCredentialPostConfigure>()); + Services.TryAddEnumerable(ServiceDescriptor.Singleton, SharedAccountPostConfigure>()); return this; } + /// + /// Connects every client that has neither a nor an + /// of its own to this storage account, so each client only needs its + /// . is either a connection + /// string or the account's queue endpoint (https://account.queue.core.windows.net/), which is what .NET Aspire + /// puts in ConnectionStrings:<name>: pass builder.Configuration.GetConnectionString("queues"). An + /// endpoint needs a credential, from or each client's own. Applies to default and + /// named clients, registered before and after this call; calling it again replaces the account. + /// + public QueueClientSettingsBuilder UseStorageAccount(string connectionStringOrServiceUri) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionStringOrServiceUri); + + if (Uri.TryCreate(connectionStringOrServiceUri.Trim(), UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp)) + return UseStorageAccount(uri); + + Registry.SharedConnectionString = connectionStringOrServiceUri; + Registry.SharedServiceUri = null; + Services.TryAddEnumerable(ServiceDescriptor.Singleton, SharedAccountPostConfigure>()); + return this; + } + + /// + /// As , with the account's queue endpoint + /// (https://account.queue.core.windows.net/). Clients connect with 's + /// credential or their own. + /// + public QueueClientSettingsBuilder UseStorageAccount(Uri serviceUri) + { + ArgumentNullException.ThrowIfNull(serviceUri); + + Registry.SharedServiceUri = serviceUri; + Registry.SharedConnectionString = null; + Services.TryAddEnumerable(ServiceDescriptor.Singleton, SharedAccountPostConfigure>()); + return this; + } + + /// + /// Runs around every message every client's processors deliver, resolved from the + /// message's scope (registered as scoped unless you registered it yourself). See . + /// + public QueueClientSettingsBuilder UseMiddleware() + where TMiddleware : class, IQueueMessageMiddleware + { + Services.TryAddScoped(); + Services.ConfigureAll(o => o.Middleware.Add(static provider => provider.GetRequiredService())); + return this; + } + + /// Runs around every message every client's processors deliver. See . + public QueueClientSettingsBuilder Use(Func middleware) + { + ArgumentNullException.ThrowIfNull(middleware); + + var instance = new DelegateMiddleware(middleware); + Services.ConfigureAll(o => o.Middleware.Add(_ => instance)); + return this; + } + + /// + /// Reports every registered client's queue and poison queue depth as the queue_approximate_messages gauge, + /// tagged queue.kind main or poison, read every (default one minute). + /// Each read is a billable call per queue, so the interval is a cost. The count is the service's approximate one, + /// and includes messages that are hidden while being processed or waiting for a retry. + /// + public QueueClientSettingsBuilder AddQueueDepthMetrics(TimeSpan? interval = null) + { + var every = interval ?? TimeSpan.FromMinutes(1); + if (every < TimeSpan.FromSeconds(1)) throw new ArgumentOutOfRangeException(nameof(interval), every, "The depth interval must be at least 1 second."); + + Registry.DepthInterval = every; + Services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return this; + } + private void AddStartupValidation(string clientName) => Services.AddOptions(clientName).ValidateOnStart(); // Post-configure, not configure: it must see each client's finished settings (every Configure call, including - // ones made after UseTokenCredential) to know whether the client brought its own credential, and the options - // factory runs post-configuration before validation, so an endpoint-only client passes ValidateOnStart and the - // factory's first-use check alike. - private sealed class SharedTokenCredentialPostConfigure(QueueClientSettingsRegistry registry) : IPostConfigureOptions + // ones made after UseStorageAccount or UseTokenCredential) to know whether the client brought its own account and + // credential, and the options factory runs post-configuration before validation, so a client with only a queue + // name passes ValidateOnStart and the factory's first-use check alike. One post-configurer for both, so the + // account is filled in before the credential that depends on it. + private sealed class SharedAccountPostConfigure(QueueClientSettingsRegistry registry) : IPostConfigureOptions { public void PostConfigure(string? name, QueueClientSettings options) { + if (!options.UsesConnectionString && options.EndpointUri is null) + { + if (registry.SharedConnectionString is { } connectionString) options.ConnectionString = connectionString; + else if (registry.SharedServiceUri is { } serviceUri) options.EndpointUri = serviceUri; + } + if (options.UsesConnectionString || options.EndpointUri is null) return; options.TokenCredential ??= registry.SharedTokenCredential; } diff --git a/src/AzureStorage.QueueService/QueueClientSettingsRegistry.cs b/src/AzureStorage.QueueService/QueueClientSettingsRegistry.cs index 2563b6d..db55551 100644 --- a/src/AzureStorage.QueueService/QueueClientSettingsRegistry.cs +++ b/src/AzureStorage.QueueService/QueueClientSettingsRegistry.cs @@ -9,6 +9,8 @@ namespace AzureStorage.QueueService; internal sealed class QueueClientSettingsRegistry { private readonly HashSet _clientNames = new(StringComparer.Ordinal); + private readonly HashSet _processors = new(StringComparer.Ordinal); + private int _processorNumber; public bool HasDefaultClient { get; set; } @@ -17,9 +19,23 @@ internal sealed class QueueClientSettingsRegistry /// The credential set by UseTokenCredential, for endpoint clients that have none of their own. public TokenCredential? SharedTokenCredential { get; set; } + /// The account set by UseStorageAccount: a connection string, for clients with no connection of their own. + public string? SharedConnectionString { get; set; } + + /// The account set by UseStorageAccount: the queue service endpoint, for clients with no connection of their own. + public Uri? SharedServiceUri { get; set; } + + /// How often the depth monitor reads queue counts; set by AddQueueDepthMetrics. + public TimeSpan DepthInterval { get; set; } = TimeSpan.FromMinutes(1); + public IReadOnlyCollection ClientNames => _clientNames; public void Add(string clientName) => _clientNames.Add(clientName); public bool Contains(string clientName) => _clientNames.Contains(clientName); + + /// False when a processor with these options was already registered: the second registration only adds to its options. + public bool AddProcessor(string optionsName) => _processors.Add(optionsName); + + public int NextProcessorNumber() => Interlocked.Increment(ref _processorNumber); } diff --git a/src/AzureStorage.QueueService/QueueDepthMonitor.cs b/src/AzureStorage.QueueService/QueueDepthMonitor.cs new file mode 100644 index 0000000..54f1440 --- /dev/null +++ b/src/AzureStorage.QueueService/QueueDepthMonitor.cs @@ -0,0 +1,77 @@ +using AzureStorage.QueueService.Telemetry; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace AzureStorage.QueueService; + +/// +/// Reads every registered client's queue and poison queue counts on an interval, for the queue_approximate_messages +/// gauge. Registered by AddQueueDepthMetrics. A count that cannot be read is dropped from the gauge, rather than +/// reported stale, until it can be read again. +/// +internal sealed class QueueDepthMonitor( + QueueClientSettingsRegistry registry, + IQueueClientFactory factory, + QueueServiceMetrics metrics, + ILogger logger, + TimeProvider? time = null) : BackgroundService +{ + private const string Main = "main"; + private const string Poison = "poison"; + + private readonly TimeProvider _time = time ?? TimeProvider.System; + private readonly HashSet _failing = new(StringComparer.Ordinal); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(registry.DepthInterval, _time); + do + { + await ReadAllAsync(stoppingToken); + } + while (await timer.WaitForNextTickAsync(stoppingToken)); + } + + internal async Task ReadAllAsync(CancellationToken cancellationToken) + { + var clients = registry.ClientNames.ToList(); + if (registry.HasDefaultClient) clients.Add(Options.DefaultName); + + foreach (var client in clients) + { + IStorageQueue queue; + try + { + queue = client.Length == 0 ? factory.GetQueue() : factory.GetQueue(client); + } + catch (InvalidOperationException) + { + continue; // a misconfigured client fails where it is used, not here + } + + await ReadAsync(queue.QueueName, Main, async ct => (await queue.GetPropertiesAsync(ct)).ApproximateMessagesCount, cancellationToken); + + var poison = factory.GetDeadLetterQueue(client); + await ReadAsync(poison.QueueName, Poison, async ct => await poison.GetApproximateCountAsync(ct), cancellationToken); + } + } + + private async Task ReadAsync(string queueName, string kind, Func> read, CancellationToken cancellationToken) + { + try + { + metrics.SetDepth(queueName, kind, await read(cancellationToken)); + _failing.Remove(queueName); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + metrics.ForgetDepth(queueName, kind); + if (_failing.Add(queueName)) logger.LogDepthReadFailed(queueName, e); + } + } +} diff --git a/src/AzureStorage.QueueService/QueueMessage.cs b/src/AzureStorage.QueueService/QueueMessage.cs index 607f622..371ecc4 100644 --- a/src/AzureStorage.QueueService/QueueMessage.cs +++ b/src/AzureStorage.QueueService/QueueMessage.cs @@ -20,6 +20,71 @@ public enum MessageSettlement DeadLettered, } +/// +/// A received message without its body's type: what middleware and other code that serves every message type sees. +/// Every is one. +/// +public interface IQueueMessage +{ + /// The service's id for the message. + string MessageId { get; } + + /// The queue the message was received from. Empty for a message built by hand without one. + string QueueName { get; } + + /// The receipt that proves this receiver holds the message; changes after a renewal or an abandon. + string PopReceipt { get; } + + /// How many times the message has been received, including this time. + long DequeueCount { get; } + + /// When the message was first put on the queue. + DateTimeOffset? InsertedOn { get; } + + /// When the service discards the message if nobody completes it. + DateTimeOffset? ExpiresOn { get; } + + /// When the message becomes visible to other consumers unless it is settled or renewed first. + DateTimeOffset? NextVisibleOn { get; } + + /// The serialized message, before deserialization. See . + BinaryData RawBody { get; } + + /// The deserialized body, or null; see . + object? Body { get; } + + /// Set when the body could not be read. The message is still returned so it can be dead-lettered. + DeserializationException? Error { get; } + + /// Headers the sender attached with ; empty when there were none. + IReadOnlyDictionary Headers { get; } + + /// + /// True when a processor will not retry this message if the handler fails: this is its last delivery before + /// moves it to the poison queue. Always false for a message + /// received with , which has no retry limit of its own. + /// + bool IsLastAttempt { get; } + + /// How the message was settled, if it has been. + MessageSettlement Settlement { get; } + + /// True once the message has been completed, abandoned or dead-lettered. + bool IsSettled { get; } + + /// Deletes the message from the queue. + ValueTask CompleteAsync(CancellationToken cancellationToken = default); + + /// Gives the message back to the queue: visible again immediately, or after (up to 7 days). + ValueTask AbandonAsync(TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default); + + /// Keeps the message hidden for another (more than zero, up to 7 days), measured from now. + ValueTask RenewLockAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken = default); + + /// Moves the message to the poison queue, recording with it. + ValueTask DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default); +} + /// /// A message received from a queue, with its identity and the operations that settle it. The message stays /// hidden from other consumers until its visibility timeout passes or it is completed, abandoned or dead-lettered. @@ -29,14 +94,15 @@ public enum MessageSettlement /// done. Operations are serialized, so a renewal and a completion cannot race on the pop receipt, and every /// operation uses the latest receipt. Once settled, further operations throw . /// -public sealed class QueueMessage +/// The type the body is read as. +public sealed class QueueMessage : IQueueMessage { - private readonly IQueueMessageOperations _operations; - private readonly SemaphoreSlim _gate = new(1, 1); - private readonly BinaryData? _wireBody; + private static readonly IReadOnlyDictionary NoHeaders = new Dictionary(); /// /// Public so a test that stubs can build messages; the library passes its real operations. + /// 's optional properties set , and + /// . /// public QueueMessage(QueueMessageInfo info, BinaryData rawBody, T? body, DeserializationException? error, IQueueMessageOperations operations) { @@ -44,63 +110,85 @@ public QueueMessage(QueueMessageInfo info, BinaryData rawBody, T? body, Deserial ArgumentNullException.ThrowIfNull(rawBody); ArgumentNullException.ThrowIfNull(operations); - MessageId = info.MessageId; - PopReceipt = info.PopReceipt; - DequeueCount = info.DequeueCount; - InsertedOn = info.InsertedOn; - ExpiresOn = info.ExpiresOn; - NextVisibleOn = info.NextVisibleOn; - RawBody = rawBody; + Lease = new MessageLease(info, rawBody, operations); Body = body; Error = error; - _operations = operations; } - public string MessageId { get; } + // A second typed view of the same received message (a routed or wrapped handler's own type), sharing its receipt + // and settlement, so whichever view settles it, the processor sees it settled. + private QueueMessage(MessageLease lease, T? body, DeserializationException? error) + { + Lease = lease; + Body = body; + Error = error; + } + + internal MessageLease Lease { get; } + + /// + public string MessageId => Lease.MessageId; + + /// + public string QueueName => Lease.QueueName; /// Changes after and . - public string PopReceipt { get; private set; } + public string PopReceipt => Lease.PopReceipt; /// How many times the message has been received, including this time. - public long DequeueCount { get; } + public long DequeueCount => Lease.DequeueCount; - public DateTimeOffset? InsertedOn { get; } + /// + public DateTimeOffset? InsertedOn => Lease.InsertedOn; - public DateTimeOffset? ExpiresOn { get; } + /// + public DateTimeOffset? ExpiresOn => Lease.ExpiresOn; - public DateTimeOffset? NextVisibleOn { get; private set; } + /// + public DateTimeOffset? NextVisibleOn => Lease.NextVisibleOn; /// - /// The serialized message, before deserialization. When the sender wrapped it in a trace-context envelope - /// ( on the sending client), this is the message inside the - /// envelope, so it is the same whichever way the message was sent. + /// The serialized message, before deserialization: what the sender's serializer produced. When the sender wrapped + /// it in the library's envelope (for trace context, headers, compression or a body kept in Blob storage), this is the + /// message inside the envelope, decompressed and downloaded, so it is the same whichever way the message was sent. /// - public BinaryData RawBody { get; } + public BinaryData RawBody => Lease.RawBody; + + /// + public IReadOnlyDictionary Headers => Lease.Headers ?? NoHeaders; + + /// + public bool IsLastAttempt => Lease.IsLastAttempt; + + /// Null when deserialization failed (see ) or when the body was JSON null. + public T? Body { get; private set; } + + object? IQueueMessage.Body => Body; + + /// Set when the body could not be decoded or deserialized. The message is still returned so it can be dead-lettered. + public DeserializationException? Error { get; private set; } + + /// How the message was settled, if it has been. + public MessageSettlement Settlement => Lease.Settlement; + + /// True once the message has been completed, abandoned or dead-lettered. + public bool IsSettled => Lease.Settlement != MessageSettlement.None; /// /// The body exactly as it arrived on the queue: the envelope when there was one, otherwise . - /// Dead-lettering copies this, so a message moved back from the poison queue replays with its trace context. + /// Dead-lettering starts from this, so a message moved back from the poison queue replays with its trace context. /// - internal BinaryData WireBody - { - get => _wireBody ?? RawBody; - init => _wireBody = value; - } + internal BinaryData WireBody => Lease.WireBody; /// The sender's trace context, when it was carried in an envelope and parsed. - internal ActivityContext? CreationContext { get; init; } + internal ActivityContext? CreationContext => Lease.CreationContext; /// - /// What a processor's read step took from the body for a wrapper handler: the CloudEvent adapter's typed data. + /// What a processor's read step took from the body for a wrapper handler: the CloudEvent adapter's typed data, or + /// the route a routed processor chose. /// internal object? HandlerData { get; set; } - /// Null when deserialization failed (see ) or when the body was JSON null. - public T? Body { get; private set; } - - /// Set when the body could not be decoded or deserialized. The message is still returned so it can be dead-lettered. - public DeserializationException? Error { get; private set; } - // The processor receives raw and deserializes only after lease renewal is armed, so a slow serializer cannot // let the lease lapse before the message is protected. internal void SetBody(T? body, DeserializationException? error) @@ -109,41 +197,155 @@ internal void SetBody(T? body, DeserializationException? error) Error = error; } - /// How the message was settled, if it has been. + /// A view of this message with a body of another type, sharing its receipt and settlement. + internal QueueMessage As(TOther? body, DeserializationException? error) => new(Lease, body, error); + + /// Deletes the message from the queue. + public ValueTask CompleteAsync(CancellationToken cancellationToken = default) => Lease.CompleteAsync(cancellationToken); + + /// + /// Gives the message back to the queue: visible again immediately, or after . + /// + /// is negative or longer than 7 days. + public ValueTask AbandonAsync(TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default) => + Lease.AbandonAsync(visibilityTimeout ?? TimeSpan.Zero, cancellationToken); + + /// Keeps the message hidden for another , measured from now. + /// is not more than zero, or is longer than 7 days. + public ValueTask RenewLockAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken = default) => + Lease.RenewAsync(visibilityTimeout, cancellationToken); + + /// + /// Copies the message to the poison queue and deletes it from this one. The copy keeps the body as it arrived and + /// records , the original message id and the dequeue count, which + /// reads back. + /// + public ValueTask DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) => + Lease.DeadLetterAsync(reason, DeadLetterCategories.Handler, cancellationToken); + + internal ValueTask DeadLetterAsync(string reason, string category, CancellationToken cancellationToken) => + Lease.DeadLetterAsync(reason, category, cancellationToken); +} + +/// The bounded set of reasons the library records for a dead-lettered message, for metrics and replay filters. +public static class DeadLetterCategories +{ + /// Delivered more than times. + public const string MaxDequeueCount = "max_dequeue"; + + /// The body could not be decoded, unwrapped, downloaded or deserialized. + public const string Deserialization = "deserialization"; + + /// The handler dead-lettered it, or threw an exception the processor's error policy dead-letters. + public const string Handler = "handler"; + + /// A routed processor had no handler for the message's type. + public const string Unroutable = "unroutable"; +} + +/// +/// One received message's identity, receipt and settlement, shared by every typed view of it. All settlement goes +/// through here, one operation at a time. +/// +internal sealed class MessageLease +{ + private readonly SemaphoreSlim _gate = new(1, 1); + + public MessageLease(QueueMessageInfo info, BinaryData rawBody, IQueueMessageOperations operations) + { + MessageId = info.MessageId; + PopReceipt = info.PopReceipt; + DequeueCount = info.DequeueCount; + InsertedOn = info.InsertedOn; + ExpiresOn = info.ExpiresOn; + NextVisibleOn = info.NextVisibleOn; + QueueName = info.QueueName ?? string.Empty; + Headers = info.Headers; + IsLastAttempt = info.IsLastAttempt; + RawBody = rawBody; + Operations = operations; + } + + public IQueueMessageOperations Operations { get; } + + public string MessageId { get; } + + public string QueueName { get; } + + public string PopReceipt { get; private set; } + + public long DequeueCount { get; } + + public DateTimeOffset? InsertedOn { get; } + + public DateTimeOffset? ExpiresOn { get; } + + public DateTimeOffset? NextVisibleOn { get; private set; } + public MessageSettlement Settlement { get; private set; } - /// True once the message has been completed, abandoned or dead-lettered. - public bool IsSettled => Settlement != MessageSettlement.None; + public IReadOnlyDictionary? Headers { get; set; } - /// Deletes the message from the queue. - public async ValueTask CompleteAsync(CancellationToken cancellationToken = default) + public bool IsLastAttempt { get; set; } + + /// The payload: set again once a body kept in Blob storage has been downloaded. + public BinaryData RawBody { get; set; } + + /// The body exactly as it arrived on the queue; unless there was an envelope. + public BinaryData WireBody + { + get => _wireBody ?? RawBody; + set => _wireBody = value; + } + + private BinaryData? _wireBody; + + /// The envelope the message arrived in, if it was one this version reads. + public MessageEnvelope? Envelope { get; set; } + + public ActivityContext? CreationContext { get; set; } + + /// Where the body is when it was kept in Blob storage and has not been downloaded yet. + public string? PendingBlob { get; set; } + + /// The span of the delivery that holds this message, for settlement events. Set by the processor. + public Activity? DeliveryActivity { get; set; } + + /// Runs after a successful completion, best effort: deletes a body kept in Blob storage. + public Func? AfterComplete { get; set; } + + public async ValueTask CompleteAsync(CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken); try { ThrowIfSettled(); - await _operations.DeleteAsync(MessageId, PopReceipt, cancellationToken); + await Operations.DeleteAsync(MessageId, PopReceipt, cancellationToken); Settlement = MessageSettlement.Completed; + DeliveryActivity?.AddEvent(new ActivityEvent("queue.message_completed")); } finally { _gate.Release(); } + + if (AfterComplete is { } after) await after(); } - /// - /// Gives the message back to the queue: visible again immediately, or after . - /// - public async ValueTask AbandonAsync(TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default) + public async ValueTask AbandonAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken) { + if (visibilityTimeout < TimeSpan.Zero || visibilityTimeout > QueueClientSettings.MaxVisibility) + throw new ArgumentOutOfRangeException(nameof(visibilityTimeout), visibilityTimeout, "The service accepts 0 seconds to 7 days."); + await _gate.WaitAsync(cancellationToken); try { ThrowIfSettled(); - var receipt = await _operations.UpdateVisibilityAsync(MessageId, PopReceipt, visibilityTimeout ?? TimeSpan.Zero, cancellationToken); + var receipt = await Operations.UpdateVisibilityAsync(MessageId, PopReceipt, visibilityTimeout, cancellationToken); PopReceipt = receipt.PopReceipt; NextVisibleOn = receipt.NextVisibleOn; Settlement = MessageSettlement.Abandoned; + DeliveryActivity?.AddEvent(new ActivityEvent("queue.message_abandoned")); } finally { @@ -151,18 +353,19 @@ public async ValueTask AbandonAsync(TimeSpan? visibilityTimeout = null, Cancella } } - /// Keeps the message hidden for another , measured from now. - public async ValueTask RenewLockAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken = default) + public async ValueTask RenewAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken) { - if (visibilityTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(visibilityTimeout), "A renewal must be longer than zero."); + if (visibilityTimeout <= TimeSpan.Zero || visibilityTimeout > QueueClientSettings.MaxVisibility) + throw new ArgumentOutOfRangeException(nameof(visibilityTimeout), visibilityTimeout, "A renewal must be longer than zero and no longer than 7 days."); await _gate.WaitAsync(cancellationToken); try { ThrowIfSettled(); - var receipt = await _operations.UpdateVisibilityAsync(MessageId, PopReceipt, visibilityTimeout, cancellationToken); + var receipt = await Operations.UpdateVisibilityAsync(MessageId, PopReceipt, visibilityTimeout, cancellationToken); PopReceipt = receipt.PopReceipt; NextVisibleOn = receipt.NextVisibleOn; + DeliveryActivity?.AddEvent(new ActivityEvent("queue.message_renewed")); } finally { @@ -170,15 +373,18 @@ public async ValueTask RenewLockAsync(TimeSpan visibilityTimeout, CancellationTo } } - /// Copies the message to the poison queue exactly as it arrived, and deletes it from this one. - public async ValueTask DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) + public async ValueTask DeadLetterAsync(string? reason, string category, CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken); try { ThrowIfSettled(); - await _operations.DeadLetterAsync(MessageId, PopReceipt, DequeueCount, WireBody, reason, cancellationToken); + if (Operations is IDeadLetterOperations library) + await library.DeadLetterAsync(this, reason, category, cancellationToken); + else + await Operations.DeadLetterAsync(MessageId, PopReceipt, DequeueCount, WireBody, reason, cancellationToken); Settlement = MessageSettlement.DeadLettered; + DeliveryActivity?.AddEvent(new ActivityEvent("queue.message_dead_lettered", tags: new ActivityTagsCollection { ["queue.dead_letter_reason"] = category })); } finally { @@ -188,6 +394,15 @@ public async ValueTask DeadLetterAsync(string? reason = null, CancellationToken private void ThrowIfSettled() { - if (IsSettled) throw new InvalidOperationException($"Message {MessageId} has already been settled ({Settlement})."); + if (Settlement != MessageSettlement.None) throw new InvalidOperationException($"Message {MessageId} has already been settled ({Settlement})."); } } + +/// +/// The library's own dead-letter path: it gets the whole lease, so the copy on the poison queue can record why and +/// from where, which the public has no parameters for. +/// +internal interface IDeadLetterOperations +{ + ValueTask DeadLetterAsync(MessageLease message, string? reason, string category, CancellationToken cancellationToken); +} diff --git a/src/AzureStorage.QueueService/QueueMessageInfo.cs b/src/AzureStorage.QueueService/QueueMessageInfo.cs index 14dafcb..525d96e 100644 --- a/src/AzureStorage.QueueService/QueueMessageInfo.cs +++ b/src/AzureStorage.QueueService/QueueMessageInfo.cs @@ -3,10 +3,26 @@ namespace AzureStorage.QueueService; /// /// The service-side identity of a received message. /// +/// The service's id for the message. +/// The receipt that proves the receiver holds the message. +/// How many times the message has been received, including this time. +/// When the message was first put on the queue. +/// When the service discards the message. +/// When the message becomes visible again unless it is settled or renewed. public sealed record QueueMessageInfo( string MessageId, string PopReceipt, long DequeueCount, DateTimeOffset? InsertedOn, DateTimeOffset? ExpiresOn, - DateTimeOffset? NextVisibleOn); + DateTimeOffset? NextVisibleOn) +{ + /// The queue the message came from. + public string? QueueName { get; init; } + + /// Headers the sender attached. + public IReadOnlyDictionary? Headers { get; init; } + + /// Whether this is the message's last delivery before a processor dead-letters it; see . + public bool IsLastAttempt { get; init; } +} diff --git a/src/AzureStorage.QueueService/QueueMessageMiddleware.cs b/src/AzureStorage.QueueService/QueueMessageMiddleware.cs new file mode 100644 index 0000000..44e4a62 --- /dev/null +++ b/src/AzureStorage.QueueService/QueueMessageMiddleware.cs @@ -0,0 +1,73 @@ +namespace AzureStorage.QueueService; + +/// The rest of a processor's pipeline: the next middleware, and finally the handler. +public delegate ValueTask QueueMessageDelegate(QueueMessageContext context); + +/// +/// Runs around every message a processor delivers, the way ASP.NET Core middleware runs around a request: for +/// idempotency checks, timing, a circuit breaker, a tenant scope or anything else every handler would otherwise repeat. +/// Resolved from the message's own dependency-injection scope, so it can take scoped services. +/// +/// +/// Call to go on to the handler, or return without calling it to stop there: the processor then +/// treats the message as handled (with on, it is completed). Throw and +/// the message counts as failed, exactly as if the handler had thrown. A middleware can settle the message itself +/// through . Register it with UseMiddleware<T>() on one client's +/// builder, or on AddAzureStorageQueueClient(queues => queues.UseMiddleware<T>()) for every client; each +/// runs in the order it was registered. +/// +public interface IQueueMessageMiddleware +{ + /// Handles , usually by doing something before and after . + ValueTask InvokeAsync(QueueMessageContext context, QueueMessageDelegate next); +} + +/// One message's way through a processor's middleware. +public sealed class QueueMessageContext +{ + internal QueueMessageContext(IQueueMessage message, IServiceProvider services, string clientName, Type messageType, string handlerName, CancellationToken cancellationToken) + { + Message = message; + Services = services; + ClientName = clientName; + MessageType = messageType; + HandlerName = handlerName; + CancellationToken = cancellationToken; + } + + /// The message, with its body already read. + public IQueueMessage Message { get; } + + /// The message's own dependency-injection scope, the one the handler is resolved from. + public IServiceProvider Services { get; } + + /// The client the processor belongs to; empty for the default client. + public string ClientName { get; } + + /// The queue the message came from. + public string QueueName => Message.QueueName; + + /// The type the processor reads bodies as. + public Type MessageType { get; } + + /// The handler's name, as it appears in the processor's logs. + public string HandlerName { get; } + + /// Cancelled when the host shuts down and the drain timeout has passed. + public CancellationToken CancellationToken { get; } + + /// A place for middleware to leave things for later middleware or the handler's scope to find. + public IDictionary Items { get; } = new Dictionary(); +} + +/// The middleware registered for one client (or, under every name, for all of them), in order. +internal sealed class QueueMiddlewareOptions +{ + public List> Middleware { get; } = new(); +} + +/// A middleware written as a lambda. +internal sealed class DelegateMiddleware(Func invoke) : IQueueMessageMiddleware +{ + public ValueTask InvokeAsync(QueueMessageContext context, QueueMessageDelegate next) => invoke(context, next); +} diff --git a/src/AzureStorage.QueueService/QueueProcessor.cs b/src/AzureStorage.QueueService/QueueProcessor.cs index 5b56905..b3f61a5 100644 --- a/src/AzureStorage.QueueService/QueueProcessor.cs +++ b/src/AzureStorage.QueueService/QueueProcessor.cs @@ -1,30 +1,59 @@ +using Azure; +using Azure.Storage.Queues.Models; using AzureStorage.QueueService.Telemetry; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using System.Collections.Concurrent; using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Threading.RateLimiting; namespace AzureStorage.QueueService; /// -/// The hosted loop behind AddProcessor<TMessage, THandler>(): -/// workers, each of which receives one message, delivers it, and goes back for the next. One instance per registration. +/// What a registration adds to the plain processor: how to build the handler, what to call it in logs, and how to read +/// a body when the client's serializer alone is not the whole story (CloudEvents, routing). +/// +/// Builds the handler from the message's scope; null resolves THandler from it. +/// The name logs use; the application's handler rather than a library wrapper. +/// Replaces deserialization with the client's serializer. Sets or . +/// Runs after the body deserialized without error; may set . +/// Runs around the handler, outermost first. +internal sealed record ProcessorHooks( + Func? CreateHandler = null, + string? HandlerName = null, + Action>? ReadBody = null, + Action>? AfterDeserialize = null, + IReadOnlyList>? Middleware = null); + +/// +/// The hosted processor behind AddProcessor<TMessage, THandler>(): one receive loop that takes as many +/// messages as there are free handlers (up to at once, 32 per +/// request) and delivers each on its own task. One instance per registration. /// /// +/// +/// No message waits behind another's handler, and the processor never holds a message no handler is free for: a receive +/// asks only for as many messages as there are free slots, and each message's lease renewal starts as its delivery does. +/// A full batch means more may be waiting, so the loop asks again at once; an empty one backs off. +/// +/// /// Lifecycle: snapshots and validates the options and resolves the queue (a -/// misconfigured client fails startup on purpose). The workers start when the host reports +/// misconfigured client fails startup on purpose). Receiving starts when the host reports /// — the signal that every startup callback succeeded — so a /// slow start does not hold leases and a start that fails part-way never polls at all; without a host lifetime (a -/// direct test) they start in . On shutdown, receiving stops; running handlers get +/// direct test) it starts in . On shutdown, receiving stops; running handlers get /// to finish; then their token is cancelled and lease renewal /// stops in the same moment; a handler that ignores its token gets the same time again, after which shutdown stops /// waiting for it. If the host's own shutdown deadline arrives first, the handlers are cancelled at that moment and /// returns rather than throwing, as BackgroundService does. The cancellations happen no -/// matter how exits. Nothing that happens at runtime stops the host, and a worker that fails +/// matter how exits. Nothing that happens at runtime stops the host, and a receive loop that fails /// unexpectedly is logged loudly and restarted rather than silently lost. +/// /// -internal sealed class QueueProcessor : IHostedService, IHostedLifecycleService, IDisposable +internal sealed class QueueProcessor : IHostedService, IHostedLifecycleService, IControlledProcessor, IDisposable where THandler : class, IQueueMessageHandler { private readonly string _clientName; @@ -36,25 +65,41 @@ internal sealed class QueueProcessor : IHostedService, IHost private readonly QueueServiceMetrics _metrics; private readonly TimeProvider _time; private readonly IHostApplicationLifetime? _lifetime; - private readonly Func? _createHandler; + private readonly QueueProcessorController? _controller; + private readonly ProcessorHooks _hooks; private readonly string _handlerName; - private readonly Action>? _afterDeserialize; private readonly CancellationTokenSource _stopReceiving = new(); private readonly CancellationTokenSource _cancelHandlers = new(); private readonly CancellationTokenSource _stopRenewals = new(); - // Taken once, up front: reading Token from a disposed source throws, and a worker can still be mid-delivery when - // a host that gave up waiting disposes the processor. + // Taken once, up front: reading Token from a disposed source throws, and a delivery can still be running when a + // host that gave up waiting disposes the processor. private readonly CancellationToken _handlerToken; private readonly CancellationToken _renewalToken; private readonly object _startLock = new(); private CancellationTokenRegistration _startWhenApplicationStarted; + // Deliveries in flight, by a sequence number, with the timestamp each started at: for draining, and for the + // oldest-handler gauge and status. + private readonly ConcurrentDictionary _deliveries = new(); + private long _deliverySequence; + private QueueProcessorOptions _options = new(); + private SemaphoreSlim? _slots; private IStorageQueue? _queue; - private Task[]? _workers; + private Task? _receiver; private string _name; + private IDisposable? _registration; + private IDisposable? _ageGauge; + private volatile QueueProcessorState _state = QueueProcessorState.NotStarted; + private bool _pausedLogged; + + // Receive health, for status. Written by the receive loop only. + private DateTimeOffset? _lastReceiveSucceededAt; + private DateTimeOffset? _lastReceiveFailedAt; + private int _consecutiveReceiveFailures; + private string? _lastReceiveError; public QueueProcessor( string clientName, @@ -66,13 +111,13 @@ public QueueProcessor( QueueServiceMetrics metrics, TimeProvider time, IHostApplicationLifetime? lifetime = null, - Func? createHandler = null, - string? handlerName = null, - Action>? afterDeserialize = null) + ProcessorHooks? hooks = null, + QueueProcessorController? controller = null) { - // A wrapper handler (the CloudEvent adapter) passes the application's handler name, so logs name the class - // the application wrote rather than the library's wrapper. - _handlerName = handlerName ?? typeof(THandler).Name; + _hooks = hooks ?? new ProcessorHooks(); + // A wrapper handler (the CloudEvent adapter, a router, a lambda) passes the application's handler name, so logs + // name what the application wrote rather than the library's wrapper. + _handlerName = _hooks.HandlerName ?? typeof(THandler).Name; _clientName = clientName; _optionsName = optionsName; _factory = factory; @@ -82,8 +127,7 @@ public QueueProcessor( _metrics = metrics; _time = time; _lifetime = lifetime; - _createHandler = createHandler; - _afterDeserialize = afterDeserialize; + _controller = controller; _name = _handlerName; _handlerToken = _cancelHandlers.Token; _renewalToken = _stopRenewals.Token; @@ -91,6 +135,8 @@ public QueueProcessor( private string QueueName => _queue?.QueueName ?? _clientName; + public string ClientName => _clientName; + // ------------------------------------------------------------------------------------------------------------ // Lifecycle // ------------------------------------------------------------------------------------------------------------ @@ -100,14 +146,18 @@ public Task StartAsync(CancellationToken cancellationToken) // A snapshot: these are the options the processor runs with for its whole life. _options = _optionsFactory.Create(_optionsName); _options.Validate($"Processor {_name}"); + _slots = new SemaphoreSlim(_options.MaxConcurrentMessages, _options.MaxConcurrentMessages); // Throws for a misconfigured client. That fails host startup, which is the right time to find out. _queue = _clientName == Options.DefaultName ? _factory.GetQueue() : _factory.GetQueue(_clientName); _name = $"{_handlerName} on queue '{_queue.QueueName}'"; + _registration = _controller?.Register(this); + _ageGauge = _metrics.ObserveHandlerAge(ObserveOldestHandler); + // Poll only once the whole application has started. If any later startup step fails, this never fires. if (_lifetime is not null) - _startWhenApplicationStarted = _lifetime.ApplicationStarted.Register(StartWorkers); + _startWhenApplicationStarted = _lifetime.ApplicationStarted.Register(StartReceiving); return Task.CompletedTask; } @@ -119,21 +169,20 @@ public Task StartedAsync(CancellationToken cancellationToken) if (_queue is null) throw new InvalidOperationException("StartAsync must run before StartedAsync."); // No host lifetime to wait for: this callback is the best signal there is. - if (_lifetime is null) StartWorkers(); + if (_lifetime is null) StartReceiving(); return Task.CompletedTask; } - private void StartWorkers() + private void StartReceiving() { lock (_startLock) { - if (_workers is not null || _stopReceiving.IsCancellationRequested) return; + if (_receiver is not null || _stopReceiving.IsCancellationRequested) return; _logger.LogProcessorStarted(_name, _options.MaxConcurrentMessages); - _workers = Enumerable.Range(0, _options.MaxConcurrentMessages) - .Select(_ => Task.Run(() => WorkerAsync(_stopReceiving.Token), CancellationToken.None)) - .ToArray(); + _state = QueueProcessorState.Running; + _receiver = Task.Run(() => ReceiverAsync(_stopReceiving.Token), CancellationToken.None); } } @@ -146,11 +195,16 @@ public Task StoppingAsync(CancellationToken cancellationToken) public async Task StopAsync(CancellationToken cancellationToken) { _stopReceiving.Cancel(); - if (_workers is null) return; + if (_receiver is null) + { + _state = QueueProcessorState.Stopped; + return; + } + _state = QueueProcessorState.Stopping; try { - var all = Task.WhenAll(_workers); + var all = AllWorkAsync(); if (all.IsCompleted) return; _logger.LogProcessorDraining(_name, _options.ShutdownDrainTimeout, Running()); @@ -174,6 +228,7 @@ public async Task StopAsync(CancellationToken cancellationToken) // nothing the processor owns keeps running afterwards. _cancelHandlers.Cancel(); _stopRenewals.Cancel(); + _state = QueueProcessorState.Stopped; _logger.LogProcessorStopped(_name); } } @@ -183,18 +238,27 @@ public async Task StopAsync(CancellationToken cancellationToken) public void Dispose() { // A host that failed part-way through startup disposes its services without ever calling StopAsync. - // Cancel first: a disposed, uncancelled source tells nobody anything, and workers may already be polling. + // Cancel first: a disposed, uncancelled source tells nobody anything, and the loop may already be polling. _startWhenApplicationStarted.Dispose(); _stopReceiving.Cancel(); _cancelHandlers.Cancel(); _stopRenewals.Cancel(); + _registration?.Dispose(); + _ageGauge?.Dispose(); _stopReceiving.Dispose(); _cancelHandlers.Dispose(); _stopRenewals.Dispose(); } - private int Running() => _workers?.Count(w => !w.IsCompleted) ?? 0; + // The receive loop first (it stops adding deliveries once it has exited), then every delivery it started. + private async Task AllWorkAsync() + { + if (_receiver is { } receiver) await receiver; + await Task.WhenAll(_deliveries.Values.Select(d => d.Task)); + } + + private int Running() => _deliveries.Count; private async Task FinishedWithinAsync(Task task, TimeSpan timeout, CancellationToken cancellationToken) { @@ -214,17 +278,33 @@ private async Task FinishedWithinAsync(Task task, TimeSpan timeout, Cancel } } + public QueueProcessorStatus GetStatus() + { + var state = _state == QueueProcessorState.Running && _controller?.IsPaused(_clientName) == true ? QueueProcessorState.Paused : _state; + TimeSpan? oldest = _deliveries.IsEmpty ? null : _time.GetElapsedTime(_deliveries.Values.Min(d => d.Started)); + return new QueueProcessorStatus( + _clientName, QueueName, _handlerName, typeof(TMessage), state, _deliveries.Count, _options.MaxConcurrentMessages, + _lastReceiveSucceededAt, _lastReceiveFailedAt, _consecutiveReceiveFailures, _lastReceiveError, oldest); + } + + private IEnumerable> ObserveOldestHandler() + { + // 0 when idle, so the series is there to alert on before anything hangs. + var oldest = _deliveries.IsEmpty ? 0 : _time.GetElapsedTime(_deliveries.Values.Min(d => d.Started)).TotalSeconds; + yield return new Measurement(oldest, new KeyValuePair(QueueServiceDiagnostics.Names.QueueName, QueueName)); + } + // ------------------------------------------------------------------------------------------------------------ - // A worker: receive one, deliver it, repeat + // The receive loop: take as many as there are free handlers, hand each to its own delivery, repeat // ------------------------------------------------------------------------------------------------------------ - private async Task WorkerAsync(CancellationToken stop) + private async Task ReceiverAsync(CancellationToken stop) { while (!stop.IsCancellationRequested) { try { - await ReceiveAndDeliverUntilStoppedAsync(stop); + await ReceiveUntilStoppedAsync(stop); return; } catch (OperationCanceledException) when (stop.IsCancellationRequested) @@ -233,58 +313,148 @@ private async Task WorkerAsync(CancellationToken stop) } catch (Exception ex) { - // Nothing in the loop is supposed to throw. If something does, say so loudly and start again - // rather than silently running one worker short for the rest of the process's life. + // Nothing in the loop is supposed to throw. If something does, say so loudly and start again rather + // than silently stop processing for the rest of the process's life. _logger.LogProcessorWorkerFailed(_name, _options.ErrorBackoffMax, ex); if (!await DelayAsync(_options.ErrorBackoffMax, stop)) return; } } } - private async Task ReceiveAndDeliverUntilStoppedAsync(CancellationToken stop) + private async Task ReceiveUntilStoppedAsync(CancellationToken stop) { var queue = _queue!; + var slots = _slots!; var owned = queue as IOwnedReceiver; var emptyBackoff = _options.EmptyQueueBackoffMin; var errorBackoff = _options.EmptyQueueBackoffMin; while (!stop.IsCancellationRequested) { - QueueMessage? message; + await WaitWhilePausedAsync(stop); + + // At least one free handler, then every other one that is free right now. + await slots.WaitAsync(stop); + var take = 1; + while (take < StorageQueue.ServiceMaxMessages && slots.Wait(0)) take++; + + IReadOnlyList> messages; try { - // The library's own queue hands the message over before deserializing it, so renewal is armed first. + if (_options.RateLimiter is { } limiter) + { + var permitted = await AcquirePermitsAsync(limiter, take, stop); + slots.Release(take - permitted); + take = permitted; + } + + // The library's own queue hands messages over before reading their bodies, so renewal is armed first. // Any other IStorageQueue (a fake in a test) deserializes on receive. - message = owned is not null - ? await owned.ReceiveUndeserializedAsync(_options.VisibilityTimeout, stop) - : (await queue.ReceiveAsync(1, _options.VisibilityTimeout, stop)).FirstOrDefault(); + messages = owned is not null + ? await owned.ReceiveForProcessingAsync(take, _options.VisibilityTimeout, stop) + : await queue.ReceiveAsync(take, _options.VisibilityTimeout, stop); + + _lastReceiveSucceededAt = _time.GetUtcNow(); + _consecutiveReceiveFailures = 0; + _lastReceiveError = null; errorBackoff = _options.EmptyQueueBackoffMin; } catch (OperationCanceledException) when (stop.IsCancellationRequested) { + slots.Release(take); return; } catch (Exception ex) { - // Transport failures, a missing queue, no permission: log, back off, keep going. - _logger.LogProcessorReceiveFailed(_name, errorBackoff, ex); + slots.Release(take); + _lastReceiveFailedAt = _time.GetUtcNow(); + _consecutiveReceiveFailures++; + _lastReceiveError = ex.Message; + + // Transport failures, a missing queue, no permission: log once, back off, keep going. + if (ex is RequestFailedException { ErrorCode: var code } && code == QueueErrorCode.QueueNotFound) + _logger.LogProcessorQueueNotFound(_name, QueueName, errorBackoff); + else + _logger.LogProcessorReceiveFailed(_name, errorBackoff, ex); + if (!await DelayAsync(errorBackoff, stop)) return; errorBackoff = Min(errorBackoff * 2, _options.ErrorBackoffMax); continue; } - if (message is null) + foreach (var message in messages) StartDelivery(message, owned); + if (messages.Count < take) slots.Release(take - messages.Count); + + if (messages.Count == 0) { if (!await DelayAsync(emptyBackoff, stop)) return; emptyBackoff = Min(emptyBackoff * 2, _options.EmptyQueueBackoffMax); continue; } + // Something arrived, so more may be waiting: ask again as soon as a handler is free. emptyBackoff = _options.EmptyQueueBackoffMin; - await DeliverAsync(message, owned); } } + private async Task WaitWhilePausedAsync(CancellationToken stop) + { + if (_controller is null || !_controller.IsPaused(_clientName)) return; + + if (!_pausedLogged) _logger.LogProcessorPaused(_name); + _pausedLogged = true; + await _controller.WhenResumedAsync(_clientName, stop); + _pausedLogged = false; + _logger.LogProcessorResumed(_name); + } + + /// At least one permit, waiting for it; then as many more as are available now, up to . + private async Task AcquirePermitsAsync(RateLimiter limiter, int wanted, CancellationToken stop) + { + while (true) + { + using var first = await limiter.AcquireAsync(1, stop); + if (first.IsAcquired) break; + + // The limiter's own queue is full: wait as long as it says, or a second. + var retryAfter = first.TryGetMetadata(MetadataName.RetryAfter, out var after) ? after : TimeSpan.FromSeconds(1); + if (!await DelayAsync(retryAfter, stop)) stop.ThrowIfCancellationRequested(); + } + + var permitted = 1; + while (permitted < wanted) + { + using var lease = limiter.AttemptAcquire(1); + if (!lease.IsAcquired) break; + permitted++; + } + + return permitted; + } + + private void StartDelivery(QueueMessage message, IOwnedReceiver? owned) + { + var id = Interlocked.Increment(ref _deliverySequence); + var started = _time.GetTimestamp(); + var delivery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _deliveries[id] = (delivery.Task, started); + + // On its own task, so neither renewal nor a handler that blocks before its first await can hold up the loop. + _ = Task.Run(async () => + { + try + { + await DeliverAsync(message, owned, started); + } + finally + { + _deliveries.TryRemove(id, out _); + _slots!.Release(); + delivery.TrySetResult(); + } + }); + } + private async Task DelayAsync(TimeSpan delay, CancellationToken stop) { try @@ -304,56 +474,91 @@ private async Task DelayAsync(TimeSpan delay, CancellationToken stop) // One delivery: the complete lifecycle of one message, with one exception boundary. Never throws. // ------------------------------------------------------------------------------------------------------------ - private async Task DeliverAsync(QueueMessage message, IOwnedReceiver? deserializer) + private async Task DeliverAsync(QueueMessage message, IOwnedReceiver? owned, long started) { - var start = Stopwatch.GetTimestamp(); var outcome = "abandoned"; + Exception? failure = null; + var lease = message.Lease; + lease.IsLastAttempt |= message.DequeueCount >= _options.MaxDequeueCount; + + using var logScope = _logger.BeginDeliveryScope(QueueName, message.MessageId, message.DequeueCount); var tags = QueueServiceDiagnostics.OperationTags(QueueName, QueueServiceDiagnostics.Messaging.Process); tags.Add(QueueServiceDiagnostics.Names.MessageId, message.MessageId); tags.Add(QueueServiceDiagnostics.Names.DequeueCount, message.DequeueCount); tags.Add(QueueServiceDiagnostics.Messaging.MessageId, message.MessageId); using Activity? activity = QueueServiceDiagnostics.Activities.StartProcessingMessageActivity(tags, message.CreationContext); + lease.DeliveryActivity = activity; - // Armed before anything else — including deserialization — so nothing that happens to this message - // while it is ours can let the lease lapse. + _metrics.ActiveHandlers.Add(1, QueueTags()); + if (message.InsertedOn is { } inserted) + _metrics.TimeInQueue.Record(Math.Max(0, (_time.GetUtcNow() - inserted).TotalSeconds), QueueTags()); + + // Armed before anything else — including downloading and deserializing — so nothing that happens to this + // message while it is ours can let the lease lapse. await using var renewal = _options.RenewVisibility ? LeaseRenewal.Start( - () => message.RenewLockAsync(_options.VisibilityTimeout, CancellationToken.None), + async () => + { + await message.RenewLockAsync(_options.VisibilityTimeout, CancellationToken.None); + _metrics.LeaseRenewals.Add(1, QueueTags()); + }, () => message.IsSettled, TimeSpan.FromTicks(_options.VisibilityTimeout.Ticks / 2), _time, _renewalToken, - ex => _logger.LogProcessorRenewFailed(_name, message.MessageId, ex)) + ex => + { + _metrics.LeaseRenewalFailures.Add(1, QueueTags()); + _logger.LogProcessorRenewFailed(_name, message.MessageId, ex); + }, + _options.MaxRenewalDuration, + () => _logger.LogProcessorRenewalLimitReached(_name, message.MessageId, _options.MaxRenewalDuration!.Value)) + : null; + + using var longRunning = _options.LongRunningHandlerWarning is { } threshold + ? _time.CreateTimer(_ => _logger.LogProcessorHandlerLongRunning(_name, message.MessageId, _time.GetElapsedTime(started), threshold, + _options.MaxRenewalDuration is { } max ? $" until MaxRenewalDuration ({max})" : " for as long as it runs"), null, threshold, Timeout.InfiniteTimeSpan) : null; try { - deserializer?.Deserialize(message); - // A wrapper's own reading of the body (the CloudEvent adapter reads the event's data), which sets Error when - // it fails, so that failure takes the path below with this processor's options snapshot, metric and outcome. - if (message.Error is null) _afterDeserialize?.Invoke(message); + if (!await ReadBodyAsync(message, owned)) + { + // Cancelled while downloading the body at shutdown: give it straight back. + await StopRenewalAsync(renewal); + _logger.LogProcessorHandlerCancelled(_name, message.MessageId); + if (!message.IsSettled) await AbandonSafelyAsync(message, TimeSpan.Zero); + return; + } + + if (message.Error is UnroutableMessageException unroutable) + { + await StopRenewalAsync(renewal); + outcome = await DeadLetterOrAbandonAsync(message, unroutable.Message, DeadLetterCategories.Unroutable); + return; + } if (message.Error is not null && _options.DeadLetterOnDeserializationFailure) { await StopRenewalAsync(renewal); - outcome = await DeadLetterOrAbandonAsync(message, $"the body could not be read: {message.Error.Message}"); + outcome = await DeadLetterOrAbandonAsync(message, $"the body could not be read: {message.Error.Message}", DeadLetterCategories.Deserialization); return; } if (message.DequeueCount > _options.MaxDequeueCount) { await StopRenewalAsync(renewal); - outcome = await DeadLetterOrAbandonAsync(message, $"delivered {message.DequeueCount} times; the limit is {_options.MaxDequeueCount}"); + outcome = await DeadLetterOrAbandonAsync(message, $"delivered {message.DequeueCount} times; the limit is {_options.MaxDequeueCount}", DeadLetterCategories.MaxDequeueCount); return; } try { - // Resolving the handler is inside the boundary: a constructor that throws is a failed delivery too. + // Resolving the handler and the middleware is inside the boundary: a constructor that throws is a failed delivery too. await using var scope = _scopes.CreateAsyncScope(); - var handler = _createHandler is null ? scope.ServiceProvider.GetRequiredService() : _createHandler(scope.ServiceProvider); - await handler.HandleAsync(message, _handlerToken); + var context = new QueueMessageContext(message, scope.ServiceProvider, _clientName, typeof(TMessage), _handlerName, _handlerToken); + await Pipeline(message)(context); } catch (OperationCanceledException) when (_handlerToken.IsCancellationRequested && !message.IsSettled) { @@ -365,10 +570,10 @@ private async Task DeliverAsync(QueueMessage message, IOwnedReceiver? catch (Exception ex) when (!message.IsSettled) { await StopRenewalAsync(renewal); - _logger.LogHandlerFailed(QueueName, message.MessageId, message.DequeueCount, ex); + failure = ex; activity.RecordFailure(ex); _metrics.MessageFailures.Add(1, QueueTags()); - if (!message.IsSettled) await AbandonSafelyAsync(message, RetryDelayFor(message.DequeueCount)); + outcome = await HandleFailureAsync(message, ex); return; } catch (Exception ex) @@ -403,6 +608,7 @@ private async Task DeliverAsync(QueueMessage message, IOwnedReceiver? catch (Exception ex) { _logger.LogDeleteFailed(QueueName, message.MessageId, _options.VisibilityTimeout, ex); + failure = ex; activity.RecordFailure(ex); _metrics.MessageFailures.Add(1, QueueTags()); outcome = "complete_failed"; @@ -423,6 +629,7 @@ private async Task DeliverAsync(QueueMessage message, IOwnedReceiver? // Anything the inner boundaries did not cover: scope disposal, an option callback, a logging sink. // A delivery must never fault silently and leave its message in limbo. _logger.LogProcessorDeliveryFailed(_name, message.MessageId, ex); + failure = ex; activity.RecordFailure(ex); _metrics.MessageFailures.Add(1, QueueTags()); if (!message.IsSettled) await AbandonSafelyAsync(message, QueueProcessorOptions.MaxRetryDelay); @@ -431,12 +638,119 @@ private async Task DeliverAsync(QueueMessage message, IOwnedReceiver? finally { await StopRenewalAsync(renewal); + _metrics.ActiveHandlers.Add(-1, QueueTags()); + var elapsed = _time.GetElapsedTime(started).TotalSeconds; var finalTags = QueueTags(); finalTags.Add(QueueServiceDiagnostics.Names.QueueProcessSuccess, outcome is "completed" or "left_to_handler"); finalTags.Add(QueueServiceDiagnostics.Names.Outcome, outcome); - _metrics.MessageProcessingDuration.Record(Stopwatch.GetElapsedTime(start).TotalSeconds, finalTags); + _metrics.MessageProcessingDuration.Record(elapsed, finalTags); + _metrics.ProcessDuration.Record(elapsed, QueueServiceDiagnostics.MessagingTags(QueueName, QueueServiceDiagnostics.Messaging.Process, QueueServiceDiagnostics.Messaging.Process, failure)); activity?.SetTag(QueueServiceDiagnostics.Names.Outcome, outcome); + lease.DeliveryActivity = null; + } + } + + /// + /// Brings in a body kept outside the queue and reads it, setting or + /// . False when shutdown cancelled the download. + /// + private async Task ReadBodyAsync(QueueMessage message, IOwnedReceiver? owned) + { + if (owned is not null) + { + try + { + if (message.Error is null && await owned.ResolvePayloadAsync(message.Lease, _handlerToken) is { } error) + message.SetBody(default, error); + } + catch (OperationCanceledException) when (_handlerToken.IsCancellationRequested) + { + return false; + } + } + + if (message.Error is null) + { + if (_hooks.ReadBody is { } read) read(message); + else owned?.Deserialize(message); + } + + // A wrapper's own reading of the body (the CloudEvent adapter reads the event's data), which sets Error when it + // fails, so that failure takes the dead-letter path with this processor's options snapshot, metric and outcome. + if (message.Error is null) _hooks.AfterDeserialize?.Invoke(message); + return true; + } + + private QueueMessageDelegate Pipeline(QueueMessage message) + { + QueueMessageDelegate next = async context => + { + var handler = _hooks.CreateHandler is { } create ? create(context.Services) : context.Services.GetRequiredService(); + await handler.HandleAsync(message, context.CancellationToken); + }; + + if (_hooks.Middleware is not { Count: > 0 } middleware) return next; + + for (var i = middleware.Count - 1; i >= 0; i--) + { + var resolve = middleware[i]; + var inner = next; + next = context => resolve(context.Services).InvokeAsync(context, inner); + } + + return next; + } + + /// The error policy: , then , then retry. + private async Task HandleFailureAsync(QueueMessage message, Exception ex) + { + var action = DecideOnFailure(message, ex); + switch (action) + { + case HandlerErrorAction.DeadLetter: + _logger.LogHandlerFailedPolicy(QueueName, message.MessageId, ex.GetType().Name, action, ex); + return await DeadLetterOrAbandonAsync(message, $"the handler threw {ex.GetType().Name}: {ex.Message}", DeadLetterCategories.Handler); + + case HandlerErrorAction.Complete: + _logger.LogHandlerFailedPolicy(QueueName, message.MessageId, ex.GetType().Name, action, ex); + try + { + await message.CompleteAsync(CancellationToken.None); + return "completed_after_failure"; + } + catch (Exception completeFailure) + { + _logger.LogDeleteFailed(QueueName, message.MessageId, _options.VisibilityTimeout, completeFailure); + return "complete_failed"; + } + + default: + var delay = RetryDelayFor(message.DequeueCount); + if (message.IsLastAttempt) _logger.LogHandlerFailedLastAttempt(QueueName, message.MessageId, message.DequeueCount, ex); + else _logger.LogHandlerFailed(LogLevel.Warning, QueueName, message.MessageId, message.DequeueCount, delay, ex); + if (!message.IsSettled) await AbandonSafelyAsync(message, delay); + return "abandoned"; + } + } + + private HandlerErrorAction DecideOnFailure(QueueMessage message, Exception ex) + { + foreach (var type in _options.DeadLetterExceptionTypes) + { + if (type.IsInstanceOfType(ex)) return HandlerErrorAction.DeadLetter; + } + + if (_options.OnHandlerError is not { } decide) return HandlerErrorAction.Retry; + + try + { + return decide(new HandlerErrorContext(ex, message)); + } + catch (Exception callbackFailure) + { + _logger.LogProcessorErrorPolicyFailed(_name, message.MessageId, callbackFailure); + return HandlerErrorAction.Retry; } } @@ -459,11 +773,11 @@ private TimeSpan RetryDelayFor(long dequeueCount) return delay < TimeSpan.Zero ? TimeSpan.Zero : Min(delay, QueueProcessorOptions.MaxVisibility); } - private async Task DeadLetterOrAbandonAsync(QueueMessage message, string reason) + private async Task DeadLetterOrAbandonAsync(QueueMessage message, string reason, string category) { try { - await message.DeadLetterAsync(reason, CancellationToken.None); + await message.DeadLetterAsync(reason, category, CancellationToken.None); _metrics.MessageFailures.Add(1, QueueTags()); return "dead_lettered"; } @@ -490,3 +804,6 @@ private async ValueTask AbandonSafelyAsync(QueueMessage message, TimeS private TagList QueueTags() => QueueServiceDiagnostics.MetricTags(QueueName); } + +/// A message a routed processor has no handler for; always dead-lettered, as no retry can find it one. +internal sealed class UnroutableMessageException(string message) : DeserializationException(message); diff --git a/src/AzureStorage.QueueService/QueueProcessorController.cs b/src/AzureStorage.QueueService/QueueProcessorController.cs new file mode 100644 index 0000000..8988116 --- /dev/null +++ b/src/AzureStorage.QueueService/QueueProcessorController.cs @@ -0,0 +1,142 @@ +using System.Collections.Concurrent; + +namespace AzureStorage.QueueService; + +/// +/// Pauses and resumes processors, and reports how each is doing. Inject it anywhere: an admin endpoint, a circuit +/// breaker that pauses a client while a downstream service is down, or a health check (the +/// AzureStorage.QueueService.HealthChecks package builds on ). +/// +/// +/// Pausing stops a client's processors from receiving; messages they already hold are delivered and settled as usual, +/// and nothing is lost. A pause made before the host starts holds from the start. Clients are named as registered; the +/// default client's name is the empty string. +/// +public interface IQueueProcessorController +{ + /// A snapshot of every processor's state. + IReadOnlyList GetStatus(); + + /// Stops every processor on from receiving until . + /// No client with that name is registered. + void Pause(string clientName); + + /// Lets the processors on receive again. + /// No client with that name is registered. + void Resume(string clientName); + + /// True when 's processors are paused. + bool IsPaused(string clientName); +} + +/// Where a processor is in its life. +public enum QueueProcessorState +{ + /// Registered; waiting for the host to finish starting. + NotStarted, + + /// Receiving and delivering. + Running, + + /// Paused through : not receiving, finishing what it holds. + Paused, + + /// The host is shutting down; draining. + Stopping, + + /// Stopped. + Stopped, +} + +/// One processor's state, from . +/// The client it belongs to; empty for the default client. +/// The queue it reads. +/// The handler, as it appears in logs. +/// The type it reads bodies as. +/// Where it is in its life. +/// Messages being delivered right now. +/// The most it delivers at once. +/// When a receive last succeeded, empty or not; null before the first. +/// When a receive last failed; null if none has. +/// Receives that have failed in a row; 0 after a success. +/// The last receive failure's message, while failures continue; null after a success. +/// How long the longest-running delivery has been running; null when none is. +public sealed record QueueProcessorStatus( + string ClientName, + string QueueName, + string HandlerName, + Type MessageType, + QueueProcessorState State, + int ActiveHandlers, + int MaxConcurrentMessages, + DateTimeOffset? LastReceiveSucceededAt, + DateTimeOffset? LastReceiveFailedAt, + int ConsecutiveReceiveFailures, + string? LastReceiveError, + TimeSpan? OldestHandlerAge); + +/// What the controller needs from a processor. +internal interface IControlledProcessor +{ + string ClientName { get; } + + QueueProcessorStatus GetStatus(); +} + +internal sealed class QueueProcessorController : IQueueProcessorController +{ + private readonly QueueClientSettingsRegistry _registry; + private readonly ConcurrentDictionary _processors = new(); + private readonly ConcurrentDictionary _paused = new(StringComparer.Ordinal); + + public QueueProcessorController(QueueClientSettingsRegistry registry) + { + _registry = registry; + } + + public IReadOnlyList GetStatus() => _processors.Keys.Select(p => p.GetStatus()).ToList(); + + public void Pause(string clientName) + { + ThrowIfUnknown(clientName); + _paused.TryAdd(clientName, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + } + + public void Resume(string clientName) + { + ThrowIfUnknown(clientName); + if (_paused.TryRemove(clientName, out var resumed)) resumed.TrySetResult(); + } + + public bool IsPaused(string clientName) => _paused.ContainsKey(clientName); + + /// Completes when is not paused (at once if it is not paused now). + public Task WhenResumedAsync(string clientName, CancellationToken cancellationToken) => + _paused.TryGetValue(clientName, out var paused) ? paused.Task.WaitAsync(cancellationToken) : Task.CompletedTask; + + public IDisposable Register(IControlledProcessor processor) + { + _processors[processor] = 0; + return new Registration(() => _processors.TryRemove(processor, out _)); + } + + private void ThrowIfUnknown(string clientName) + { + ArgumentNullException.ThrowIfNull(clientName); + var known = clientName.Length == 0 ? _registry.HasDefaultClient : _registry.Contains(clientName); + if (known) return; + + var registered = _registry.ClientNames.Select(n => $"'{n}'").Concat(_registry.HasDefaultClient ? ["'' (the default client)"] : []).ToList(); + throw new InvalidOperationException($"No queue client named '{clientName}' has been registered (registered: {(registered.Count == 0 ? "none" : string.Join(", ", registered))})."); + } + + private sealed class Registration(Action dispose) : IDisposable + { + private int _done; + + public void Dispose() + { + if (Interlocked.Exchange(ref _done, 1) == 0) dispose(); + } + } +} diff --git a/src/AzureStorage.QueueService/QueueProcessorOptions.cs b/src/AzureStorage.QueueService/QueueProcessorOptions.cs index d8239aa..078a2f9 100644 --- a/src/AzureStorage.QueueService/QueueProcessorOptions.cs +++ b/src/AzureStorage.QueueService/QueueProcessorOptions.cs @@ -1,9 +1,17 @@ +using System.Threading.RateLimiting; + namespace AzureStorage.QueueService; /// -/// How a processor registered with AddProcessor<TMessage, THandler>() runs. The processor is a set of -/// workers; each receives one message, delivers it, and goes back for the next. +/// How a processor registered with AddProcessor<TMessage, THandler>() runs. One receive loop takes messages +/// for whichever handlers are free (up to at once) and delivers each in its own +/// scope, with its own lease renewal. /// +/// +/// Everything except the callbacks (, ), +/// and can be bound from configuration with the +/// AddProcessor<TMessage, THandler>(IConfiguration) overload. +/// public sealed class QueueProcessorOptions { internal static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(5); @@ -14,7 +22,14 @@ public sealed class QueueProcessorOptions /// The longest visibility timeout the service accepts. internal static readonly TimeSpan MaxVisibility = TimeSpan.FromDays(7); - /// How many messages may be handled at once: one worker each, with its own scope and its own lease renewal. + private readonly List _deadLetterOn = new(); + + + /// + /// How many messages may be handled at once, each in its own scope with its own lease renewal. The processor receives + /// only as many messages as there are free handlers (up to 32 per request), so it never holds a message no handler is + /// free for. + /// public int MaxConcurrentMessages { get; set; } = 1; /// @@ -30,6 +45,19 @@ public sealed class QueueProcessorOptions /// public bool RenewVisibility { get; set; } = true; + /// + /// The longest time a lease is renewed for one delivery; null (the default) renews for as long as the handler runs. + /// Past it, renewal stops and the message becomes visible to other consumers when its lease lapses, so a hung handler + /// cannot hide its message for ever. Like Service Bus's MaxAutoLockRenewalDuration. + /// + public TimeSpan? MaxRenewalDuration { get; set; } + + /// + /// Logs a warning, once per delivery, when a handler has been running this long; null turns it off. Default 10 + /// minutes. The queue_processor_oldest_handler_age_seconds gauge shows the same thing as a number. + /// + public TimeSpan? LongRunningHandlerWarning { get; set; } = TimeSpan.FromMinutes(10); + /// /// Complete the message when the handler returns. Off, a message the handler returns without settling stays /// hidden until its lease lapses and is then delivered again. Either way, a message whose handler throws is abandoned for retry. @@ -55,11 +83,38 @@ public sealed class QueueProcessorOptions }; /// - /// How long a worker waits after an empty receive before polling again; doubles up to - /// and resets on the next message. Each worker polls on its own, so an idle processor with N workers makes N requests per interval. + /// Decides what happens to a message whose handler threw, when no rule matched: + /// retry it (the default without a callback), dead-letter it, or complete it. If the callback throws, the message is retried. + /// + public Func? OnHandlerError { get; set; } + + /// + /// Dead-letters a message straight away, without retrying it, when its handler throws + /// or an exception derived from it: one that retrying cannot fix, such as a validation failure. Checked before + /// . + /// + public QueueProcessorOptions DeadLetterOn() where TException : Exception + { + if (!_deadLetterOn.Contains(typeof(TException))) _deadLetterOn.Add(typeof(TException)); + return this; + } + + internal IReadOnlyList DeadLetterExceptionTypes => _deadLetterOn; + + /// + /// Limits how fast the processor takes messages: one permit per message, acquired before each receive, and a receive + /// asks for no more messages than it has permits for. For example a TokenBucketRateLimiter to stay under a + /// downstream API's limit. The processor does not dispose it. + /// + public RateLimiter? RateLimiter { get; set; } + + /// + /// How long the receive loop waits after an empty receive before polling again; doubles up to + /// and resets on the next message. One loop polls per processor, however many messages it handles at once. /// public TimeSpan EmptyQueueBackoffMin { get; set; } = TimeSpan.FromSeconds(1); + /// The longest wait between polls of an empty queue. public TimeSpan EmptyQueueBackoffMax { get; set; } = TimeSpan.FromSeconds(30); /// The longest wait after a failed receive; the wait doubles from and resets on the next success. @@ -84,9 +139,15 @@ internal void Validate(string processorName) if (ErrorBackoffMax < EmptyQueueBackoffMin) Fail(processorName, "ErrorBackoffMax must be at least EmptyQueueBackoffMin."); if (ShutdownDrainTimeout < TimeSpan.Zero) Fail(processorName, "ShutdownDrainTimeout cannot be negative."); if (RetryDelay is null) Fail(processorName, "RetryDelay must be set."); + if (MaxRenewalDuration is { } maxRenewal && maxRenewal <= TimeSpan.Zero) Fail(processorName, "MaxRenewalDuration must be positive, or null to renew for as long as the handler runs."); + if (LongRunningHandlerWarning is { } warning && warning <= TimeSpan.Zero) Fail(processorName, "LongRunningHandlerWarning must be positive, or null for no warning."); // Anything a timer waits on must be within what the timer APIs accept, or the wait throws at runtime instead. - foreach (var (name, value) in new[] { (nameof(EmptyQueueBackoffMax), EmptyQueueBackoffMax), (nameof(ErrorBackoffMax), ErrorBackoffMax), (nameof(ShutdownDrainTimeout), ShutdownDrainTimeout) }) + foreach (var (name, value) in new[] + { + (nameof(EmptyQueueBackoffMax), EmptyQueueBackoffMax), (nameof(ErrorBackoffMax), ErrorBackoffMax), (nameof(ShutdownDrainTimeout), ShutdownDrainTimeout), + (nameof(MaxRenewalDuration), MaxRenewalDuration ?? TimeSpan.Zero), (nameof(LongRunningHandlerWarning), LongRunningHandlerWarning ?? TimeSpan.Zero), + }) { if (value > MaxTimer) Fail(processorName, $"{name} must be no longer than {MaxTimer.TotalDays:0} days (it is {value})."); } @@ -94,3 +155,32 @@ internal void Validate(string processorName) private static void Fail(string processorName, string problem) => throw new InvalidOperationException($"{processorName}: {problem}"); } + +/// What a processor does with a message whose handler threw. +public enum HandlerErrorAction +{ + /// Abandon it with , to be delivered again; dead-lettered once it passes . + Retry, + + /// Move it to the poison queue now. + DeadLetter, + + /// Delete it: the failure does not matter, or the handler has dealt with it. + Complete, +} + +/// What is told about a failed delivery. +public sealed class HandlerErrorContext +{ + internal HandlerErrorContext(Exception exception, IQueueMessage message) + { + Exception = exception; + Message = message; + } + + /// What the handler threw. + public Exception Exception { get; } + + /// The message, still unsettled. Its says whether a retry is left. + public IQueueMessage Message { get; } +} diff --git a/src/AzureStorage.QueueService/QueueTransport.cs b/src/AzureStorage.QueueService/QueueTransport.cs new file mode 100644 index 0000000..2bdc198 --- /dev/null +++ b/src/AzureStorage.QueueService/QueueTransport.cs @@ -0,0 +1,220 @@ +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using SdkQueueMessage = Azure.Storage.Queues.Models.QueueMessage; + +namespace AzureStorage.QueueService; + +/// +/// The raw calls one queue answers: bytes in, bytes out, no serialization, envelopes or telemetry. +/// does everything else once, over either the Azure SDK () or an in-memory queue +/// (), so a test against the in-memory queue runs the code production runs. +/// +internal interface IQueueTransport +{ + string Name { get; } + + /// The queue's URI, for the spans' server attributes; null for an in-memory queue. + Uri? Uri { get; } + + /// True when what is sent is Base64-encoded on the way, which makes it a third bigger against the size limit. + bool Base64Encoded { get; } + + ValueTask CreateIfNotExistsAsync(IDictionary? metadata, CancellationToken cancellationToken); + + /// In the service's terms: -1 second for never. + ValueTask SendAsync(BinaryData body, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken); + + /// Null for the service's default of 30 seconds. + ValueTask> ReceiveAsync(int maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken); + + ValueTask> PeekAsync(int maxMessages, CancellationToken cancellationToken); + + ValueTask DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken); + + ValueTask UpdateVisibilityAsync(string messageId, string popReceipt, TimeSpan visibilityTimeout, CancellationToken cancellationToken); + + ValueTask ClearAsync(CancellationToken cancellationToken); + + ValueTask GetPropertiesAsync(CancellationToken cancellationToken); +} + +/// +/// A message as the transport returns it. marks one the SDK could not Base64-decode, whose +/// body is the text as it was on the queue. A peeked message has an empty pop receipt. +/// +internal sealed record TransportMessage(QueueMessageInfo Info, BinaryData Body, bool Undecodable = false); + +internal sealed class SdkQueueTransport : IQueueTransport +{ + private readonly QueueClient _client; + + public SdkQueueTransport(QueueClient client, bool base64Encoded) + { + _client = client; + Base64Encoded = base64Encoded; + } + + public string Name => _client.Name; + + public Uri? Uri => _client.Uri; + + public bool Base64Encoded { get; } + + public async ValueTask CreateIfNotExistsAsync(IDictionary? metadata, CancellationToken cancellationToken) => + await _client.CreateIfNotExistsAsync(metadata, cancellationToken); + + public async ValueTask SendAsync(BinaryData body, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken) => + await _client.SendMessageAsync(body, visibilityTimeout, timeToLive, cancellationToken); + + public async ValueTask> ReceiveAsync(int maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) + { + // Messages the SDK could not Base64-decode are raised as events during the call; collect them so they come back + // as messages with Error set instead of failing the whole batch. + SdkQueueMessage[] received; + var undecodable = DecodingFailures.Begin(); + try + { + received = await _client.ReceiveMessagesAsync(maxMessages, visibilityTimeout, cancellationToken); + } + finally + { + DecodingFailures.End(); + } + + var messages = new List(received.Length + undecodable.Count); + foreach (var m in received) + messages.Add(new TransportMessage(Info(m), m.Body ?? BinaryData.FromString(m.MessageText ?? string.Empty))); + + foreach (var failure in undecodable) + { + if (failure.ReceivedMessage is { } m) + messages.Add(new TransportMessage(Info(m), BinaryData.FromString(m.MessageText ?? string.Empty), Undecodable: true)); + } + + return messages; + } + + public async ValueTask> PeekAsync(int maxMessages, CancellationToken cancellationToken) + { + PeekedMessage[] peeked; + var undecodable = DecodingFailures.Begin(); + try + { + peeked = await _client.PeekMessagesAsync(maxMessages, cancellationToken); + } + finally + { + DecodingFailures.End(); + } + + var messages = new List(peeked.Length + undecodable.Count); + foreach (var m in peeked) + messages.Add(new TransportMessage(Info(m), m.Body ?? BinaryData.FromString(m.MessageText ?? string.Empty))); + + foreach (var failure in undecodable) + { + if (failure.PeekedMessage is { } m) + messages.Add(new TransportMessage(Info(m), BinaryData.FromString(m.MessageText ?? string.Empty), Undecodable: true)); + } + + return messages; + } + + public async ValueTask DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken) => + await _client.DeleteMessageAsync(messageId, popReceipt, cancellationToken); + + public async ValueTask UpdateVisibilityAsync(string messageId, string popReceipt, TimeSpan visibilityTimeout, CancellationToken cancellationToken) + { + // No message text: only the visibility changes. + UpdateReceipt receipt = await _client.UpdateMessageAsync(messageId, popReceipt, (string?)null, visibilityTimeout, cancellationToken); + return new VisibilityReceipt(receipt.PopReceipt, receipt.NextVisibleOn); + } + + public async ValueTask ClearAsync(CancellationToken cancellationToken) => await _client.ClearMessagesAsync(cancellationToken); + + public async ValueTask GetPropertiesAsync(CancellationToken cancellationToken) => + (await _client.GetPropertiesAsync(cancellationToken)).Value; + + private QueueMessageInfo Info(SdkQueueMessage m) => + new(m.MessageId, m.PopReceipt, m.DequeueCount, m.InsertedOn, m.ExpiresOn, m.NextVisibleOn) { QueueName = _client.Name }; + + private QueueMessageInfo Info(PeekedMessage m) => + new(m.MessageId, string.Empty, m.DequeueCount, m.InsertedOn, m.ExpiresOn, null) { QueueName = _client.Name }; +} + +internal sealed class InMemoryQueueTransport : IQueueTransport +{ + private static readonly TimeSpan ServiceDefaultVisibility = TimeSpan.FromSeconds(30); + + private readonly InMemoryQueue _queue; + + /// What the client's options say, so the size limit is checked as it would be in production. + public InMemoryQueueTransport(InMemoryQueue queue, bool base64Encoded) + { + _queue = queue; + Base64Encoded = base64Encoded; + } + + public string Name => _queue.Name; + + public Uri? Uri => null; + + public bool Base64Encoded { get; } + + public ValueTask CreateIfNotExistsAsync(IDictionary? metadata, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _queue.CreateIfNotExists(metadata); + return ValueTask.CompletedTask; + } + + public ValueTask SendAsync(BinaryData body, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(_queue.Send(body, visibilityTimeout, timeToLive)); + } + + public ValueTask> ReceiveAsync(int maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + IReadOnlyList messages = _queue.Receive(maxMessages, visibilityTimeout ?? ServiceDefaultVisibility) + .Select(m => new TransportMessage(new QueueMessageInfo(m.MessageId, m.PopReceipt, m.DequeueCount, m.InsertedOn, m.ExpiresOn, m.NextVisibleOn) { QueueName = _queue.Name }, m.Body)) + .ToList(); + return new(messages); + } + + public ValueTask> PeekAsync(int maxMessages, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + IReadOnlyList messages = _queue.Peek(maxMessages) + .Select(m => new TransportMessage(new QueueMessageInfo(m.MessageId, string.Empty, m.DequeueCount, m.InsertedOn, m.ExpiresOn, null) { QueueName = _queue.Name }, m.Body)) + .ToList(); + return new(messages); + } + + public ValueTask DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _queue.Delete(messageId, popReceipt); + return ValueTask.CompletedTask; + } + + public ValueTask UpdateVisibilityAsync(string messageId, string popReceipt, TimeSpan visibilityTimeout, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(_queue.UpdateVisibility(messageId, popReceipt, visibilityTimeout)); + } + + public ValueTask ClearAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _queue.Clear(); + return ValueTask.CompletedTask; + } + + public ValueTask GetPropertiesAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(_queue.GetProperties()); + } +} diff --git a/src/AzureStorage.QueueService/Routing.cs b/src/AzureStorage.QueueService/Routing.cs new file mode 100644 index 0000000..4321ad0 --- /dev/null +++ b/src/AzureStorage.QueueService/Routing.cs @@ -0,0 +1,164 @@ +using Azure.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace AzureStorage.QueueService; + +/// +/// The handlers of a routed processor (AddRoutedProcessor): one queue carrying several message types, each read +/// as its own type and handed to its own handler, chosen by the message's +/// header. Senders set the header with or . +/// +public sealed class QueueMessageRoutes +{ + private readonly IServiceCollection _services; + + internal QueueMessageRoutes(IServiceCollection services) + { + _services = services; + } + + internal Dictionary Routes { get; } = new(StringComparer.Ordinal); + + /// + /// Sends messages whose type header is (by default 's + /// name, which is what sends) to . + /// + public QueueMessageRoutes Route(string? messageType = null) + where THandler : class, IQueueMessageHandler + { + var key = messageType ?? typeof(TMessage).Name; + if (!Routes.TryAdd(key, new MessageRoute())) + throw new InvalidOperationException($"Message type '{key}' is routed twice; each type can have one handler."); + + _services.TryAddScoped(); + return this; + } + + internal static string Describe(IEnumerable keys) => string.Join(", ", keys.Select(k => $"'{k}'")); +} + +/// +/// The handlers of a CloudEvent router (AddCloudEventRouter): one queue of CloudEvents, such as an Event Grid +/// subscription with several event types, each event's data read as its own type and handed to its own handler, chosen +/// by the event's type. +/// +public sealed class CloudEventRoutes +{ + private readonly IServiceCollection _services; + + internal CloudEventRoutes(IServiceCollection services) + { + _services = services; + } + + internal Dictionary Routes { get; } = new(StringComparer.Ordinal); + + /// Sends events whose type is to , with their data read as . + public CloudEventRoutes Route(string eventType) + where THandler : class, ICloudEventHandler + { + ArgumentException.ThrowIfNullOrEmpty(eventType); + if (!Routes.TryAdd(eventType, new CloudEventRoute())) + throw new InvalidOperationException($"CloudEvent type '{eventType}' is routed twice; each type can have one handler."); + + _services.TryAddScoped(); + return this; + } +} + +internal interface IMessageRoute +{ + (object? Body, DeserializationException? Error) Read(IQueueMessageSerializer serializer, BinaryData raw); + + ValueTask HandleAsync(QueueMessage message, IServiceProvider services, CancellationToken cancellationToken); +} + +internal sealed class MessageRoute : IMessageRoute + where THandler : class, IQueueMessageHandler +{ + public (object? Body, DeserializationException? Error) Read(IQueueMessageSerializer serializer, BinaryData raw) => + MessageCodec.TryDeserialize(serializer, raw); + + // The handler gets a view typed as its own message, sharing the received message's receipt and settlement. + public ValueTask HandleAsync(QueueMessage message, IServiceProvider services, CancellationToken cancellationToken) => + services.GetRequiredService().HandleAsync(message.As(message.Body is TMessage body ? body : default, message.Error), cancellationToken); +} + +/// The handler behind a routed processor: hands each message to the route its read step chose. +internal sealed class RoutingHandler(IServiceProvider services) : IQueueMessageHandler +{ + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) => + ((IMessageRoute)message.HandlerData!).HandleAsync(message, services, cancellationToken); + + /// The read step: choose the route by the type header, and read the body as that route's type. + public static void Read(QueueMessage message, IReadOnlyDictionary routes, IQueueMessageSerializer serializer) + { + if (!message.Headers.TryGetValue(QueueMessageHeaders.MessageType, out var type)) + { + message.SetBody(null, new UnroutableMessageException( + $"Message {message.MessageId} has no '{QueueMessageHeaders.MessageType}' header, so the routed processor cannot choose a handler. Send it with SendOptions.MessageType, or set SendMessageType on the sending client.")); + return; + } + + if (!routes.TryGetValue(type, out var route)) + { + message.SetBody(null, new UnroutableMessageException( + $"Message {message.MessageId} has type '{type}', which no handler is routed for (routed: {QueueMessageRoutes.Describe(routes.Keys)}).")); + return; + } + + message.HandlerData = route; + var (body, error) = route.Read(serializer, message.RawBody); + message.SetBody(body, error); + } +} + +internal interface ICloudEventRoute +{ + void ReadData(QueueMessage message, IQueueMessageSerializer serializer); + + ValueTask HandleAsync(QueueMessage message, object? data, IServiceProvider services, CancellationToken cancellationToken); +} + +internal sealed class CloudEventRoute : ICloudEventRoute + where THandler : class, ICloudEventHandler +{ + public void ReadData(QueueMessage message, IQueueMessageSerializer serializer) => + CloudEventHandlerAdapter.ReadData(message, serializer); + + public ValueTask HandleAsync(QueueMessage message, object? data, IServiceProvider services, CancellationToken cancellationToken) => + services.GetRequiredService().HandleAsync(message, data is TData typed ? typed : default, cancellationToken); +} + +/// The handler behind a CloudEvent router: hands each event to the route its type chose. +internal sealed class CloudEventRoutingHandler(IServiceProvider services) : IQueueMessageHandler +{ + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) + { + var routed = (RoutedCloudEvent)message.HandlerData!; + return routed.Route.HandleAsync(message, routed.Data, services, cancellationToken); + } + + /// The read step after the envelope deserialized: choose the route by the event's type, and read its data. + public static void Read(QueueMessage message, IReadOnlyDictionary routes, IQueueMessageSerializer serializer) + { + if (message.Body is not { } cloudEvent) + { + message.SetBody(null, new DeserializationException("The queue message is JSON null, not a CloudEvent.")); + return; + } + + if (!routes.TryGetValue(cloudEvent.Type, out var route)) + { + message.SetBody(cloudEvent, new UnroutableMessageException( + $"CloudEvent '{cloudEvent.Id}' has type '{cloudEvent.Type}', which no handler is routed for (routed: {QueueMessageRoutes.Describe(routes.Keys)}).")); + return; + } + + route.ReadData(message, serializer); + message.HandlerData = new RoutedCloudEvent(route, message.Error is null ? message.HandlerData : null); + } + + private sealed record RoutedCloudEvent(ICloudEventRoute Route, object? Data); +} diff --git a/src/AzureStorage.QueueService/SdkQueueClientFactory.cs b/src/AzureStorage.QueueService/SdkQueueClientFactory.cs index 77e75d6..e29a156 100644 --- a/src/AzureStorage.QueueService/SdkQueueClientFactory.cs +++ b/src/AzureStorage.QueueService/SdkQueueClientFactory.cs @@ -31,7 +31,7 @@ public QueueClient CreateClient(string clientDescription, QueueClientSettings se QueueClient Create(QueueClientOptions? options) => settings.UsesConnectionString ? new QueueClient(settings.ConnectionString, settings.QueueName, options) - : new QueueClient(settings.EndpointUri, settings.TokenCredential, options); + : new QueueClient(settings.QueueUri, settings.TokenCredential, options); if (settings.ClientOptions is not { MessageEncoding: QueueMessageEncoding.Base64 } callerOptions) return Create(settings.ClientOptions); diff --git a/src/AzureStorage.QueueService/ServiceCollectionExtensions.cs b/src/AzureStorage.QueueService/ServiceCollectionExtensions.cs index c3670c0..e55dcc2 100644 --- a/src/AzureStorage.QueueService/ServiceCollectionExtensions.cs +++ b/src/AzureStorage.QueueService/ServiceCollectionExtensions.cs @@ -6,15 +6,19 @@ namespace AzureStorage.QueueService; +/// Registers the library with a service collection. public static class ServiceCollectionExtensions { /// /// Registers the queue client factory and the clients configured in . /// Can be called more than once; later calls add clients to the same registration. /// + /// The application's services. + /// Registers clients, and settings shared by all of them. /// /// JSON options shared by every client that does not set its own serializer with UseSerializer / - /// UseJsonSerializer. When omitted, an you registered yourself is used if present. + /// UseJsonSerializer. When omitted, an you registered yourself is used if + /// present, and otherwise System.Text.Json's defaults with property names read case-insensitively. /// public static IServiceCollection AddAzureStorageQueueClient(this IServiceCollection services, Action configure, JsonSerializerOptions? serializerOptions = null) { @@ -25,6 +29,8 @@ public static IServiceCollection AddAzureStorageQueueClient(this IServiceCollect services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static provider => provider.GetRequiredService()); // Through the host's meter factory when there is one, so the meter is scoped to and disposed with this container. services.TryAddSingleton(static provider => QueueServiceMetrics.Create(provider.GetService())); @@ -33,11 +39,28 @@ public static IServiceCollection AddAzureStorageQueueClient(this IServiceCollect else services.TryAddSingleton(new JsonQueueMessageSerializer()); + // Any client asked for by a name — registered or not — goes through the factory, so a typo or a missing + // registration fails with the factory's message listing the registered names, not the container's generic one. + // A registered name also has its own registration (AddClient); this is the fallback for the rest. + services.TryAddKeyedSingleton(KeyedService.AnyKey, static (provider, key) => provider.GetRequiredService().GetQueue(ClientName(key))); + services.TryAddKeyedSingleton(KeyedService.AnyKey, static (provider, key) => provider.GetRequiredKeyedService(key)); + services.TryAddKeyedSingleton(KeyedService.AnyKey, static (provider, key) => provider.GetRequiredKeyedService(key)); + services.TryAddKeyedSingleton(KeyedService.AnyKey, static (provider, key) => provider.GetRequiredService().GetDeadLetterQueue(ClientName(key))); + + // The same for the default client: injecting one when none is registered says so, and names the clients that are. + services.TryAddSingleton(static provider => provider.GetRequiredService().GetQueue()); + services.TryAddSingleton(static provider => provider.GetRequiredService()); + services.TryAddSingleton(static provider => provider.GetRequiredService()); + services.TryAddSingleton(static provider => provider.GetRequiredService().GetDeadLetterQueue()); + configure(new QueueClientSettingsBuilder(services, registry)); return services; } + private static string ClientName(object? key) => key as string + ?? throw new InvalidOperationException($"Queue clients are keyed by their registered name, a string; [FromKeyedServices] was given {(key is null ? "null" : $"a {key.GetType().Name}")}."); + /// /// For tests: makes every registered queue client an in-memory queue — default and named, injected or from /// , and the queue each processor reads — so a handler can be exercised end to @@ -49,13 +72,15 @@ public static IServiceCollection AddAzureStorageQueueClient(this IServiceCollect /// The queues follow the service's rules: messages are serialized with each client's serializer; a received message /// is hidden for its visibility timeout, its dequeue count goes up and it gets a new pop receipt; settling it with an /// outdated receipt throws with status 404; are - /// honoured. Time comes from the registered , or the system clock, so a fake clock drives - /// visibility, retry delays and expiry. + /// honoured, and so is the 64 KiB size limit. Time comes from the registered , or the system + /// clock, so a fake clock drives visibility, retry delays and expiry. Sends, receives and settlements emit the same + /// spans and metrics as against the service. /// /// - /// Clients that name the same queue share it, as on the service. To see what was dead-lettered, register a client - /// whose is the poison queue's name (the queue name plus -poison - /// unless says otherwise) and peek it. + /// Clients that name the same queue share it, as on the service. To see what was dead-lettered, inject + /// for the client, or register a client whose + /// is the poison queue's name (the queue name plus -poison unless + /// says otherwise) and peek it. /// /// /// Connection details are neither needed nor checked, here or by . @@ -72,6 +97,7 @@ public static IServiceCollection AddInMemoryQueues(this IServiceCollection servi return services; } + /// Turns the optional spans for individual sends and receives on or off. public static IServiceCollection ConfigureQueueServiceTelemetry(this IServiceCollection services, Action options) { services.Configure(options); @@ -79,7 +105,7 @@ public static IServiceCollection ConfigureQueueServiceTelemetry(this IServiceCol } // One registry per service collection, so two hosts in one process never share client settings. - private static QueueClientSettingsRegistry GetOrAddRegistry(IServiceCollection services) + internal static QueueClientSettingsRegistry GetOrAddRegistry(IServiceCollection services) { foreach (var descriptor in services) { diff --git a/src/AzureStorage.QueueService/StorageQueue.cs b/src/AzureStorage.QueueService/StorageQueue.cs index 1965238..80cef4e 100644 --- a/src/AzureStorage.QueueService/StorageQueue.cs +++ b/src/AzureStorage.QueueService/StorageQueue.cs @@ -4,32 +4,67 @@ using AzureStorage.QueueService.Telemetry; using Microsoft.Extensions.Logging; using System.Diagnostics; -using SdkQueueMessage = Azure.Storage.Queues.Models.QueueMessage; namespace AzureStorage.QueueService; +/// Per-client behaviour of a that is not about the message format. +/// What a receive uses when the call does not say; null for the service default. +/// Create the queue on the first operation. +/// How long a dead-lettered message lives on the poison queue; null for never. +/// The clock for the dead-letter timestamp. +internal sealed record StorageQueueOptions( + TimeSpan? DefaultVisibilityTimeout = null, + bool CreateIfNotExists = false, + TimeSpan? PoisonTimeToLive = null, + TimeProvider? Time = null); + /// -/// One queue over one SDK client. Thread-safe and long-lived; the factory keeps one per registered name. -/// Also implements the operations a uses to settle itself. +/// One queue: sending, receiving, settling, dead-lettering and administration, over a raw transport (the Azure SDK, or +/// an in-memory queue in tests). Thread-safe and long-lived; the factory keeps one per registered name. Also implements +/// the operations a uses to settle itself. /// -internal sealed class StorageQueue : IStorageQueue, IQueueMessageOperations, IOwnedReceiver +internal sealed class StorageQueue : IStorageQueue, IQueueMessageOperations, IDeadLetterOperations, IOwnedReceiver { - private readonly QueueClient _client; - private readonly IQueueMessageSerializer _serializer; + /// The longest dead-letter reason kept with the message; the rest is in the log. + private const int MaxReasonLength = 1024; + + private readonly IQueueTransport _transport; + private readonly Func? _poisonQueueFactory; private readonly ILogger _logger; private readonly QueueServiceTelemetrySettings _telemetry; private readonly QueueServiceMetrics _metrics; - private readonly bool _propagateTraceContext; + private readonly StorageQueueOptions _options; + private readonly TimeProvider _time; private readonly string? _serverAddress; private readonly int _serverPort; - private readonly TimeSpan? _defaultVisibilityTimeout; - private readonly bool _createIfNotExists; - private readonly Func? _poisonQueueFactory; private readonly SemaphoreSlim _createLock = new(1, 1); private volatile bool _queueEnsured; private volatile bool _poisonQueueEnsured; - private QueueClient? _poisonQueue; + private IQueueTransport? _poisonQueue; + private DeadLetterQueue? _deadLetterQueue; + public StorageQueue( + IQueueTransport transport, + Func? poisonQueueFactory, + MessageCodec codec, + ILoggerFactory loggerFactory, + QueueServiceTelemetrySettings telemetry, + QueueServiceMetrics metrics, + StorageQueueOptions options) + { + _transport = transport; + _poisonQueueFactory = poisonQueueFactory; + Codec = codec; + _logger = loggerFactory.CreateLogger(); + _telemetry = telemetry; + _metrics = metrics; + _options = options; + _time = options.Time ?? TimeProvider.System; + _serverAddress = transport.Uri?.Host; + _serverPort = transport.Uri?.Port ?? 0; + } + + /// Over an SDK client, with the 3.0 knobs; for tests. public StorageQueue( QueueClient client, IQueueMessageSerializer serializer, @@ -40,21 +75,25 @@ public StorageQueue( bool createIfNotExists, Func? poisonQueueFactory, bool propagateTraceContext) + : this( + new SdkQueueTransport(client, base64Encoded: false), + poisonQueueFactory is null ? null : () => new SdkQueueTransport(poisonQueueFactory(), base64Encoded: false), + new MessageCodec(serializer, propagateTraceContext), + loggerFactory, + telemetry, + metrics, + new StorageQueueOptions(defaultVisibilityTimeout, createIfNotExists)) { - _client = client; - _serializer = serializer; - _logger = loggerFactory.CreateLogger(); - _telemetry = telemetry; - _metrics = metrics; - _propagateTraceContext = propagateTraceContext; - _serverAddress = client.Uri?.Host; - _serverPort = client.Uri?.Port ?? 0; - _defaultVisibilityTimeout = defaultVisibilityTimeout; - _createIfNotExists = createIfNotExists; - _poisonQueueFactory = poisonQueueFactory; } - public string QueueName => _client.Name; + public string QueueName => _transport.Name; + + internal MessageCodec Codec { get; } + + internal IQueueTransport Transport => _transport; + + /// This queue's poison queue, for inspection and replay. + internal IDeadLetterQueue DeadLetterQueue => _deadLetterQueue ??= new DeadLetterQueue(this); // ------------------------------------------------------------------------------------------------------------ // Send @@ -62,25 +101,39 @@ public StorageQueue( public async ValueTask SendAsync(T message, SendOptions? options = null, CancellationToken cancellationToken = default) { + ValidateSendOptions(options); + using Activity? activity = _telemetry.CreateNewActivityOnMessageSend ? QueueServiceDiagnostics.Activities.StartSendingMessageActivity(SpanTags(QueueServiceDiagnostics.Messaging.Send)) : null; + var start = Stopwatch.GetTimestamp(); try { await EnsureQueueExistsAsync(cancellationToken); // The send span when there is one, as the conventions expect of a producer; otherwise whatever the caller // is doing (an HTTP request, say). With neither there is no trace to continue and the message goes out plain. - var body = StorageQueueCommon.Encode(_serializer, message, _propagateTraceContext, activity ?? Activity.Current, out var serializedSize); - var timeToLive = StorageQueueCommon.ServiceTimeToLive(options?.TimeToLive); - SendReceipt receipt = await _client.SendMessageAsync(body, options?.VisibilityTimeout, timeToLive, cancellationToken); + var encoded = await Codec.EncodeAsync(QueueName, _transport.Base64Encoded, message, options, activity ?? Activity.Current, cancellationToken); + SendReceipt receipt; + try + { + receipt = await _transport.SendAsync(encoded.Wire, options?.VisibilityTimeout, ServiceTimeToLive(options?.TimeToLive), cancellationToken); + } + catch when (encoded.BlobReference is { } blob) + { + // Nothing on the queue refers to it. + await DeleteStoredBodyAsync(blob); + throw; + } activity?.SetTag(QueueServiceDiagnostics.Messaging.MessageId, receipt.MessageId); var tags = QueueServiceDiagnostics.MetricTags(QueueName); _metrics.MessagesSent.Add(1, tags); - // The serializer's output, before any envelope, so the metric means what it meant in 3.0 whatever the setting. - _metrics.MessageSize.Record(serializedSize, tags); + // The serializer's output, before any envelope, so the metric means what it meant in 3.0 whatever the settings. + _metrics.MessageSize.Record(encoded.SerializedSize, tags); + RecordOperation(QueueServiceDiagnostics.Messaging.Send, QueueServiceDiagnostics.Messaging.Send, start); + _metrics.SentMessages.Add(1, QueueServiceDiagnostics.MessagingTags(QueueName, QueueServiceDiagnostics.Messaging.Send, QueueServiceDiagnostics.Messaging.Send)); return receipt; } @@ -88,34 +141,67 @@ public async ValueTask SendAsync(T message, SendOptions? options { _logger.LogSendFailed(QueueName, e); activity.RecordFailure(e); + RecordOperation(QueueServiceDiagnostics.Messaging.Send, QueueServiceDiagnostics.Messaging.Send, start, e); + _metrics.SentMessages.Add(1, QueueServiceDiagnostics.MessagingTags(QueueName, QueueServiceDiagnostics.Messaging.Send, QueueServiceDiagnostics.Messaging.Send, e)); throw; } } + private static void ValidateSendOptions(SendOptions? options) + { + if (options is null) return; + + if (options.VisibilityTimeout is { } delay && (delay < TimeSpan.Zero || delay > QueueClientSettings.MaxVisibility)) + throw new ArgumentOutOfRangeException(nameof(options), delay, "SendOptions.VisibilityTimeout (Delay) must be 0 to 7 days."); + + // Never expiring, in .NET's terms or the service's. + if (options.TimeToLive is not { } ttl || ttl == Timeout.InfiniteTimeSpan || ttl == InfiniteTimeToLive) return; + + if (ttl < TimeSpan.FromSeconds(1)) + throw new ArgumentOutOfRangeException(nameof(options), ttl, "SendOptions.TimeToLive must be at least 1 second, or Timeout.InfiniteTimeSpan for never."); + + if (options.VisibilityTimeout is { } visibility && visibility >= ttl) + throw new ArgumentOutOfRangeException(nameof(options), visibility, $"SendOptions.VisibilityTimeout (Delay) of {visibility} is not less than the TimeToLive of {ttl}: the message would expire before anyone could receive it."); + } + + /// The service's "never expires": -1 whole second. + internal static readonly TimeSpan InfiniteTimeToLive = TimeSpan.FromSeconds(-1); + + /// .NET's infinite sentinel is -1 ms; the service requires -1 whole second. + internal static TimeSpan? ServiceTimeToLive(TimeSpan? timeToLive) => + timeToLive == Timeout.InfiniteTimeSpan ? InfiniteTimeToLive : timeToLive; + // ------------------------------------------------------------------------------------------------------------ // Receive / peek // ------------------------------------------------------------------------------------------------------------ public async ValueTask>> ReceiveAsync(int maxMessages = 1, TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default) { - var messages = await ReceiveUndeserializedAsync(maxMessages, visibilityTimeout, cancellationToken); - foreach (var message in messages) Deserialize(message); + if (visibilityTimeout is { } visibility && (visibility < QueueClientSettings.MinVisibility || visibility > QueueClientSettings.MaxVisibility)) + throw new ArgumentOutOfRangeException(nameof(visibilityTimeout), visibility, "The service accepts a visibility timeout of 1 second to 7 days."); + + var messages = await ReceiveCoreAsync(maxMessages, visibilityTimeout ?? _options.DefaultVisibilityTimeout, logFailure: true, cancellationToken); + foreach (var message in messages) + { + if (await ResolvePayloadAsync(message.Lease, cancellationToken) is { } error) message.SetBody(default, error); + else Deserialize(message); + } + return messages; } - // IOwnedReceiver: the processor's path. The message is handed over with its raw body so renewal can be armed - // before deserialization runs; see IOwnedReceiver. - async ValueTask?> IOwnedReceiver.ReceiveUndeserializedAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken) - { - var messages = await ReceiveUndeserializedAsync(1, visibilityTimeout, cancellationToken); - return messages.Count == 0 ? null : messages[0]; - } + // IOwnedReceiver: the processor's path; see IOwnedReceiver. + ValueTask>> IOwnedReceiver.ReceiveForProcessingAsync(int maxMessages, TimeSpan visibilityTimeout, CancellationToken cancellationToken) => + ReceiveCoreAsync(maxMessages, visibilityTimeout, logFailure: false, cancellationToken); - public void Deserialize(QueueMessage message) => StorageQueueCommon.Deserialize(_serializer, message); + public ValueTask ResolvePayloadAsync(MessageLease message, CancellationToken cancellationToken) => + Codec.ResolvePayloadAsync(message, (reference, e) => _logger.LogLargeMessageDeleteFailed(QueueName, message.MessageId, reference, e), cancellationToken); - private async ValueTask>> ReceiveUndeserializedAsync(int maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) + public void Deserialize(QueueMessage message) => Codec.Deserialize(message); + + private async ValueTask>> ReceiveCoreAsync(int maxMessages, TimeSpan? visibilityTimeout, bool logFailure, CancellationToken cancellationToken) { - StorageQueueCommon.ValidateMaxMessages(maxMessages); + ValidateMaxMessages(maxMessages); var tags = SpanTags(QueueServiceDiagnostics.Messaging.Receive); tags.Add(QueueServiceDiagnostics.Names.QueueMessageCount, maxMessages); @@ -123,24 +209,16 @@ private async ValueTask>> ReceiveUndeserializedAsync(int ? QueueServiceDiagnostics.Activities.StartReadingQueueActivity(tags) : null; + var start = Stopwatch.GetTimestamp(); try { await EnsureQueueExistsAsync(cancellationToken); + var received = await _transport.ReceiveAsync(maxMessages, visibilityTimeout, cancellationToken); - // Messages the SDK could not Base64-decode are raised as events during the call; collect them so they - // come back as messages with Error set instead of being dropped from the batch. - SdkQueueMessage[] received; - var undecodable = DecodingFailures.Begin(); - try - { - received = await _client.ReceiveMessagesAsync(maxMessages, visibilityTimeout ?? _defaultVisibilityTimeout, cancellationToken); - } - finally - { - DecodingFailures.End(); - } + var count = received.Count; + RecordOperation(QueueServiceDiagnostics.Messaging.Receive, QueueServiceDiagnostics.Messaging.Receive, start); + _metrics.ReceiveRequests.Add(1, ReceiveResultTags(count == 0 ? "empty" : "message")); - var count = received.Length + undecodable.Count; // The conventions: a batch count for the batch form of the call, a message id for the single-message form. if (maxMessages > 1) activity?.SetTag(QueueServiceDiagnostics.Messaging.BatchMessageCount, count); var messages = new List>(count); @@ -149,21 +227,15 @@ private async ValueTask>> ReceiveUndeserializedAsync(int _logger.LogMessagesReceived(QueueName, count); activity?.AddEvent(new ActivityEvent(QueueServiceDiagnostics.Events.QueueMessageReceived)); _metrics.MessagesReceived.Add(count, QueueServiceDiagnostics.MetricTags(QueueName)); + _metrics.ConsumedMessages.Add(count, QueueServiceDiagnostics.MessagingTags(QueueName, QueueServiceDiagnostics.Messaging.Receive, QueueServiceDiagnostics.Messaging.Receive)); foreach (var m in received) - messages.Add(StorageQueueCommon.Received(Info(m), m.Body ?? BinaryData.FromString(m.MessageText ?? string.Empty), this)); - - if (maxMessages == 1 && messages.Count == 1) activity?.SetTag(QueueServiceDiagnostics.Messaging.MessageId, messages[0].MessageId); - - foreach (var failure in undecodable) { - if (failure.ReceivedMessage is not { } m) continue; - _logger.LogDecodingFailed(QueueName, m.MessageId, m.DequeueCount); - var raw = BinaryData.FromString(m.MessageText ?? string.Empty); - var error = new DeserializationException($"Message {m.MessageId} is not valid Base64, so the SDK could not decode it."); - messages.Add(new QueueMessage(Info(m), raw, default, error, this)); + if (m.Undecodable) _logger.LogDecodingFailed(QueueName, m.Info.MessageId, m.Info.DequeueCount); + messages.Add(MessageCodec.Received(m, this)); } + if (maxMessages == 1 && messages.Count == 1) activity?.SetTag(QueueServiceDiagnostics.Messaging.MessageId, messages[0].MessageId); return messages; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -172,7 +244,10 @@ private async ValueTask>> ReceiveUndeserializedAsync(int } catch (Exception e) { - _logger.LogReceiveFailed(QueueName, e); + _metrics.ReceiveRequests.Add(1, ReceiveResultTags("error")); + RecordOperation(QueueServiceDiagnostics.Messaging.Receive, QueueServiceDiagnostics.Messaging.Receive, start, e); + // The processor logs its own failures, once, with the backoff it is about to take. + if (logFailure) _logger.LogReceiveFailed(QueueName, e); activity.RecordFailure(e); throw; } @@ -180,41 +255,33 @@ private async ValueTask>> ReceiveUndeserializedAsync(int public async ValueTask>> PeekAsync(int maxMessages = 1, CancellationToken cancellationToken = default) { - StorageQueueCommon.ValidateMaxMessages(maxMessages); + ValidateMaxMessages(maxMessages); await EnsureQueueExistsAsync(cancellationToken); - PeekedMessage[] peeked; - var undecodable = DecodingFailures.Begin(); - try - { - peeked = await _client.PeekMessagesAsync(maxMessages, cancellationToken); - } - finally - { - DecodingFailures.End(); - } - - var messages = new List>(peeked.Length + undecodable.Count); + var peeked = await _transport.PeekAsync(maxMessages, cancellationToken); + var messages = new List>(peeked.Count); foreach (var m in peeked) { - var raw = StorageQueueCommon.Peeked(m.Body ?? BinaryData.FromString(m.MessageText ?? string.Empty)); - var (body, error) = StorageQueueCommon.TryDeserialize(_serializer, raw); - messages.Add(new PeekedQueueMessage(m.MessageId, m.DequeueCount, m.InsertedOn, m.ExpiresOn, raw, body, error)); - } + var message = MessageCodec.Received(m, this); + if (await ResolvePayloadAsync(message.Lease, cancellationToken) is { } error) message.SetBody(default, error); + else Deserialize(message); - foreach (var failure in undecodable) - { - if (failure.PeekedMessage is not { } m) continue; - var raw = BinaryData.FromString(m.MessageText ?? string.Empty); - var error = new DeserializationException($"Message {m.MessageId} is not valid Base64, so the SDK could not decode it."); - messages.Add(new PeekedQueueMessage(m.MessageId, m.DequeueCount, m.InsertedOn, m.ExpiresOn, raw, default, error)); + messages.Add(new PeekedQueueMessage(m.Info.MessageId, m.Info.DequeueCount, m.Info.InsertedOn, m.Info.ExpiresOn, message.RawBody, message.Body, message.Error) + { + Headers = message.Headers, + }); } return messages; } - private static QueueMessageInfo Info(SdkQueueMessage m) => - new(m.MessageId, m.PopReceipt, m.DequeueCount, m.InsertedOn, m.ExpiresOn, m.NextVisibleOn); + internal static void ValidateMaxMessages(int maxMessages) + { + if (maxMessages is < 1 or > ServiceMaxMessages) + throw new ArgumentOutOfRangeException(nameof(maxMessages), maxMessages, $"The service allows 1 to {ServiceMaxMessages} messages per receive."); + } + + internal const int ServiceMaxMessages = 32; // ------------------------------------------------------------------------------------------------------------ // Admin @@ -222,21 +289,20 @@ private static QueueMessageInfo Info(SdkQueueMessage m) => public async ValueTask CreateIfNotExistsAsync(IDictionary? metadata = null, CancellationToken cancellationToken = default) { - await _client.CreateIfNotExistsAsync(metadata, cancellationToken); + await _transport.CreateIfNotExistsAsync(metadata, cancellationToken); _queueEnsured = true; } public async ValueTask ClearAsync(CancellationToken cancellationToken = default) { await EnsureQueueExistsAsync(cancellationToken); - await _client.ClearMessagesAsync(cancellationToken); + await _transport.ClearAsync(cancellationToken); } public async ValueTask GetPropertiesAsync(CancellationToken cancellationToken = default) { await EnsureQueueExistsAsync(cancellationToken); - var response = await _client.GetPropertiesAsync(cancellationToken); - return response.Value; + return await _transport.GetPropertiesAsync(cancellationToken); } // ------------------------------------------------------------------------------------------------------------ @@ -245,36 +311,53 @@ public async ValueTask GetPropertiesAsync(CancellationToken can async ValueTask IQueueMessageOperations.DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken) { - await _client.DeleteMessageAsync(messageId, popReceipt, cancellationToken); + var start = Stopwatch.GetTimestamp(); + try + { + await _transport.DeleteAsync(messageId, popReceipt, cancellationToken); + RecordOperation(QueueServiceDiagnostics.Messaging.Settle, "complete", start); + } + catch (Exception e) + { + RecordOperation(QueueServiceDiagnostics.Messaging.Settle, "complete", start, e); + throw; + } } async ValueTask IQueueMessageOperations.UpdateVisibilityAsync(string messageId, string popReceipt, TimeSpan visibilityTimeout, CancellationToken cancellationToken) { - // No message text: only the visibility changes. - UpdateReceipt receipt = await _client.UpdateMessageAsync(messageId, popReceipt, (string?)null, visibilityTimeout, cancellationToken); - return new VisibilityReceipt(receipt.PopReceipt, receipt.NextVisibleOn); + var start = Stopwatch.GetTimestamp(); + try + { + var receipt = await _transport.UpdateVisibilityAsync(messageId, popReceipt, visibilityTimeout, cancellationToken); + RecordOperation(QueueServiceDiagnostics.Messaging.Settle, "update_visibility", start); + return receipt; + } + catch (Exception e) + { + RecordOperation(QueueServiceDiagnostics.Messaging.Settle, "update_visibility", start, e); + throw; + } } - async ValueTask IQueueMessageOperations.DeadLetterAsync(string messageId, string popReceipt, long dequeueCount, BinaryData rawBody, string? reason, CancellationToken cancellationToken) - { - if (_poisonQueueFactory is null) - throw new InvalidOperationException($"Queue '{QueueName}' has no poison queue configured, so messages cannot be dead-lettered."); + ValueTask IQueueMessageOperations.DeadLetterAsync(string messageId, string popReceipt, long dequeueCount, BinaryData rawBody, string? reason, CancellationToken cancellationToken) => + DeadLetterCoreAsync(messageId, popReceipt, dequeueCount, rawBody, envelope: null, reason, DeadLetterCategories.Handler, cancellationToken); - var poison = _poisonQueue ??= _poisonQueueFactory(); + ValueTask IDeadLetterOperations.DeadLetterAsync(MessageLease message, string? reason, string category, CancellationToken cancellationToken) => + DeadLetterCoreAsync(message.MessageId, message.PopReceipt, message.DequeueCount, message.WireBody, message.Envelope, reason, category, cancellationToken); - // The body exactly as it arrived, trace-context envelope and all, so a poison message can be moved straight back - // to replay it. The reason goes to the log; storage queue messages have no properties to carry it in. - // Copy-then-delete is two calls: a lease that expires between them can leave a duplicate in the poison queue, - // which is the at-least-once trade. - if (!_poisonQueueEnsured) - { - await poison.CreateIfNotExistsAsync(cancellationToken: cancellationToken); - _poisonQueueEnsured = true; - } + private async ValueTask DeadLetterCoreAsync(string messageId, string popReceipt, long dequeueCount, BinaryData wire, MessageEnvelope? envelope, string? reason, string category, CancellationToken cancellationToken) + { + var poison = PoisonQueue(); + var info = new DeadLetterInfo(Truncate(reason), category, messageId, dequeueCount, QueueName, _time.GetUtcNow()); + var body = DeadLetterBody(wire, envelope, info, poison.Base64Encoded); + // Copy-then-delete is two calls: a lease that expires between them can leave a duplicate in the poison queue, + // which is the at-least-once trade. A body kept in Blob storage stays there, so a replay still has it. + await EnsurePoisonQueueExistsAsync(cancellationToken); try { - await poison.SendMessageAsync(rawBody, cancellationToken: cancellationToken); + await poison.SendAsync(body, visibilityTimeout: null, ServiceTimeToLive(_options.PoisonTimeToLive ?? Timeout.InfiniteTimeSpan), cancellationToken); } catch (RequestFailedException e) when (e.ErrorCode == QueueErrorCode.QueueNotFound) { @@ -282,16 +365,97 @@ async ValueTask IQueueMessageOperations.DeadLetterAsync(string messageId, string _poisonQueueEnsured = false; throw; } - await _client.DeleteMessageAsync(messageId, popReceipt, cancellationToken); + + await _transport.DeleteAsync(messageId, popReceipt, cancellationToken); _logger.LogDeadLettered(QueueName, messageId, poison.Name, dequeueCount, reason ?? "(none)"); + var tags = QueueServiceDiagnostics.MetricTags(QueueName); + tags.Add(QueueServiceDiagnostics.Names.DeadLetterReason, category); + _metrics.DeadLettered.Add(1, tags); + } + + /// + /// The copy for the poison queue: the message as it arrived, in an envelope that also records why and from where. + /// What cannot be annotated (an envelope this version cannot read, or a copy that would be over the size limit) goes + /// as it arrived: a copy without the reason beats no copy. + /// + private static BinaryData DeadLetterBody(BinaryData wire, MessageEnvelope? envelope, DeadLetterInfo info, bool base64Encoded) + { + if (envelope is null) + { + switch (MessageEnvelope.TryRead(wire, out var parsed, out _)) + { + case EnvelopeKind.Unreadable: + return wire; + case EnvelopeKind.Envelope: + envelope = parsed; + break; + } + } + + MessageEnvelope copy; + if (envelope is null) + { + copy = new MessageEnvelope(); + copy.SetBody(wire); + } + else + { + copy = envelope.Clone(); + } + + copy.DeadLetter = info; + var body = copy.Write(); + return MessageCodec.WireSize(body.ToMemory().Length, base64Encoded) <= MessageCodec.MaxMessageBytes ? body : wire; + } + + private static string? Truncate(string? reason) => + reason is { Length: > MaxReasonLength } ? reason[..MaxReasonLength] + "…" : reason; + + // ------------------------------------------------------------------------------------------------------------ + // The poison queue, for DeadLetterQueue + // ------------------------------------------------------------------------------------------------------------ + + internal IQueueTransport PoisonQueue() + { + if (_poisonQueueFactory is null) + throw new InvalidOperationException($"Queue '{QueueName}' has no poison queue configured, so messages cannot be dead-lettered."); + + return _poisonQueue ??= _poisonQueueFactory(); + } + + internal async ValueTask EnsurePoisonQueueExistsAsync(CancellationToken cancellationToken) + { + if (_poisonQueueEnsured) return; + await PoisonQueue().CreateIfNotExistsAsync(null, cancellationToken); + _poisonQueueEnsured = true; + } + + /// Puts bytes on this queue as they are: a replayed message. + internal async ValueTask SendWireAsync(BinaryData wire, CancellationToken cancellationToken) + { + await EnsureQueueExistsAsync(cancellationToken); + await _transport.SendAsync(wire, visibilityTimeout: null, timeToLive: null, cancellationToken); + } + + internal async ValueTask DeleteStoredBodyAsync(string reference) + { + if (Codec.LargeMessageStore is not { } store) return; + try + { + await store.DeleteAsync(reference, CancellationToken.None); + } + catch (Exception e) + { + _logger.LogLargeMessageDeleteFailed(QueueName, null, reference, e); + } } // ------------------------------------------------------------------------------------------------------------ // Helpers // ------------------------------------------------------------------------------------------------------------ - // Spans only, and they also say which account they talked to; metrics keep their 3.0 tags (MetricTags). + // Spans only, and they also say which account they talked to; the library's own metrics keep their 3.0 tags. private TagList SpanTags(string operation) { var tags = QueueServiceDiagnostics.OperationTags(QueueName, operation); @@ -301,16 +465,34 @@ private TagList SpanTags(string operation) return tags; } + private TagList ReceiveResultTags(string result) + { + var tags = QueueServiceDiagnostics.MetricTags(QueueName); + tags.Add(QueueServiceDiagnostics.Names.ReceiveResult, result); + return tags; + } + + private void RecordOperation(string operationType, string operationName, long start, Exception? error = null) + { + var tags = QueueServiceDiagnostics.MessagingTags(QueueName, operationType, operationName, error); + if (_serverAddress is not null) + { + tags.Add(QueueServiceDiagnostics.Messaging.ServerAddress, _serverAddress); + tags.Add(QueueServiceDiagnostics.Messaging.ServerPort, _serverPort); + } + _metrics.OperationDuration.Record(Stopwatch.GetElapsedTime(start).TotalSeconds, tags); + } + // Queue creation happens on the first operation, once, asynchronously — never while resolving from DI. private async ValueTask EnsureQueueExistsAsync(CancellationToken cancellationToken) { - if (!_createIfNotExists || _queueEnsured) return; + if (!_options.CreateIfNotExists || _queueEnsured) return; await _createLock.WaitAsync(cancellationToken); try { if (_queueEnsured) return; - await _client.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + await _transport.CreateIfNotExistsAsync(null, cancellationToken); _queueEnsured = true; } finally diff --git a/src/AzureStorage.QueueService/StorageQueueCommon.cs b/src/AzureStorage.QueueService/StorageQueueCommon.cs deleted file mode 100644 index 5967c3e..0000000 --- a/src/AzureStorage.QueueService/StorageQueueCommon.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System.Diagnostics; -using AzureStorage.QueueService.Telemetry; - -namespace AzureStorage.QueueService; - -/// -/// What and share: the service's limits, the -/// trace-context envelope on the way out and in, and deserialization. One copy, so the in-memory queue a test runs -/// against cannot drift from the code that runs in production. -/// -internal static class StorageQueueCommon -{ - public const int ServiceMaxMessages = 32; - - /// The service's "never expires": -1 whole second. - public static readonly TimeSpan InfiniteTimeToLive = TimeSpan.FromSeconds(-1); - - public static void ValidateMaxMessages(int maxMessages) - { - if (maxMessages is < 1 or > ServiceMaxMessages) - throw new ArgumentOutOfRangeException(nameof(maxMessages), maxMessages, $"The service allows 1 to {ServiceMaxMessages} messages per receive."); - } - - /// .NET's infinite sentinel is -1 ms; the service requires -1 whole second. - public static TimeSpan? ServiceTimeToLive(TimeSpan? timeToLive) => - timeToLive == Timeout.InfiniteTimeSpan ? InfiniteTimeToLive : timeToLive; - - /// - /// Serializes and, when is on and there is a - /// span to carry, wraps it in the trace-context envelope. - /// is the serializer's output alone, which is what the message size metric has always measured. - /// - public static BinaryData Encode(IQueueMessageSerializer serializer, T message, bool propagateTraceContext, Activity? creation, out int serializedSize) - { - BinaryData body = serializer.Serialize(message); - serializedSize = body.ToMemory().Length; - if (!propagateTraceContext || creation is null) return body; - - // The library's own serializer always writes exactly one JSON value, so there is no need to parse it to find out. - return TraceContextEnvelope.Wrap(body, creation, payloadIsJson: serializer is JsonQueueMessageSerializer) ?? body; - } - - /// - /// A received message, not yet deserialized. Unwrapped whatever the client's own setting, which governs sending - /// only: a receiver reads what any sender put on the queue. - /// - public static QueueMessage Received(QueueMessageInfo info, BinaryData wire, IQueueMessageOperations operations) - { - if (!TraceContextEnvelope.TryUnwrap(wire, out var payload, out var creationContext)) - return new QueueMessage(info, wire, default, null, operations); - - // The handler sees the message the sender serialized; dead-lettering still copies the whole envelope. - return new QueueMessage(info, payload, default, null, operations) { WireBody = wire, CreationContext = creationContext }; - } - - /// A peeked message's body: the payload inside an envelope, or the body as it is. - public static BinaryData Peeked(BinaryData wire) => - TraceContextEnvelope.TryUnwrap(wire, out var payload, out _) ? payload : wire; - - public static void Deserialize(IQueueMessageSerializer serializer, QueueMessage message) - { - if (message.Error is not null) return; // a decoding failure; there is nothing to deserialize - var (body, error) = TryDeserialize(serializer, message.RawBody); - message.SetBody(body, error); - } - - public static (T? Body, DeserializationException? Error) TryDeserialize(IQueueMessageSerializer serializer, BinaryData raw) - { - try - { - return (serializer.Deserialize(raw), null); - } - catch (DeserializationException e) - { - return (default, e); - } - catch (Exception e) - { - // A custom serializer that throws something else still must not take the receive down. - return (default, new DeserializationException($"Could not deserialize the queue message as {typeof(T).Name}: {e.Message}", e)); - } - } -} diff --git a/src/AzureStorage.QueueService/Telemetry/MeterProviderBuilderExtensions.cs b/src/AzureStorage.QueueService/Telemetry/MeterProviderBuilderExtensions.cs index 95e7851..c529180 100644 --- a/src/AzureStorage.QueueService/Telemetry/MeterProviderBuilderExtensions.cs +++ b/src/AzureStorage.QueueService/Telemetry/MeterProviderBuilderExtensions.cs @@ -2,11 +2,25 @@ namespace AzureStorage.QueueService.Telemetry; +/// Subscribes an OpenTelemetry meter provider to the library's metrics. public static class MeterProviderBuilderExtensions { + /// The library's metrics under their own names (queue_messages_processed_total and so on). public static MeterProviderBuilder AddAzureStorageQueueMetrics(this MeterProviderBuilder builder) { builder.AddMeter(QueueServiceDiagnostics.ServiceName); return builder; } + + /// + /// The same activity under the OpenTelemetry messaging metric names (messaging.client.sent.messages, + /// messaging.client.consumed.messages, messaging.process.duration and + /// messaging.client.operation.duration), which standard dashboards recognise. On a meter of its own, so turning + /// it on leaves the library's own metrics and their history untouched. + /// + public static MeterProviderBuilder AddAzureStorageQueueMessagingMetrics(this MeterProviderBuilder builder) + { + builder.AddMeter(QueueServiceDiagnostics.MessagingMeterName); + return builder; + } } diff --git a/src/AzureStorage.QueueService/Telemetry/QueueServiceDiagnostics.cs b/src/AzureStorage.QueueService/Telemetry/QueueServiceDiagnostics.cs index 5dd9c46..412d6e3 100644 --- a/src/AzureStorage.QueueService/Telemetry/QueueServiceDiagnostics.cs +++ b/src/AzureStorage.QueueService/Telemetry/QueueServiceDiagnostics.cs @@ -6,6 +6,9 @@ internal class QueueServiceDiagnostics { public static string ServiceName { get; } = "AzureStorage.QueueService"; + /// The meter with the OpenTelemetry messaging metric names; opt in by subscribing to it. + public const string MessagingMeterName = "AzureStorage.QueueService.Messaging"; + // Static, as ActivitySources normally are: listeners subscribe by name, and there is nothing to dispose per host. // The Meter is not: see QueueServiceMetrics. public static ActivitySource ActivitySource { get; } = new ActivitySource(ServiceName); @@ -42,6 +45,9 @@ public struct Names public const string MessageId = "queue.message_id"; public const string DequeueCount = "queue.dequeue_count"; public const string Outcome = "queue.outcome"; + public const string ReceiveResult = "queue.receive_result"; + public const string DeadLetterReason = "queue.dead_letter_reason"; + public const string QueueKind = "queue.kind"; } /// @@ -65,10 +71,12 @@ public struct Messaging // and spelled out so it can never be mistaken for Service Bus queues. public const string SystemValue = "azure.storage.queue"; - // Operation types from the conventions. The operation name is free-form per system; ours is the same word. + // Operation types from the conventions. The operation name is free-form per system; ours is the same word, + // except for settlement, whose names say which settlement it was. public const string Send = "send"; public const string Receive = "receive"; public const string Process = "process"; + public const string Settle = "settle"; } /// @@ -85,11 +93,26 @@ public struct Messaging }; /// - /// The tags on every measurement: exactly the 3.0 set. The semantic-convention attributes stay on spans, because a - /// new label on an existing metric starts a new series and breaks continuity with stored history. + /// The tags on every measurement of the library's own metrics: exactly the 3.0 set. The semantic-convention + /// attributes stay on spans and on the separate messaging meter, because a new label on an existing metric starts a + /// new series and breaks continuity with stored history. /// public static TagList MetricTags(string queueName) => new() { { Names.QueueName, queueName } }; + /// The tags of the messaging meter's measurements: the conventions' attributes only. + public static TagList MessagingTags(string queueName, string operationType, string operationName, Exception? error = null) + { + var tags = new TagList + { + { Messaging.System, Messaging.SystemValue }, + { Messaging.DestinationName, queueName }, + { Messaging.OperationType, operationType }, + { Messaging.OperationName, operationName }, + }; + if (error is not null) tags.Add(Messaging.ErrorType, error.GetType().FullName); + return tags; + } + // Event names only. An ActivityEvent captures its timestamp when constructed, so a shared static instance // would stamp every event with the time the process started. public struct Events diff --git a/src/AzureStorage.QueueService/Telemetry/QueueServiceMetrics.cs b/src/AzureStorage.QueueService/Telemetry/QueueServiceMetrics.cs index 79e1b63..5a1e25c 100644 --- a/src/AzureStorage.QueueService/Telemetry/QueueServiceMetrics.cs +++ b/src/AzureStorage.QueueService/Telemetry/QueueServiceMetrics.cs @@ -1,27 +1,48 @@ +using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.Metrics; namespace AzureStorage.QueueService.Telemetry; /// /// The library's instruments. Created through the host's when one is registered (the -/// generic host registers one), so the meter belongs to that container: it is disposed with it, and a test can observe -/// it with MetricCollector<T> without seeing other hosts' measurements. Without a factory every client shares -/// one library-owned meter, as before. Either way the meter is named , -/// so AddAzureStorageQueueMetrics() picks it up. +/// generic host registers one), so the meters belong to that container: they are disposed with it, and a test can +/// observe them with MetricCollector<T> without seeing other hosts' measurements. Without a factory every +/// client shares library-owned meters. Two meters: with the library's +/// own metric names (AddAzureStorageQueueMetrics()), and +/// with the OpenTelemetry messaging names, which nobody receives unless they subscribe to it +/// (AddAzureStorageQueueMessagingMetrics()). /// internal sealed class QueueServiceMetrics { - // Lazy, so a process whose hosts all have a meter factory never creates the fallback meter at all. - private static readonly Lazy FallbackInstance = new(() => new QueueServiceMetrics(new Meter(QueueServiceDiagnostics.ServiceName))); + // Lazy, so a process whose hosts all have a meter factory never creates the fallback meters at all. + private static readonly Lazy FallbackInstance = new(() => + new QueueServiceMetrics(new Meter(QueueServiceDiagnostics.ServiceName), new Meter(QueueServiceDiagnostics.MessagingMeterName))); + + // Bucket boundaries are part of the stored history: changing them later splits it, so they are set once, here. + private static readonly double[] DurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800, 3600]; + private static readonly double[] TimeInQueueBuckets = [0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600, 7200, 21600, 86400, 604800]; + private static readonly long[] SizeBuckets = [128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 49152, 65536]; + + // The messaging conventions' own advice for their duration histograms. + private static readonly double[] SemanticConventionDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10]; + + private readonly ConcurrentDictionary>>> _handlerAges = new(); + private readonly ConcurrentDictionary<(string Queue, string Kind), long> _depths = new(); /// The library-owned instruments, for when no is registered. public static QueueServiceMetrics Fallback => FallbackInstance.Value; public static QueueServiceMetrics Create(IMeterFactory? meterFactory) => - meterFactory is null ? Fallback : new QueueServiceMetrics(meterFactory.Create(new MeterOptions(QueueServiceDiagnostics.ServiceName))); + meterFactory is null + ? Fallback + : new QueueServiceMetrics( + meterFactory.Create(new MeterOptions(QueueServiceDiagnostics.ServiceName)), + meterFactory.Create(new MeterOptions(QueueServiceDiagnostics.MessagingMeterName))); - private QueueServiceMetrics(Meter meter) + private QueueServiceMetrics(Meter meter, Meter messaging) { + // The 3.0 instruments: names, kinds and tags unchanged, so stored history stays continuous. MessagesReceived = meter.CreateCounter( "queue_messages_received_total", description: "Total number of messages received from the queue."); @@ -38,13 +59,67 @@ private QueueServiceMetrics(Meter meter) "queue_messages_failed_total", description: "Total number of messages that failed processing."); - MessageProcessingDuration = meter.CreateHistogram( + MessageProcessingDuration = meter.CreateHistogram( "queue_message_processing_duration_seconds", - description: "Time taken to process a message from the queue."); + unit: "s", + description: "Time taken to process a message from the queue.", + tags: null, + advice: new InstrumentAdvice { HistogramBucketBoundaries = DurationBuckets }); - MessageSize = meter.CreateHistogram( + MessageSize = meter.CreateHistogram( "queue_message_size_bytes", - description: "Size of messages sent to the queue."); + unit: "By", + description: "Size of messages sent to the queue, as the serializer wrote them.", + tags: null, + advice: new InstrumentAdvice { HistogramBucketBoundaries = SizeBuckets }); + + // Added in 3.1. + TimeInQueue = meter.CreateHistogram( + "queue_message_time_in_queue_seconds", + unit: "s", + description: "How long a message waited on the queue before a processor started delivering it, measured from when it was first sent; retries count.", + tags: null, + advice: new InstrumentAdvice { HistogramBucketBoundaries = TimeInQueueBuckets }); + + ReceiveRequests = meter.CreateCounter( + "queue_receive_requests_total", + description: "Receive requests made to the queue, tagged queue.receive_result: message, empty or error. Each is a billable transaction, empty ones included."); + + ActiveHandlers = meter.CreateUpDownCounter( + "queue_processor_active_handlers", + description: "Messages a processor is delivering right now. Compare with MaxConcurrentMessages for saturation."); + + LeaseRenewals = meter.CreateCounter( + "queue_lease_renewals_total", + description: "Visibility renewals made by processors while handlers ran."); + + LeaseRenewalFailures = meter.CreateCounter( + "queue_lease_renewal_failures_total", + description: "Visibility renewals that failed; the message becomes visible to other consumers when its lease lapses."); + + DeadLettered = meter.CreateCounter( + "queue_messages_dead_lettered_total", + description: "Messages moved to the poison queue, tagged queue.dead_letter_reason: max_dequeue, deserialization, handler or unroutable."); + + meter.CreateObservableGauge( + "queue_processor_oldest_handler_age_seconds", + ObserveHandlerAges, + unit: "s", + description: "How long the longest-running handler of each processor has been running. A value that keeps growing is a hung handler."); + + meter.CreateObservableGauge( + "queue_approximate_messages", + ObserveDepths, + unit: "{message}", + description: "The service's approximate message count, hidden messages included, for queues with depth metrics turned on; tagged queue.kind: main or poison."); + + // The OpenTelemetry messaging names, on their own meter. + SentMessages = messaging.CreateCounter("messaging.client.sent.messages", unit: "{message}", description: "Number of messages producer attempted to send to the broker."); + ConsumedMessages = messaging.CreateCounter("messaging.client.consumed.messages", unit: "{message}", description: "Number of messages that were delivered to the application."); + ProcessDuration = messaging.CreateHistogram("messaging.process.duration", unit: "s", description: "Duration of processing operation.", tags: null, + advice: new InstrumentAdvice { HistogramBucketBoundaries = SemanticConventionDurationBuckets }); + OperationDuration = messaging.CreateHistogram("messaging.client.operation.duration", unit: "s", description: "Duration of messaging operation initiated by a producer or consumer client.", tags: null, + advice: new InstrumentAdvice { HistogramBucketBoundaries = SemanticConventionDurationBuckets }); } public Counter MessagesReceived { get; } @@ -58,4 +133,61 @@ private QueueServiceMetrics(Meter meter) public Histogram MessageProcessingDuration { get; } public Histogram MessageSize { get; } + + public Histogram TimeInQueue { get; } + + public Counter ReceiveRequests { get; } + + public UpDownCounter ActiveHandlers { get; } + + public Counter LeaseRenewals { get; } + + public Counter LeaseRenewalFailures { get; } + + public Counter DeadLettered { get; } + + public Counter SentMessages { get; } + + public Counter ConsumedMessages { get; } + + public Histogram ProcessDuration { get; } + + public Histogram OperationDuration { get; } + + /// Adds a processor's handler ages to the gauge until the returned handle is disposed. + public IDisposable ObserveHandlerAge(Func>> observe) + { + var key = new object(); + _handlerAges[key] = observe; + return new Unregister(() => _handlerAges.TryRemove(key, out _)); + } + + /// Records the latest approximate count for a queue; reported by the depth gauge until . + public void SetDepth(string queueName, string kind, long count) => _depths[(queueName, kind)] = count; + + public void ForgetDepth(string queueName, string kind) => _depths.TryRemove((queueName, kind), out _); + + private IEnumerable> ObserveHandlerAges() + { + foreach (var observe in _handlerAges.Values) + { + foreach (var measurement in observe()) yield return measurement; + } + } + + private IEnumerable> ObserveDepths() + { + foreach (var ((queue, kind), count) in _depths) + yield return new Measurement(count, new KeyValuePair(QueueServiceDiagnostics.Names.QueueName, queue), new KeyValuePair(QueueServiceDiagnostics.Names.QueueKind, kind)); + } + + private sealed class Unregister(Action action) : IDisposable + { + private int _done; + + public void Dispose() + { + if (Interlocked.Exchange(ref _done, 1) == 0) action(); + } + } } diff --git a/src/AzureStorage.QueueService/Telemetry/TraceContextEnvelope.cs b/src/AzureStorage.QueueService/Telemetry/TraceContextEnvelope.cs deleted file mode 100644 index 23df079..0000000 --- a/src/AzureStorage.QueueService/Telemetry/TraceContextEnvelope.cs +++ /dev/null @@ -1,176 +0,0 @@ -using System.Buffers; -using System.Diagnostics; -using System.Text.Json; - -namespace AzureStorage.QueueService.Telemetry; - -/// -/// Carries the sender's W3C trace context inside a message body, because a storage queue message has nothing else to -/// carry it in: no properties, no headers. Clients with on wrap -/// what they send; every client unwraps what it receives. -/// -/// -/// Wire format, version 1 — a JSON object whose first property is the marker: -/// -/// {"$azqs-envelope":1,"traceparent":"00-<trace id>-<span id>-<flags>","tracestate":"...","body":<the serialized message>} -/// -/// tracestate is present only when there is one. body holds the serializer's output verbatim when it is -/// JSON (the default serializer always is); a custom serializer's non-JSON output goes in body64 as Base64 -/// instead. The SDK's message encoding (for example Base64) applies to the whole envelope, as it did to the plain body. -/// Detection is strict so an ordinary payload is never mistaken for an envelope: the marker must come first with a -/// known version, and only the properties above may follow. The marker's hyphen means no C# property name can -/// produce it by accident. -/// -internal static class TraceContextEnvelope -{ - public const int Version = 1; - - private static ReadOnlySpan MarkerUtf8 => "$azqs-envelope"u8; - private static ReadOnlySpan TraceParentUtf8 => "traceparent"u8; - private static ReadOnlySpan TraceStateUtf8 => "tracestate"u8; - private static ReadOnlySpan BodyUtf8 => "body"u8; - private static ReadOnlySpan Body64Utf8 => "body64"u8; - - // The writer nests the payload one level inside the envelope, so reading allows deeper than the check on write - // (64, System.Text.Json's default); an envelope this library wrote can never fail to unwrap on depth. - private static readonly JsonReaderOptions ReadOptions = new() { MaxDepth = 256 }; - - /// - /// Wraps with the context of , or returns null when the - /// activity has no W3C identity to carry (the legacy hierarchical id format), in which case the message goes out plain. - /// says the caller already knows the payload is exactly one JSON value (the library's - /// own serializer), which saves parsing the whole payload on every send; otherwise it is checked. - /// - public static BinaryData? Wrap(BinaryData payload, Activity activity, bool payloadIsJson = false) - { - if (activity.IdFormat != ActivityIdFormat.W3C) return null; - - var context = activity.Context; - var traceParent = $"00-{context.TraceId.ToHexString()}-{context.SpanId.ToHexString()}-{(byte)context.TraceFlags:x2}"; - var body = payload.ToMemory().Span; - - var buffer = new ArrayBufferWriter(body.Length + 160); - using (var writer = new Utf8JsonWriter(buffer)) - { - writer.WriteStartObject(); - writer.WriteNumber(MarkerUtf8, Version); - writer.WriteString(TraceParentUtf8, traceParent); - if (!string.IsNullOrEmpty(activity.TraceStateString)) writer.WriteString(TraceStateUtf8, activity.TraceStateString); - if (payloadIsJson || IsSingleJsonValue(body)) - { - writer.WritePropertyName(BodyUtf8); - writer.WriteRawValue(body, skipInputValidation: true); - } - else - { - writer.WriteBase64String(Body64Utf8, body); - } - writer.WriteEndObject(); - } - - return new BinaryData(buffer.WrittenMemory); - } - - /// - /// When is an envelope, returns the original payload (a slice of the same bytes for a JSON - /// body) and the sender's context, if it parses. Anything else is a plain message and is left alone. - /// Never throws: it runs inside every receive, so one malformed message must not fail the batch it arrived in. A - /// lookalike that breaks part-way is delivered as it arrived, and its deserialization failure takes the usual - /// dead-letter path. - /// - public static bool TryUnwrap(BinaryData wire, out BinaryData payload, out ActivityContext? creationContext) - { - payload = wire; - creationContext = null; - - var memory = wire.ToMemory(); - if (!StartsWithObject(memory.Span)) return false; // the cheap test first: most plain payloads stop here - - try - { - return TryRead(memory, ref payload, ref creationContext); - } - catch (Exception e) when (e is JsonException or InvalidOperationException) - { - // Starts like JSON but is not (JsonException), or has a string that is not valid UTF-8, which the reader - // only discovers when the string is read (InvalidOperationException): a plain message. - payload = wire; - creationContext = null; - return false; - } - } - - private static bool TryRead(ReadOnlyMemory memory, ref BinaryData payload, ref ActivityContext? creationContext) - { - var reader = new Utf8JsonReader(memory.Span, ReadOptions); - if (!reader.Read() || reader.TokenType != JsonTokenType.StartObject) return false; - if (!reader.Read() || reader.TokenType != JsonTokenType.PropertyName || !reader.ValueTextEquals(MarkerUtf8)) return false; - if (!reader.Read() || reader.TokenType != JsonTokenType.Number || !reader.TryGetInt32(out var version) || version != Version) return false; - - string? traceParent = null, traceState = null; - BinaryData? body = null; - - while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) - { - if (reader.ValueTextEquals(TraceParentUtf8) && traceParent is null) - { - if (!reader.Read() || reader.TokenType != JsonTokenType.String) return false; - traceParent = reader.GetString(); - } - else if (reader.ValueTextEquals(TraceStateUtf8) && traceState is null) - { - if (!reader.Read() || reader.TokenType != JsonTokenType.String) return false; - traceState = reader.GetString(); - } - else if (reader.ValueTextEquals(BodyUtf8) && body is null) - { - if (!reader.Read()) return false; - var start = (int)reader.TokenStartIndex; - reader.Skip(); - body = new BinaryData(memory[start..(int)reader.BytesConsumed]); - } - else if (reader.ValueTextEquals(Body64Utf8) && body is null) - { - if (!reader.Read() || reader.TokenType != JsonTokenType.String || !reader.TryGetBytesFromBase64(out var bytes)) return false; - body = new BinaryData(bytes); - } - else - { - return false; // an unknown or repeated property: not ours - } - } - - if (reader.TokenType != JsonTokenType.EndObject || traceParent is null || body is null) return false; - if (reader.Read()) return false; // something after the object - - payload = body; - // A malformed traceparent still unwraps: the body is intact, there is just no context to continue. - creationContext = ActivityContext.TryParse(traceParent, traceState, isRemote: true, out var context) ? context : null; - return true; - } - - private static bool StartsWithObject(ReadOnlySpan bytes) - { - foreach (var b in bytes) - { - if (b is (byte)' ' or (byte)'\t' or (byte)'\r' or (byte)'\n') continue; - return b == (byte)'{'; - } - return false; - } - - private static bool IsSingleJsonValue(ReadOnlySpan bytes) - { - try - { - var reader = new Utf8JsonReader(bytes); - if (!reader.Read()) return false; - reader.Skip(); - return !reader.Read(); - } - catch (JsonException) - { - return false; - } - } -} diff --git a/tests/AzureStorage.QueueClient.IntegrationTests/IntegrationTests.cs b/tests/AzureStorage.QueueClient.IntegrationTests/IntegrationTests.cs index a3d4d9b..1301af9 100644 --- a/tests/AzureStorage.QueueClient.IntegrationTests/IntegrationTests.cs +++ b/tests/AzureStorage.QueueClient.IntegrationTests/IntegrationTests.cs @@ -1,4 +1,4 @@ -using System.Diagnostics; +using System.Diagnostics; using System.Net.Http.Json; using System.Text; using Azure.Storage.Queues; @@ -238,7 +238,11 @@ public async Task Base64Queue_UndecodableMessage_DoesNotBreakTheBatch() await healthy.CompleteAsync(); var poisoned = (await poison.PeekMessagesAsync(10)).Value.Should().ContainSingle().Subject; - Encoding.UTF8.GetString(Convert.FromBase64String(poisoned.MessageText)).Should().Be("this is not base64!!", "the poison client encodes like the source queue, so the raw text round-trips"); + Encoding.UTF8.GetString(Convert.FromBase64String(poisoned.MessageText)).Should().StartWith("{\"$azqs-envelope\":1", "the poison client encodes like the source queue"); + var deadLettered = (await _factory.Services.GetRequiredService().GetDeadLetterQueue("base64").PeekAsync()).Should().ContainSingle().Subject; + deadLettered.RawBody.ToString().Should().Be("this is not base64!!", "the text as it was on the queue is kept"); + deadLettered.Reason.Should().Be("not base64"); + deadLettered.OriginalMessageId.Should().Be(bad.MessageId); (await raw.PeekMessagesAsync(10)).Value.Should().BeEmpty("both messages were settled"); } } diff --git a/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs b/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs index 50f3578..43580e1 100644 --- a/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs +++ b/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs @@ -1,4 +1,4 @@ -using System.Diagnostics; +using System.Diagnostics; using System.Diagnostics.Metrics; using System.Text.Json; using Azure.Messaging; @@ -231,7 +231,7 @@ public async ValueTask DisposeAsync() // Delivery // --------------------------------------------------------------------------------------------------------- - [Fact(DisplayName = "Each worker receives one message, handles it in its own scope, and completes it")] + [Fact(DisplayName = "The processor asks for as many messages as it has free handlers and handles each in its own scope")] public async Task Each_Worker_Handles_One_Message_In_Its_Own_Scope() { await using var h = new Harness(output, o => o.MaxConcurrentMessages = 2); @@ -242,7 +242,8 @@ public async Task Each_Worker_Handles_One_Message_In_Its_Own_Scope() h.State.Invocations.Select(i => i.MessageId).Should().BeEquivalentTo("1", "2"); h.State.Invocations.Select(i => i.Scope).Distinct().Should().HaveCount(2, "every message gets a fresh scope"); - h.Queue.Receives.Should().OnlyContain(r => r.Take == 1 && r.Visibility == TimeSpan.FromMinutes(1)); + h.Queue.Receives[0].Should().Be((2, TimeSpan.FromMinutes(1)), "both handlers are free at the start"); + h.Queue.Receives.Should().OnlyContain(r => r.Take >= 1 && r.Take <= 2 && r.Visibility == TimeSpan.FromMinutes(1)); } [Fact(DisplayName = "A throwing handler abandons the message with the retry delay for its dequeue count")] diff --git a/tests/AzureStorage.QueueService.Tests/StorageQueueTests.cs b/tests/AzureStorage.QueueService.Tests/StorageQueueTests.cs index 164ff1d..a934e70 100644 --- a/tests/AzureStorage.QueueService.Tests/StorageQueueTests.cs +++ b/tests/AzureStorage.QueueService.Tests/StorageQueueTests.cs @@ -1,4 +1,4 @@ -using System.Diagnostics; +using System.Diagnostics; using Azure; using Azure.Storage.Queues; using Azure.Storage.Queues.Models; @@ -92,8 +92,9 @@ public async Task Receive_Undecodable_Message_Is_Returned_With_Error() bad.DequeueCount.Should().Be(4); bad.RawBody.ToString().Should().Be("this is not base64!!"); + var poisoned = CaptureSends(poison); await bad.DeadLetterAsync("undecodable"); - poison.Verify(x => x.SendMessageAsync(It.Is(b => b.ToString() == "this is not base64!!"), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + DeadLetterQueue.WithoutDeadLetterRecord(poisoned.Should().ContainSingle().Subject).ToString().Should().Be("this is not base64!!", "the text as it was on the queue is kept"); client.Verify(x => x.DeleteMessageAsync("bad", "receipt-bad", It.IsAny()), Times.Once); } @@ -206,22 +207,29 @@ public async Task Propagation_Unwraps_Envelopes_And_Still_Reads_Plain_Messages() plain.CreationContext.Should().BeNull(); } - [Fact(DisplayName = "Dead-lettering an enveloped message copies the whole envelope, so moving it back replays with its trace context")] + [Fact(DisplayName = "Dead-lettering keeps the envelope and records why; replaying strips the record and keeps the trace context")] public async Task DeadLetter_Keeps_The_Envelope() { using var caller = new Activity("caller").Start(); - var wire = TraceContextEnvelope.Wrap(BinaryData.FromString(ValidJson), caller)!.ToString(); + var sent = new MessageEnvelope(); + sent.SetTraceContext(caller); + sent.SetBody(BinaryData.FromString(ValidJson)); + var wire = sent.Write().ToString(); var poison = new Mock(); var poisoned = CaptureSends(poison); var message = (await Create(ClientReturning(("1", wire)), () => poison.Object).ReceiveAsync())[0]; await message.DeadLetterAsync("gave up"); - var copy = poisoned.Should().ContainSingle().Subject.ToString(); - copy.Should().Be(wire); - var replayed = (await Create(ClientReturning(("1", copy))).ReceiveAsync())[0]; + var copy = poisoned.Should().ContainSingle().Subject; + MessageEnvelope.TryRead(copy, out var record, out _).Should().Be(EnvelopeKind.Envelope); + record!.TraceParent.Should().Be(sent.TraceParent); + record.DeadLetter.Should().BeEquivalentTo(new { Reason = "gave up", Category = DeadLetterCategories.Handler, MessageId = "1", DequeueCount = 2L, Queue = "orders" }); + + var replayed = (await Create(ClientReturning(("1", DeadLetterQueue.WithoutDeadLetterRecord(copy).ToString()))).ReceiveAsync())[0]; replayed.Body!.Property1.Should().Be("test"); replayed.CreationContext!.Value.TraceId.Should().Be(caller.TraceId); + replayed.WireBody.ToString().Should().Be(wire, "the replayed message is the one that was sent"); } [Fact(DisplayName = "Complete deletes with the current receipt, settles, and refuses a second operation")] @@ -264,17 +272,21 @@ public async Task Abandon_Makes_Visible_Now() client.Verify(x => x.UpdateMessageAsync("1", "receipt-1", (string?)null, TimeSpan.Zero, It.IsAny()), Times.Once); } - [Fact(DisplayName = "Dead-letter copies the raw body to the poison queue unchanged, then deletes the original")] - public async Task DeadLetter_Copies_Raw_Body_Then_Deletes() + [Fact(DisplayName = "Dead-letter copies the body to the poison queue with a record of why, never to expire, then deletes the original")] + public async Task DeadLetter_Copies_Body_With_Record_Then_Deletes() { var client = ClientReturning(("1", ValidJson)); var poison = new Mock(); + var poisoned = CaptureSends(poison); var message = (await Create(client, () => poison.Object).ReceiveAsync())[0]; await message.DeadLetterAsync("gave up"); poison.Verify(x => x.CreateIfNotExistsAsync(It.IsAny>(), It.IsAny()), Times.Once); - poison.Verify(x => x.SendMessageAsync(It.Is(b => b.ToString() == ValidJson), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + poison.Verify(x => x.SendMessageAsync(It.IsAny(), null, TimeSpan.FromSeconds(-1), It.IsAny()), Times.Once); + MessageEnvelope.TryRead(poisoned.Single(), out var record, out _).Should().Be(EnvelopeKind.Envelope); + record!.DeadLetter!.Reason.Should().Be("gave up"); + DeadLetterQueue.WithoutDeadLetterRecord(poisoned.Single()).ToString().Should().Be(ValidJson, "a plain message replays as it was sent"); client.Verify(x => x.DeleteMessageAsync("1", "receipt-1", It.IsAny()), Times.Once); message.Settlement.Should().Be(MessageSettlement.DeadLettered); } diff --git a/tests/AzureStorage.QueueService.Tests/TraceContextEnvelopeTests.cs b/tests/AzureStorage.QueueService.Tests/TraceContextEnvelopeTests.cs deleted file mode 100644 index 4053f5c..0000000 --- a/tests/AzureStorage.QueueService.Tests/TraceContextEnvelopeTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Diagnostics; -using AzureStorage.QueueService.Telemetry; -using FluentAssertions; - -namespace AzureStorage.QueueService.Tests; - -public class TraceContextEnvelopeTests -{ - private const string TraceParent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; - - [Fact(DisplayName = "A custom serializer's non-JSON output travels Base64-encoded and comes back byte for byte")] - public void Non_Json_Payload_Round_Trips() - { - byte[] bytes = [0xFF, 0x00, (byte)'{', 0x80]; - using var caller = new Activity("caller").Start(); - - var wire = TraceContextEnvelope.Wrap(new BinaryData(bytes), caller)!; - - wire.ToString().Should().Contain("\"body64\":"); - TraceContextEnvelope.TryUnwrap(wire, out var payload, out var context).Should().BeTrue(); - payload.ToArray().Should().Equal(bytes); - context!.Value.TraceId.Should().Be(caller.TraceId); - } - - [Theory(DisplayName = "A payload that only resembles an envelope is left alone")] - [InlineData("{\"traceparent\":\"" + TraceParent + "\",\"body\":{}}")] // no marker - [InlineData("{\"body\":{},\"$azqs-envelope\":1,\"traceparent\":\"" + TraceParent + "\"}")] // marker not first - [InlineData("{\"$azqs-envelope\":2,\"traceparent\":\"" + TraceParent + "\",\"body\":{}}")] // a version this reader does not know - [InlineData("{\"$azqs-envelope\":1,\"traceparent\":\"" + TraceParent + "\",\"body\":{},\"orderId\":7}")] // a property the format does not have - [InlineData("{\"$azqs-envelope\":1,\"traceparent\":\"" + TraceParent + "\",\"body\":{}")] // truncated - [InlineData("{\"$azqs-envelope\":1,\"traceparent\":\"" + TraceParent + "\",\"body64\":\"not base64!\"}")] // body64 that is not Base64 - public void Lookalikes_Are_Not_Unwrapped(string text) - { - TraceContextEnvelope.TryUnwrap(BinaryData.FromString(text), out var payload, out var context).Should().BeFalse(); - - payload.ToString().Should().Be(text); - context.Should().BeNull(); - } - - [Fact(DisplayName = "A lookalike with a string that is not valid UTF-8 is left alone rather than failing the receive")] - public void Lookalike_With_Invalid_Utf8_Is_Not_Unwrapped() - { - byte[] bytes = [.. "{\"$azqs-envelope\":1,\"traceparent\":\""u8, 0xFF, 0xFE, .. "\",\"body\":{}}"u8]; - - TraceContextEnvelope.TryUnwrap(new BinaryData(bytes), out var payload, out var context).Should().BeFalse(); - - payload.ToArray().Should().Equal(bytes); - context.Should().BeNull(); - } -} From a0524da7e0904053ca75b4e595c9a4500dae8b07 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:37:28 -0600 Subject: [PATCH 2/8] Add AzureStorage.QueueService.HealthChecks package Two IHealthChecksBuilder extensions: - AddAzureStorageQueueProcessors: reads IQueueProcessorController.GetStatus() (no storage calls). Unhealthy after N consecutive receive failures, Degraded when paused, stopping, or a handler exceeds MaxHandlerAge. - AddAzureStorageQueue: reads queue properties (billable call), optionally the poison queue count; per-run timeout (10 s default). Not yet added to the .sln, so CI and nuget.yml do not build or pack it. Co-Authored-By: Claude Opus 5.5 --- ...reStorage.QueueService.HealthChecks.csproj | 47 ++++++++ .../HealthCheckOptions.cs | 54 +++++++++ .../QueueHealthCheck.cs | 63 +++++++++++ .../QueueHealthChecksBuilderExtensions.cs | 106 ++++++++++++++++++ .../QueueProcessorHealthCheck.cs | 93 +++++++++++++++ .../README.md | 45 ++++++++ ...age.QueueService.HealthChecks.Tests.csproj | 34 ++++++ .../QueueHealthCheckTests.cs | 41 +++++++ .../QueueProcessorHealthCheckTests.cs | 86 ++++++++++++++ 9 files changed, 569 insertions(+) create mode 100644 src/AzureStorage.QueueService.HealthChecks/AzureStorage.QueueService.HealthChecks.csproj create mode 100644 src/AzureStorage.QueueService.HealthChecks/HealthCheckOptions.cs create mode 100644 src/AzureStorage.QueueService.HealthChecks/QueueHealthCheck.cs create mode 100644 src/AzureStorage.QueueService.HealthChecks/QueueHealthChecksBuilderExtensions.cs create mode 100644 src/AzureStorage.QueueService.HealthChecks/QueueProcessorHealthCheck.cs create mode 100644 src/AzureStorage.QueueService.HealthChecks/README.md create mode 100644 tests/AzureStorage.QueueService.HealthChecks.Tests/AzureStorage.QueueService.HealthChecks.Tests.csproj create mode 100644 tests/AzureStorage.QueueService.HealthChecks.Tests/QueueHealthCheckTests.cs create mode 100644 tests/AzureStorage.QueueService.HealthChecks.Tests/QueueProcessorHealthCheckTests.cs diff --git a/src/AzureStorage.QueueService.HealthChecks/AzureStorage.QueueService.HealthChecks.csproj b/src/AzureStorage.QueueService.HealthChecks/AzureStorage.QueueService.HealthChecks.csproj new file mode 100644 index 0000000..7de49db --- /dev/null +++ b/src/AzureStorage.QueueService.HealthChecks/AzureStorage.QueueService.HealthChecks.csproj @@ -0,0 +1,47 @@ + + + + net10.0 + enable + enable + $(MSBuildProjectName) + $(MSBuildProjectName.Replace(" ", "_")) + true + true + True + Unifinity Software Corporation + True + MIT + ASP.NET Core health checks for AzureStorage.QueueService: one that reports whether the hosted queue processors are receiving, paused or stuck (free, no storage calls), and one that checks a queue can be reached and, optionally, that its poison queue is empty. + Unifinity Software Corporation + Azure, Azure Storage, Azure Storage Queue, Storage Queue, Queue Service, Health Checks, HealthChecks, Jason Shave, Unifinity Software Corporation + https://github.com/jasonshave/AzureStorage.QueueService + logo.png + README.md + git + https://github.com/jasonshave/AzureStorage.QueueService + Unifinity Software Corporation + + $(VersionPrefix) + + + + + + + + + + + + + True + \ + + + True + \ + + + + diff --git a/src/AzureStorage.QueueService.HealthChecks/HealthCheckOptions.cs b/src/AzureStorage.QueueService.HealthChecks/HealthCheckOptions.cs new file mode 100644 index 0000000..683b1c8 --- /dev/null +++ b/src/AzureStorage.QueueService.HealthChecks/HealthCheckOptions.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace AzureStorage.QueueService.HealthChecks; + +/// +/// Settings for the processor health check added by +/// . +/// The defaults suit most applications; change them only when a threshold does not match how your handlers behave. +/// +public sealed class QueueProcessorHealthOptions +{ + /// + /// How many receives in a row must fail before a processor counts as failing and the check reports its failure + /// status (Unhealthy unless you chose another). A single failed receive is often a network blip the processor + /// recovers from on its own, so the default is 3. Must be at least 1. + /// + public int UnhealthyAfterReceiveFailures { get; set; } = 3; + + /// + /// What to report while a processor is paused through . Paused + /// processors receive nothing, so the default is . Set it to + /// when pausing is a normal part of how you run (for example, a circuit breaker + /// that pauses while a downstream service is down). + /// + public HealthStatus PausedStatus { get; set; } = HealthStatus.Degraded; + + /// + /// How long a single message may stay in its handler before the check reports , + /// which usually means a handler is stuck. Null (the default) turns this off. Set it comfortably above the longest + /// time a handler normally takes. Must be greater than zero when set. + /// + public TimeSpan? MaxHandlerAge { get; set; } +} + +/// +/// Settings for the queue health check added by +/// . +/// +public sealed class QueueHealthOptions +{ + /// + /// When true, the check also reads how many messages are on the queue's poison queue (where messages go after + /// failing too many times) and reports when there are any. This costs a second + /// storage call on every run. Off by default. + /// + public bool DegradedWhenPoisonMessages { get; set; } + + /// + /// How long one run of the check may take before it gives up and reports its failure status. The Azure SDK retries + /// a failing call several times, so without a limit an unreachable storage account can hold a health probe open for + /// minutes. Defaults to 10 seconds; null means no limit. + /// + public TimeSpan? Timeout { get; set; } = TimeSpan.FromSeconds(10); +} diff --git a/src/AzureStorage.QueueService.HealthChecks/QueueHealthCheck.cs b/src/AzureStorage.QueueService.HealthChecks/QueueHealthCheck.cs new file mode 100644 index 0000000..72be73b --- /dev/null +++ b/src/AzureStorage.QueueService.HealthChecks/QueueHealthCheck.cs @@ -0,0 +1,63 @@ +using Azure; +using Azure.Storage.Queues.Models; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace AzureStorage.QueueService.HealthChecks; + +/// +/// Checks that a client's queue can be reached by reading its properties, and optionally that its poison queue is +/// empty. Every run makes one or two billable storage calls. +/// +internal sealed class QueueHealthCheck(IQueueClientFactory factory, string clientName, QueueHealthOptions options) : IHealthCheck +{ + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var data = new Dictionary(StringComparer.Ordinal); + string? queueName = null; + try + { + var queue = clientName.Length == 0 ? factory.GetQueue() : factory.GetQueue(clientName); + queueName = queue.QueueName; + data["queue"] = queueName; + + var properties = await queue.GetPropertiesAsync(cancellationToken); + data["approximateMessageCount"] = properties.ApproximateMessagesCount; + + if (options.DegradedWhenPoisonMessages) + { + var poison = factory.GetDeadLetterQueue(clientName); + var poisonCount = await poison.GetApproximateCountAsync(cancellationToken); + data["poisonQueue"] = poison.QueueName; + data["poisonMessageCount"] = poisonCount; + + if (poisonCount > 0) + return HealthCheckResult.Degraded( + $"Queue '{queueName}' is reachable, but its poison queue '{poison.QueueName}' holds about {poisonCount} message(s) that failed processing.", + data: data); + } + + return HealthCheckResult.Healthy($"Queue '{queueName}' is reachable.", data); + } + catch (Exception e) when (!cancellationToken.IsCancellationRequested) + { + return new HealthCheckResult(context.Registration.FailureStatus, Explain(e, queueName), e, data); + } + } + + private string Explain(Exception e, string? queueName) + { + var queue = queueName is null ? $"The queue for client {Client()}" : $"Queue '{queueName}'"; + return e switch + { + RequestFailedException { ErrorCode: var code } when code == QueueErrorCode.QueueNotFound => + $"{queue} does not exist.", + RequestFailedException { Status: 401 or 403 } r => + $"{queue} could not be read: the storage account refused access ({r.Status} {r.ErrorCode}). Check the credential and its role assignment.", + RequestFailedException r => + $"{queue} could not be read: the storage service returned {r.Status} {r.ErrorCode}.", + _ => $"{queue} could not be read: {e.Message}", + }; + } + + private string Client() => clientName.Length == 0 ? "(the default client)" : $"'{clientName}'"; +} diff --git a/src/AzureStorage.QueueService.HealthChecks/QueueHealthChecksBuilderExtensions.cs b/src/AzureStorage.QueueService.HealthChecks/QueueHealthChecksBuilderExtensions.cs new file mode 100644 index 0000000..c4fa746 --- /dev/null +++ b/src/AzureStorage.QueueService.HealthChecks/QueueHealthChecksBuilderExtensions.cs @@ -0,0 +1,106 @@ +using AzureStorage.QueueService; +using AzureStorage.QueueService.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Adds AzureStorage.QueueService health checks to services.AddHealthChecks(). Expose them with ASP.NET Core's +/// app.MapHealthChecks("/health"), or read them through . +/// +public static class QueueHealthChecksBuilderExtensions +{ + /// + /// Adds a check that reports how the hosted queue processors are doing, from + /// . It reads in-memory state only and makes no storage calls, so + /// it is free to run on every probe. + /// + /// + /// The result is the worst of these, across every processor: + /// + /// (Unhealthy by default) when a processor's last + /// receives (3 by default) all failed: it + /// cannot reach its queue, the queue is missing, or it has lost access. + /// (Degraded by default) when a processor is paused. + /// Degraded when a message has been in its handler longer than + /// (off unless you set it), or when a processor is stopping + /// or stopped because the host is shutting down. + /// + /// A processor that has not started yet (the host is still starting) counts as healthy. The description names + /// each processor with a problem and says why, and the result's data has one entry per processor, keyed + /// "{queue}/{handler}", with its state, receive failures, last error and active handlers. + /// + /// The builder from services.AddHealthChecks(). + /// The check's name, as it appears in the health report. + /// What to report when a processor cannot receive; Unhealthy when null. + /// Tags for filtering checks, for example to serve this one only on a liveness endpoint. + /// Changes the thresholds; see . + /// The same builder, for chaining. + /// A threshold in is out of range. + public static IHealthChecksBuilder AddAzureStorageQueueProcessors( + this IHealthChecksBuilder builder, + string name = "azure-storage-queue-processors", + HealthStatus? failureStatus = null, + IEnumerable? tags = null, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + + var options = new QueueProcessorHealthOptions(); + configure?.Invoke(options); + ArgumentOutOfRangeException.ThrowIfLessThan(options.UnhealthyAfterReceiveFailures, 1, nameof(QueueProcessorHealthOptions.UnhealthyAfterReceiveFailures)); + if (options.MaxHandlerAge is { } age) ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(age, TimeSpan.Zero, nameof(QueueProcessorHealthOptions.MaxHandlerAge)); + + return builder.Add(new HealthCheckRegistration( + name, + provider => new QueueProcessorHealthCheck(provider.GetRequiredService(), options), + failureStatus, + tags)); + } + + /// + /// Adds a check that a queue client's queue can be reached, by reading the queue's properties. Optionally it also + /// reports Degraded while the queue's poison queue holds messages (see + /// ). + /// + /// + /// Every run is a real, billable call to Azure Storage (two with the poison-queue option), so mind how often your + /// probes run it, and consider tagging it so only an occasional readiness probe uses it. A missing queue, a refused + /// credential, an unreachable account or a run longer than reports + /// , with the exception attached. + /// + /// The builder from services.AddHealthChecks(). + /// The queue client to check, as registered; null for the default client. + /// The check's name, as it appears in the health report; null for + /// "azure-storage-queue", or "azure-storage-queue-{clientName}" for a named client. + /// What to report when the queue cannot be read; Unhealthy when null. + /// Tags for filtering checks, for example to serve this one only on a readiness endpoint. + /// Changes the check's settings; see . + /// The same builder, for chaining. + /// is zero or negative. + public static IHealthChecksBuilder AddAzureStorageQueue( + this IHealthChecksBuilder builder, + string? clientName = null, + string? name = null, + HealthStatus? failureStatus = null, + IEnumerable? tags = null, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + var options = new QueueHealthOptions(); + configure?.Invoke(options); + if (options.Timeout is { } timeout) ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeout, TimeSpan.Zero, nameof(QueueHealthOptions.Timeout)); + + var client = clientName ?? string.Empty; + name ??= client.Length == 0 ? "azure-storage-queue" : $"azure-storage-queue-{client}"; + + return builder.Add(new HealthCheckRegistration( + name, + provider => new QueueHealthCheck(provider.GetRequiredService(), client, options), + failureStatus, + tags, + options.Timeout)); + } +} diff --git a/src/AzureStorage.QueueService.HealthChecks/QueueProcessorHealthCheck.cs b/src/AzureStorage.QueueService.HealthChecks/QueueProcessorHealthCheck.cs new file mode 100644 index 0000000..9d5a811 --- /dev/null +++ b/src/AzureStorage.QueueService.HealthChecks/QueueProcessorHealthCheck.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace AzureStorage.QueueService.HealthChecks; + +/// +/// Reports on the hosted processors from . It reads in-memory state +/// only, so it costs nothing to run as often as a probe likes. +/// +internal sealed class QueueProcessorHealthCheck(IQueueProcessorController controller, QueueProcessorHealthOptions options) : IHealthCheck +{ + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var statuses = controller.GetStatus(); + if (statuses.Count == 0) + return Task.FromResult(HealthCheckResult.Healthy("No queue processors are registered.")); + + var worst = HealthStatus.Healthy; + var problems = new List<(HealthStatus Status, string Text)>(); + var data = new Dictionary(StringComparer.Ordinal); + + foreach (var s in statuses) + { + var key = $"{s.QueueName}/{s.HandlerName}"; + if (data.ContainsKey(key)) key = $"{s.ClientName}:{key}"; + data[key] = Describe(s); + + foreach (var (status, text) in Assess(s, context.Registration.FailureStatus)) + { + problems.Add((status, $"{key} {text}.")); + if (status < worst) worst = status; + } + } + + if (problems.Count == 0) + { + var waiting = statuses.Count(s => s.State == QueueProcessorState.NotStarted); + var healthy = waiting == 0 + ? $"{Count(statuses.Count, "queue processor")} running." + : $"{Count(statuses.Count - waiting, "queue processor")} running, {waiting} waiting for the host to finish starting."; + return Task.FromResult(HealthCheckResult.Healthy(healthy, data)); + } + + // Worst first, so the first sentence is the one that decided the result. + var description = string.Join(" ", problems.OrderBy(p => p.Status).Select(p => p.Text)); + return Task.FromResult(new HealthCheckResult(worst, description, data: data)); + } + + private IEnumerable<(HealthStatus Status, string Text)> Assess(QueueProcessorStatus s, HealthStatus failureStatus) + { + if (s.ConsecutiveReceiveFailures >= options.UnhealthyAfterReceiveFailures) + { + var error = s.LastReceiveError is null ? "" : $" (last error: {s.LastReceiveError})"; + yield return (failureStatus, $"cannot receive: the last {s.ConsecutiveReceiveFailures} attempts failed{error}"); + } + + switch (s.State) + { + case QueueProcessorState.Paused when options.PausedStatus != HealthStatus.Healthy: + yield return (options.PausedStatus, "is paused and not receiving"); + break; + case QueueProcessorState.Stopping: + yield return (HealthStatus.Degraded, "is stopping because the host is shutting down"); + break; + case QueueProcessorState.Stopped: + yield return (HealthStatus.Degraded, "has stopped and is not receiving"); + break; + } + + if (options.MaxHandlerAge is { } limit && s.OldestHandlerAge is { } age && age > limit) + yield return (HealthStatus.Degraded, $"has had a message in its handler for {Format(age)}, longer than the {Format(limit)} allowed; the handler may be stuck"); + } + + private static Dictionary Describe(QueueProcessorStatus s) => new(StringComparer.Ordinal) + { + ["client"] = s.ClientName, + ["queue"] = s.QueueName, + ["handler"] = s.HandlerName, + ["state"] = s.State.ToString(), + ["activeHandlers"] = s.ActiveHandlers, + ["maxConcurrentMessages"] = s.MaxConcurrentMessages, + ["consecutiveReceiveFailures"] = s.ConsecutiveReceiveFailures, + ["lastReceiveSucceededAt"] = s.LastReceiveSucceededAt, + ["lastReceiveError"] = s.LastReceiveError, + ["oldestHandlerAge"] = s.OldestHandlerAge, + }; + + private static string Count(int n, string noun) => n == 1 ? $"1 {noun}" : $"{n} {noun}s"; + + private static string Format(TimeSpan t) => + t.TotalHours >= 1 ? $"{(int)t.TotalHours}h {t.Minutes}m" + : t.TotalMinutes >= 1 ? $"{t.Minutes}m {t.Seconds}s" + : $"{Math.Max(0, (int)t.TotalSeconds)}s"; +} diff --git a/src/AzureStorage.QueueService.HealthChecks/README.md b/src/AzureStorage.QueueService.HealthChecks/README.md new file mode 100644 index 0000000..6c04cdf --- /dev/null +++ b/src/AzureStorage.QueueService.HealthChecks/README.md @@ -0,0 +1,45 @@ +# AzureStorage.QueueService.HealthChecks + +Health checks for [AzureStorage.QueueService](https://github.com/jasonshave/AzureStorage.QueueService), for use with +ASP.NET Core's `AddHealthChecks()` and `MapHealthChecks()`. + +``` +dotnet add package AzureStorage.QueueService.HealthChecks +``` + +## The two checks + +**`AddAzureStorageQueueProcessors()`** reports whether your hosted queue processors are working. It reads their +in-memory state only, so it makes no storage calls and costs nothing to run on every probe. + +- **Unhealthy** when a processor's last 3 receives all failed (it cannot reach its queue, the queue is missing, or it + lost access). Change the count with `UnhealthyAfterReceiveFailures`. +- **Degraded** when a processor is paused (change with `PausedStatus`), or when a message has been in its handler + longer than `MaxHandlerAge` (off unless you set it; a sign of a stuck handler). +- **Healthy** otherwise, including while the host is still starting. + +**`AddAzureStorageQueue(clientName)`** checks that a queue can be reached by reading its properties. With +`DegradedWhenPoisonMessages` it also reports Degraded while the poison queue (where messages go after failing too many +times) holds any messages. **Each run is a billable Azure Storage call** (two with the poison option), so keep it off +a probe that runs every few seconds. A run that takes longer than `Timeout` (10 seconds by default) is reported as +failed rather than holding the probe open. + +## Example + +```csharp +builder.Services.AddAzureStorageQueueClient(queues => + queues.AddClient("orders", settings => builder.Configuration.Bind("Queues:Orders", settings)) + .AddProcessor()); + +builder.Services.AddHealthChecks() + .AddAzureStorageQueueProcessors(tags: ["live"], configure: o => o.MaxHandlerAge = TimeSpan.FromMinutes(10)) + .AddAzureStorageQueue("orders", tags: ["ready"], configure: o => o.DegradedWhenPoisonMessages = true); + +var app = builder.Build(); +app.MapHealthChecks("/health/live", new() { Predicate = check => check.Tags.Contains("live") }); +app.MapHealthChecks("/health/ready", new() { Predicate = check => check.Tags.Contains("ready") }); +``` + +By default `MapHealthChecks` answers 200 for Healthy and Degraded and 503 for Unhealthy. Each result's description +says which processor or queue has a problem and why, and its data carries the details (state, receive failures, last +error, message counts) for a response writer to show. diff --git a/tests/AzureStorage.QueueService.HealthChecks.Tests/AzureStorage.QueueService.HealthChecks.Tests.csproj b/tests/AzureStorage.QueueService.HealthChecks.Tests/AzureStorage.QueueService.HealthChecks.Tests.csproj new file mode 100644 index 0000000..6604490 --- /dev/null +++ b/tests/AzureStorage.QueueService.HealthChecks.Tests/AzureStorage.QueueService.HealthChecks.Tests.csproj @@ -0,0 +1,34 @@ + + + + net10.0 + enable + enable + + false + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/tests/AzureStorage.QueueService.HealthChecks.Tests/QueueHealthCheckTests.cs b/tests/AzureStorage.QueueService.HealthChecks.Tests/QueueHealthCheckTests.cs new file mode 100644 index 0000000..582f51a --- /dev/null +++ b/tests/AzureStorage.QueueService.HealthChecks.Tests/QueueHealthCheckTests.cs @@ -0,0 +1,41 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace AzureStorage.QueueService.HealthChecks.Tests; + +/// The queue check against in-memory queues: the real factory, queue and poison queue, no storage account. +public class QueueHealthCheckTests +{ + private sealed record Order(string Id); + + [Fact(DisplayName = "Queue: healthy while reachable, degraded once a message is dead-lettered, unhealthy for a client that is not registered")] + public async Task Reports_Reachability_And_Poison_Messages() + { + var services = new ServiceCollection().AddLogging(); + services.AddAzureStorageQueueClient(queues => queues.AddClient("orders", settings => settings.QueueName = "orders")); + services.AddInMemoryQueues(); + services.AddHealthChecks() + .AddAzureStorageQueue("orders", configure: o => o.DegradedWhenPoisonMessages = true) + .AddAzureStorageQueue("missing"); + await using var provider = services.BuildServiceProvider(); + var health = provider.GetRequiredService(); + + var before = await health.CheckHealthAsync(); + before.Entries["azure-storage-queue-orders"].Status.Should().Be(HealthStatus.Healthy); + var missing = before.Entries["azure-storage-queue-missing"]; + missing.Status.Should().Be(HealthStatus.Unhealthy); + missing.Exception.Should().BeOfType(); + + var queue = provider.GetRequiredKeyedService("orders"); + await queue.SendAsync(new Order("1")); + var received = (await queue.ReceiveAsync()).Single(); + await received.DeadLetterAsync("bad order"); + + var after = (await health.CheckHealthAsync()).Entries["azure-storage-queue-orders"]; + after.Status.Should().Be(HealthStatus.Degraded); + after.Data["poisonQueue"].Should().Be("orders-poison"); + after.Data["poisonMessageCount"].Should().Be(1); + after.Description.Should().Contain("'orders-poison' holds about 1 message(s)"); + } +} diff --git a/tests/AzureStorage.QueueService.HealthChecks.Tests/QueueProcessorHealthCheckTests.cs b/tests/AzureStorage.QueueService.HealthChecks.Tests/QueueProcessorHealthCheckTests.cs new file mode 100644 index 0000000..d32fd31 --- /dev/null +++ b/tests/AzureStorage.QueueService.HealthChecks.Tests/QueueProcessorHealthCheckTests.cs @@ -0,0 +1,86 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace AzureStorage.QueueService.HealthChecks.Tests; + +/// +/// The processor check, fed a fake controller: the thresholds that decide Healthy, Degraded and Unhealthy. Run through +/// as an application would, so the registration (name, failure status) is covered too. +/// +public class QueueProcessorHealthCheckTests +{ + private const string CheckName = "azure-storage-queue-processors"; + + private sealed class FakeController(params QueueProcessorStatus[] statuses) : IQueueProcessorController + { + public IReadOnlyList GetStatus() => statuses; + public void Pause(string clientName) => throw new NotSupportedException(); + public void Resume(string clientName) => throw new NotSupportedException(); + public bool IsPaused(string clientName) => false; + } + + private static QueueProcessorStatus Processor( + string queue = "orders", + QueueProcessorState state = QueueProcessorState.Running, + int failures = 0, + string? error = null, + TimeSpan? oldestHandlerAge = null) => + new("", queue, "OrderHandler", typeof(string), state, oldestHandlerAge is null ? 0 : 1, 16, + DateTimeOffset.UnixEpoch, failures > 0 ? DateTimeOffset.UnixEpoch : null, failures, error, oldestHandlerAge); + + private static async Task CheckAsync(IQueueProcessorController controller, Action? configure = null) + { + var services = new ServiceCollection().AddLogging().AddSingleton(controller); + services.AddHealthChecks().AddAzureStorageQueueProcessors(configure: configure); + await using var provider = services.BuildServiceProvider(); + var report = await provider.GetRequiredService().CheckHealthAsync(); + return report.Entries[CheckName]; + } + + [Fact(DisplayName = "Processors: running and not-yet-started processors are healthy, with one data entry each")] + public async Task Running_And_Starting_Processors_Are_Healthy() + { + var result = await CheckAsync(new FakeController(Processor("orders"), Processor("invoices", QueueProcessorState.NotStarted))); + + result.Status.Should().Be(HealthStatus.Healthy); + result.Description.Should().Be("1 queue processor running, 1 waiting for the host to finish starting."); + result.Data.Keys.Should().BeEquivalentTo("orders/OrderHandler", "invoices/OrderHandler"); + result.Data["orders/OrderHandler"].Should().BeAssignableTo>() + .Which["state"].Should().Be("Running"); + } + + [Theory(DisplayName = "Processors: receive failures turn the check unhealthy only at the threshold, naming the processor and the error")] + [InlineData(2, HealthStatus.Healthy)] + [InlineData(3, HealthStatus.Unhealthy)] + public async Task Consecutive_Receive_Failures_Are_Unhealthy_At_The_Threshold(int failures, HealthStatus expected) + { + var result = await CheckAsync(new FakeController(Processor(failures: failures, error: "Connection refused"))); + + result.Status.Should().Be(expected); + if (expected == HealthStatus.Unhealthy) + result.Description.Should().Be("orders/OrderHandler cannot receive: the last 3 attempts failed (last error: Connection refused)."); + } + + [Theory(DisplayName = "Processors: a paused processor is degraded unless PausedStatus says pausing is normal")] + [InlineData(HealthStatus.Degraded)] + [InlineData(HealthStatus.Healthy)] + public async Task Paused_Processor_Reports_PausedStatus(HealthStatus pausedStatus) + { + var result = await CheckAsync(new FakeController(Processor(state: QueueProcessorState.Paused)), o => o.PausedStatus = pausedStatus); + + result.Status.Should().Be(pausedStatus); + } + + [Fact(DisplayName = "Processors: a handler running longer than MaxHandlerAge is degraded; with no limit set it is not")] + public async Task Long_Running_Handler_Is_Degraded_Only_With_MaxHandlerAge() + { + var controller = new FakeController(Processor(oldestHandlerAge: TimeSpan.FromMinutes(11))); + + (await CheckAsync(controller)).Status.Should().Be(HealthStatus.Healthy, "the age limit is off by default"); + + var limited = await CheckAsync(controller, o => o.MaxHandlerAge = TimeSpan.FromMinutes(10)); + limited.Status.Should().Be(HealthStatus.Degraded); + limited.Description.Should().Contain("for 11m 0s, longer than the 10m 0s allowed"); + } +} From 10b9ef8c2fcd144047f2e190f1ea2739df95f284 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:38:30 -0600 Subject: [PATCH 3/8] WIP: read-all helper, XML docs, AOT/trim analysis --- .claude/worktrees/agent-abe49847f2bdc9f48 | 1 + .claude/worktrees/agent-afb7e0b0c74b2bbe0 | 1 + .../AzureStorage.QueueService.csproj | 7 ++ .../CloudEventHandlerAdapter.cs | 2 +- .../DeserializationException.cs | 4 +- .../ICloudEventHandler.cs | 10 +- src/AzureStorage.QueueService/INamedQueue.cs | 1 + .../IQueueClientBuilder.cs | 1 + .../IQueueMessageHandler.cs | 5 +- .../IQueueMessageOperations.cs | 1 + .../IQueueMessageSerializer.cs | 3 + .../ISdkQueueClientFactory.cs | 1 + .../IStorageQueue.cs | 6 + src/AzureStorage.QueueService/LeaseRenewal.cs | 3 + .../QueueClientBuilderExtensions.cs | 19 +-- .../QueueClientFactory.cs | 6 +- .../QueueClientSettingsBuilder.cs | 34 +++++- .../QueueMessageMiddleware.cs | 2 +- .../QueueReceiverExtensions.cs | 110 ++++++++++++++++++ .../QueueTransport.cs | 6 +- src/AzureStorage.QueueService/Routing.cs | 5 +- .../SdkQueueClientFactory.cs | 2 +- .../QueueServiceTelemetrySettings.cs | 6 +- .../TracerProviderBuilderExtensions.cs | 4 +- 24 files changed, 210 insertions(+), 30 deletions(-) create mode 160000 .claude/worktrees/agent-abe49847f2bdc9f48 create mode 160000 .claude/worktrees/agent-afb7e0b0c74b2bbe0 create mode 100644 src/AzureStorage.QueueService/QueueReceiverExtensions.cs diff --git a/.claude/worktrees/agent-abe49847f2bdc9f48 b/.claude/worktrees/agent-abe49847f2bdc9f48 new file mode 160000 index 0000000..a0524da --- /dev/null +++ b/.claude/worktrees/agent-abe49847f2bdc9f48 @@ -0,0 +1 @@ +Subproject commit a0524da7e0904053ca75b4e595c9a4500dae8b07 diff --git a/.claude/worktrees/agent-afb7e0b0c74b2bbe0 b/.claude/worktrees/agent-afb7e0b0c74b2bbe0 new file mode 160000 index 0000000..b225c3c --- /dev/null +++ b/.claude/worktrees/agent-afb7e0b0c74b2bbe0 @@ -0,0 +1 @@ +Subproject commit b225c3cd600ff29a8ea17b94f2c2bac877a91660 diff --git a/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj b/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj index a6aa590..3546ec5 100644 --- a/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj +++ b/src/AzureStorage.QueueService/AzureStorage.QueueService.csproj @@ -21,6 +21,13 @@ Unifinity Software Corporation $(VersionPrefix) + + true + + true + true + + $(NoWarn);SYSLIB1100;SYSLIB1101 diff --git a/src/AzureStorage.QueueService/CloudEventHandlerAdapter.cs b/src/AzureStorage.QueueService/CloudEventHandlerAdapter.cs index f7159a3..9e7cadd 100644 --- a/src/AzureStorage.QueueService/CloudEventHandlerAdapter.cs +++ b/src/AzureStorage.QueueService/CloudEventHandlerAdapter.cs @@ -5,7 +5,7 @@ namespace AzureStorage.QueueService; /// /// The processor handler behind AddCloudEventProcessor<TData, THandler>(). The processor receives and /// settles the CloudEvent envelope like any other message, and reads its data as part of reading the body -/// (, its after-deserialize step); this hands the event and its data to the application's +/// (, its after-deserialize step); this hands the event and its data to the application's /// handler. Created per message, in the message's scope. /// internal sealed class CloudEventHandlerAdapter : IQueueMessageHandler diff --git a/src/AzureStorage.QueueService/DeserializationException.cs b/src/AzureStorage.QueueService/DeserializationException.cs index 7ffbdfb..1798495 100644 --- a/src/AzureStorage.QueueService/DeserializationException.cs +++ b/src/AzureStorage.QueueService/DeserializationException.cs @@ -1,4 +1,4 @@ -namespace AzureStorage.QueueService; +namespace AzureStorage.QueueService; /// /// Thrown when a queue message cannot be deserialized into the requested type. The original serializer error @@ -6,11 +6,13 @@ /// public class DeserializationException : Exception { + /// Creates the exception with a message saying what could not be read. public DeserializationException(string message) : base(message) { } + /// Creates the exception with the serializer's own error as . public DeserializationException(string message, Exception innerException) : base(message, innerException) { diff --git a/src/AzureStorage.QueueService/ICloudEventHandler.cs b/src/AzureStorage.QueueService/ICloudEventHandler.cs index 4490248..6390573 100644 --- a/src/AzureStorage.QueueService/ICloudEventHandler.cs +++ b/src/AzureStorage.QueueService/ICloudEventHandler.cs @@ -4,16 +4,16 @@ namespace AzureStorage.QueueService; /// /// Handles one CloudEvent for a processor registered with AddCloudEventProcessor<TData, THandler>(): -/// is the event's data read as , and +/// data is the event's data read as , and /// is the event itself, for its type, source, id, subject, time and extension /// attributes. A new instance is resolved from a new dependency-injection scope for every message. /// /// /// Everything says about settlement, retries and cancellation applies: /// return and the message is completed, throw and it is retried, or settle it yourself through the message. An -/// event with no data arrives with null. With +/// event with no data arrives with data null. With /// off, an envelope or data that could not be -/// read arrives with null and set. +/// read arrives with data null and set. /// /// /// The type of the event's data, read with the client's serializer. Use for the raw bytes, @@ -21,5 +21,9 @@ namespace AzureStorage.QueueService; /// public interface ICloudEventHandler { + /// Handles one event. + /// The message; its is the event, for its type, source, id, subject and time. + /// The event's data read as ; null when the event has none. + /// Cancelled when the host shuts down and the drain timeout has passed. ValueTask HandleAsync(QueueMessage message, TData? data, CancellationToken cancellationToken); } diff --git a/src/AzureStorage.QueueService/INamedQueue.cs b/src/AzureStorage.QueueService/INamedQueue.cs index 14a438d..2a59731 100644 --- a/src/AzureStorage.QueueService/INamedQueue.cs +++ b/src/AzureStorage.QueueService/INamedQueue.cs @@ -6,5 +6,6 @@ namespace AzureStorage.QueueService; /// public interface INamedQueue { + /// The name of the queue on the service. string QueueName { get; } } diff --git a/src/AzureStorage.QueueService/IQueueClientBuilder.cs b/src/AzureStorage.QueueService/IQueueClientBuilder.cs index 8b42e1d..ced677a 100644 --- a/src/AzureStorage.QueueService/IQueueClientBuilder.cs +++ b/src/AzureStorage.QueueService/IQueueClientBuilder.cs @@ -12,6 +12,7 @@ public interface IQueueClientBuilder /// The registered name; for the default client. string Name { get; } + /// The application's services, for extension methods to register what they need. IServiceCollection Services { get; } } diff --git a/src/AzureStorage.QueueService/IQueueMessageHandler.cs b/src/AzureStorage.QueueService/IQueueMessageHandler.cs index 66230f0..9d060c4 100644 --- a/src/AzureStorage.QueueService/IQueueMessageHandler.cs +++ b/src/AzureStorage.QueueService/IQueueMessageHandler.cs @@ -8,11 +8,14 @@ namespace AzureStorage.QueueService; /// /// Return normally and the processor completes the message; throw and it is abandoned and retried later; or settle /// it yourself with , or -/// and the processor leaves it alone. With +/// and the processor leaves it alone. With /// off, /// can be null with set. The token is cancelled when the host shuts down. /// public interface IQueueMessageHandler { + /// Handles one message. + /// The message, with its body read as , its identity, and the operations that settle it. + /// Cancelled when the host shuts down and the drain timeout has passed. ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken); } diff --git a/src/AzureStorage.QueueService/IQueueMessageOperations.cs b/src/AzureStorage.QueueService/IQueueMessageOperations.cs index 3a9688e..e8d4d6e 100644 --- a/src/AzureStorage.QueueService/IQueueMessageOperations.cs +++ b/src/AzureStorage.QueueService/IQueueMessageOperations.cs @@ -6,6 +6,7 @@ namespace AzureStorage.QueueService; /// public interface IQueueMessageOperations { + /// Deletes the message: what does. ValueTask DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken); /// diff --git a/src/AzureStorage.QueueService/IQueueMessageSerializer.cs b/src/AzureStorage.QueueService/IQueueMessageSerializer.cs index ea0e358..164f888 100644 --- a/src/AzureStorage.QueueService/IQueueMessageSerializer.cs +++ b/src/AzureStorage.QueueService/IQueueMessageSerializer.cs @@ -7,8 +7,11 @@ namespace AzureStorage.QueueService; /// public interface IQueueMessageSerializer { + /// Writes as the bytes that go on the queue. BinaryData Serialize(T value); + /// Reads a message body back as . + /// The body could not be read as . T? Deserialize(BinaryData data); } diff --git a/src/AzureStorage.QueueService/ISdkQueueClientFactory.cs b/src/AzureStorage.QueueService/ISdkQueueClientFactory.cs index 78f18da..7861159 100644 --- a/src/AzureStorage.QueueService/ISdkQueueClientFactory.cs +++ b/src/AzureStorage.QueueService/ISdkQueueClientFactory.cs @@ -7,6 +7,7 @@ namespace AzureStorage.QueueService; /// internal interface ISdkQueueClientFactory { + /// The client's finished settings. /// Names the client (for example "Queue client 'orders'") so configuration errors can say which one is at fault. QueueClient CreateClient(string clientDescription, QueueClientSettings settings); } diff --git a/src/AzureStorage.QueueService/IStorageQueue.cs b/src/AzureStorage.QueueService/IStorageQueue.cs index e46fb3d..bea4c64 100644 --- a/src/AzureStorage.QueueService/IStorageQueue.cs +++ b/src/AzureStorage.QueueService/IStorageQueue.cs @@ -8,9 +8,15 @@ namespace AzureStorage.QueueService; /// public interface IStorageQueue : IQueueSender, IQueueReceiver { + /// Creates the queue, with , unless it already exists (an existing queue keeps its own metadata). ValueTask CreateIfNotExistsAsync(IDictionary? metadata = null, CancellationToken cancellationToken = default); + /// Deletes every message on the queue, hidden ones included. ValueTask ClearAsync(CancellationToken cancellationToken = default); + /// + /// The queue's metadata and approximate message count. The count includes messages that are hidden (being + /// processed, or waiting for a retry), so it is not the number a receive would return. + /// ValueTask GetPropertiesAsync(CancellationToken cancellationToken = default); } diff --git a/src/AzureStorage.QueueService/LeaseRenewal.cs b/src/AzureStorage.QueueService/LeaseRenewal.cs index 723678e..10564c5 100644 --- a/src/AzureStorage.QueueService/LeaseRenewal.cs +++ b/src/AzureStorage.QueueService/LeaseRenewal.cs @@ -22,6 +22,9 @@ private LeaseRenewal(Func renewAsync, Func isSettled, TimeSpan _loop = RunAsync(renewAsync, isSettled, interval, time, _stopWaiting.Token, onFailed, maxDuration, onLimitReached); } + /// How long to wait before each renewal. + /// The clock the waits use. + /// Called once if a renewal fails; renewal stops there. /// Renews the lease once. Must not observe a cancellation token: an accepted renewal changes the receipt, so the response must be read. /// True once the message no longer needs renewing. /// The processor-wide signal that ends every renewal, for example at shutdown. diff --git a/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs b/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs index db27ee0..b27c44e 100644 --- a/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs +++ b/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using System.Text.Json; using Azure.Messaging; using AzureStorage.QueueService.Telemetry; @@ -98,7 +99,7 @@ public static IQueueClientBuilder UseLargeMessageStore(this IQueueClientBuilder /// Runs around every message this client's processors deliver, resolved from the /// message's scope (registered as scoped unless you registered it yourself). See . /// - public static IQueueClientBuilder UseMiddleware(this IQueueClientBuilder builder) + public static IQueueClientBuilder UseMiddleware<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TMiddleware>(this IQueueClientBuilder builder) where TMiddleware : class, IQueueMessageMiddleware { ArgumentNullException.ThrowIfNull(builder); @@ -127,7 +128,7 @@ public static IQueueClientBuilder Use(this IQueueClientBuilder builder, Func - public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, Action? configure = null) + public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, Action? configure = null) where THandler : class, IQueueMessageHandler { ArgumentNullException.ThrowIfNull(builder); @@ -141,13 +142,13 @@ public static IQueueClientBuilder AddProcessor(this IQueueCl /// options bound from (for example configuration.GetSection("Queues:Orders:Processor")) /// and then , so MaxConcurrentMessages and the rest can differ by environment. /// - public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, IConfiguration configuration, Action? configure = null) + public static IQueueClientBuilder AddProcessor(this IQueueClientBuilder builder, IConfiguration configuration, Action? configure = null) where THandler : class, IQueueMessageHandler { ArgumentNullException.ThrowIfNull(configuration); return builder.AddProcessor(options => { - configuration.Bind(options); + BindProcessorOptions(configuration, options); configure?.Invoke(options); }); } @@ -186,7 +187,7 @@ public static IQueueClientBuilder AddProcessor(this IQueueClientBuilde /// read as counts as an unreadable body (see /// ). /// - public static IQueueClientBuilder AddCloudEventProcessor(this IQueueClientBuilder builder, Action? configure = null) + public static IQueueClientBuilder AddCloudEventProcessor(this IQueueClientBuilder builder, Action? configure = null) where THandler : class, ICloudEventHandler { ArgumentNullException.ThrowIfNull(builder); @@ -200,13 +201,13 @@ public static IQueueClientBuilder AddCloudEventProcessor(this I } /// As , with the options bound from first. - public static IQueueClientBuilder AddCloudEventProcessor(this IQueueClientBuilder builder, IConfiguration configuration, Action? configure = null) + public static IQueueClientBuilder AddCloudEventProcessor(this IQueueClientBuilder builder, IConfiguration configuration, Action? configure = null) where THandler : class, ICloudEventHandler { ArgumentNullException.ThrowIfNull(configuration); return builder.AddCloudEventProcessor(options => { - configuration.Bind(options); + BindProcessorOptions(configuration, options); configure?.Invoke(options); }); } @@ -253,6 +254,10 @@ public static IQueueClientBuilder AddCloudEventRouter(this IQueueClientBuilder b AfterDeserialize: ReadWithClientSerializer(provider, serializer, (message, s) => CloudEventRoutingHandler.Read(message, table.Routes, s)))); } + // Source-generated, so it is trim- and AOT-safe. The generator skips what configuration cannot hold (the callbacks + // and RateLimiter), as reflection binding would. + private static void BindProcessorOptions(IConfiguration configuration, QueueProcessorOptions options) => configuration.Bind(options); + private static string Describe(IQueueClientBuilder builder) => builder.Name.Length == 0 ? "The default queue client" : $"Queue client '{builder.Name}'"; // Options are named per client + handler so the same handler type can serve two queues with different settings. diff --git a/src/AzureStorage.QueueService/QueueClientFactory.cs b/src/AzureStorage.QueueService/QueueClientFactory.cs index 703965c..c83ce14 100644 --- a/src/AzureStorage.QueueService/QueueClientFactory.cs +++ b/src/AzureStorage.QueueService/QueueClientFactory.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using Azure.Storage.Queues; using AzureStorage.QueueService.Telemetry; using Microsoft.Extensions.DependencyInjection; @@ -22,8 +22,8 @@ internal sealed class QueueClientFactory : IQueueClientFactory private readonly IServiceProvider? _services; private readonly TimeProvider _time; - /// Registered only by AddInMemoryQueues; when present, every client is an in-memory queue. - /// For a large message store registered with UseLargeMessageStore(provider => ...). + // inMemory: registered only by AddInMemoryQueues; when present, every client is an in-memory queue. + // services: for a large message store registered with UseLargeMessageStore(provider => ...). public QueueClientFactory( QueueClientSettingsRegistry registry, IOptionsFactory settings, diff --git a/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs b/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs index e6eeee3..1938e77 100644 --- a/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs +++ b/src/AzureStorage.QueueService/QueueClientSettingsBuilder.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using Azure.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -51,12 +52,20 @@ public IQueueClientBuilder AddClient(string clientName, Action /// Registers a named client bound from a configuration section, for example - /// configuration.GetSection("Queues:Orders"). + /// configuration.GetSection("Queues:Orders"). Every setting that has a value in configuration can be bound, + /// including "ClientOptions": { "MessageEncoding": "Base64" }; the credential, serializer and large message + /// store are set in code. /// + /// + /// Binding uses reflection, which trimming and native AOT cannot see through; in such an application, set the + /// settings in code with instead. + /// + [RequiresUnreferencedCode(BindingMessage)] + [RequiresDynamicCode(BindingMessage)] public IQueueClientBuilder AddClient(string clientName, IConfiguration configuration) { ArgumentNullException.ThrowIfNull(configuration); - return AddClient(clientName, settings => configuration.Bind(settings)); + return AddClient(clientName, settings => Bind(configuration, settings)); } /// @@ -79,12 +88,27 @@ public IQueueClientBuilder AddDefaultClient(Action configur } /// - /// Registers the default client bound from a configuration section. + /// Registers the default client bound from a configuration section. See . /// + [RequiresUnreferencedCode(BindingMessage)] + [RequiresDynamicCode(BindingMessage)] public IQueueClientBuilder AddDefaultClient(IConfiguration configuration) { ArgumentNullException.ThrowIfNull(configuration); - return AddDefaultClient(settings => configuration.Bind(settings)); + return AddDefaultClient(settings => Bind(configuration, settings)); + } + + private const string BindingMessage = "Binds QueueClientSettings from configuration with reflection. In a trimmed or native AOT application, set the settings in code with the Action overload."; + + // Reflection binding on purpose, not the source generator this project uses for processor options: the generator + // cannot create or fill QueueClientOptions, so "ClientOptions": { "MessageEncoding": "Base64" } would be silently + // ignored. Through a delegate, because the generator replaces direct calls to Bind. + [RequiresUnreferencedCode(BindingMessage)] + [RequiresDynamicCode(BindingMessage)] + private static void Bind(IConfiguration configuration, QueueClientSettings settings) + { + Action bind = ConfigurationBinder.Bind; + bind(configuration, settings); } /// @@ -166,7 +190,7 @@ public QueueClientSettingsBuilder UseStorageAccount(Uri serviceUri) /// Runs around every message every client's processors deliver, resolved from the /// message's scope (registered as scoped unless you registered it yourself). See . /// - public QueueClientSettingsBuilder UseMiddleware() + public QueueClientSettingsBuilder UseMiddleware<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TMiddleware>() where TMiddleware : class, IQueueMessageMiddleware { Services.TryAddScoped(); diff --git a/src/AzureStorage.QueueService/QueueMessageMiddleware.cs b/src/AzureStorage.QueueService/QueueMessageMiddleware.cs index 44e4a62..9ff56bc 100644 --- a/src/AzureStorage.QueueService/QueueMessageMiddleware.cs +++ b/src/AzureStorage.QueueService/QueueMessageMiddleware.cs @@ -9,7 +9,7 @@ namespace AzureStorage.QueueService; /// Resolved from the message's own dependency-injection scope, so it can take scoped services. /// /// -/// Call to go on to the handler, or return without calling it to stop there: the processor then +/// Call next to go on to the handler, or return without calling it to stop there: the processor then /// treats the message as handled (with on, it is completed). Throw and /// the message counts as failed, exactly as if the handler had thrown. A middleware can settle the message itself /// through . Register it with UseMiddleware<T>() on one client's diff --git a/src/AzureStorage.QueueService/QueueReceiverExtensions.cs b/src/AzureStorage.QueueService/QueueReceiverExtensions.cs new file mode 100644 index 0000000..77fa41f --- /dev/null +++ b/src/AzureStorage.QueueService/QueueReceiverExtensions.cs @@ -0,0 +1,110 @@ +using System.Runtime.CompilerServices; + +namespace AzureStorage.QueueService; + +/// How polls. +public sealed class ReadAllOptions +{ + /// + /// How many messages each receive asks for (1 to 32). Every message in a batch is hidden from the moment it is + /// received, so the visibility timeout has to cover handling the whole batch; keep it at 1 unless handling is quick. + /// + public int MaxMessages { get; set; } = 1; + + /// How long each received message stays hidden; null for the client's , then the service's 30 seconds. + public TimeSpan? VisibilityTimeout { get; set; } + + /// The wait after an empty receive; doubles up to and resets when a message arrives. + public TimeSpan EmptyQueueBackoffMin { get; set; } = TimeSpan.FromSeconds(1); + + /// The longest wait between receives of an empty queue. + public TimeSpan EmptyQueueBackoffMax { get; set; } = TimeSpan.FromSeconds(30); + + /// The longest wait after a failed receive; the wait doubles from . + public TimeSpan ErrorBackoffMax { get; set; } = TimeSpan.FromMinutes(1); + + /// + /// Called with each failed receive before the loop backs off and tries again: log it here. Failed receives never end + /// the loop; only cancellation does. + /// + public Action? OnReceiveError { get; set; } + + /// The clock the waits use; a fake one in tests. + public TimeProvider TimeProvider { get; set; } = TimeProvider.System; +} + +/// A polling loop over , for code that owns its loop rather than using a processor. +public static class QueueReceiverExtensions +{ + /// + /// Receives messages for as long as allows, waiting with a growing backoff while + /// the queue is empty and after a failed receive, and hands them over one at a time. Settling each message — + /// and the rest — is the caller's job, as is anything a processor would do + /// for you (lease renewal, retries, dead-lettering). Ends when the token is cancelled, without throwing. + /// + /// + /// await foreach (var message in receiver.ReadAllAsync<Order>(new ReadAllOptions { OnReceiveError = e => logger.LogError(e, "Receive failed") }, stoppingToken)) + /// { + /// if (message.Error is not null) { await message.DeadLetterAsync(message.Error.Message); continue; } + /// await Fulfil(message.Body!); + /// await message.CompleteAsync(); + /// } + /// + public static async IAsyncEnumerable> ReadAllAsync(this IQueueReceiver receiver, ReadAllOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(receiver); + options ??= new ReadAllOptions(); + if (options.MaxMessages is < 1 or > StorageQueue.ServiceMaxMessages) throw new ArgumentOutOfRangeException(nameof(options), options.MaxMessages, "MaxMessages must be 1 to 32."); + if (options.EmptyQueueBackoffMin <= TimeSpan.Zero || options.EmptyQueueBackoffMax < options.EmptyQueueBackoffMin || options.ErrorBackoffMax < options.EmptyQueueBackoffMin) + throw new ArgumentOutOfRangeException(nameof(options), "EmptyQueueBackoffMin must be positive and no larger than EmptyQueueBackoffMax or ErrorBackoffMax."); + + var emptyBackoff = options.EmptyQueueBackoffMin; + var errorBackoff = options.EmptyQueueBackoffMin; + + while (!cancellationToken.IsCancellationRequested) + { + IReadOnlyList> batch; + try + { + batch = await receiver.ReceiveAsync(options.MaxMessages, options.VisibilityTimeout, cancellationToken); + errorBackoff = options.EmptyQueueBackoffMin; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + yield break; + } + catch (Exception e) + { + options.OnReceiveError?.Invoke(e); + if (!await DelayAsync(errorBackoff, options.TimeProvider, cancellationToken)) yield break; + errorBackoff = Min(errorBackoff * 2, options.ErrorBackoffMax); + continue; + } + + if (batch.Count == 0) + { + if (!await DelayAsync(emptyBackoff, options.TimeProvider, cancellationToken)) yield break; + emptyBackoff = Min(emptyBackoff * 2, options.EmptyQueueBackoffMax); + continue; + } + + emptyBackoff = options.EmptyQueueBackoffMin; + foreach (var message in batch) yield return message; + } + } + + private static async Task DelayAsync(TimeSpan delay, TimeProvider time, CancellationToken cancellationToken) + { + try + { + await Task.Delay(delay, time, cancellationToken); + return true; + } + catch (OperationCanceledException) + { + return false; + } + } + + private static TimeSpan Min(TimeSpan a, TimeSpan b) => a < b ? a : b; +} diff --git a/src/AzureStorage.QueueService/QueueTransport.cs b/src/AzureStorage.QueueService/QueueTransport.cs index 2bdc198..385823f 100644 --- a/src/AzureStorage.QueueService/QueueTransport.cs +++ b/src/AzureStorage.QueueService/QueueTransport.cs @@ -21,10 +21,10 @@ internal interface IQueueTransport ValueTask CreateIfNotExistsAsync(IDictionary? metadata, CancellationToken cancellationToken); - /// In the service's terms: -1 second for never. + // timeToLive is in the service's terms: -1 second for never. ValueTask SendAsync(BinaryData body, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken); - /// Null for the service's default of 30 seconds. + // visibilityTimeout: null for the service's default of 30 seconds. ValueTask> ReceiveAsync(int maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken); ValueTask> PeekAsync(int maxMessages, CancellationToken cancellationToken); @@ -148,7 +148,7 @@ internal sealed class InMemoryQueueTransport : IQueueTransport private readonly InMemoryQueue _queue; - /// What the client's options say, so the size limit is checked as it would be in production. + // base64Encoded: what the client's options say, so the size limit is checked as it would be in production. public InMemoryQueueTransport(InMemoryQueue queue, bool base64Encoded) { _queue = queue; diff --git a/src/AzureStorage.QueueService/Routing.cs b/src/AzureStorage.QueueService/Routing.cs index 4321ad0..a527992 100644 --- a/src/AzureStorage.QueueService/Routing.cs +++ b/src/AzureStorage.QueueService/Routing.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using Azure.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -24,7 +25,7 @@ internal QueueMessageRoutes(IServiceCollection services) /// Sends messages whose type header is (by default 's /// name, which is what sends) to . /// - public QueueMessageRoutes Route(string? messageType = null) + public QueueMessageRoutes Route(string? messageType = null) where THandler : class, IQueueMessageHandler { var key = messageType ?? typeof(TMessage).Name; @@ -55,7 +56,7 @@ internal CloudEventRoutes(IServiceCollection services) internal Dictionary Routes { get; } = new(StringComparer.Ordinal); /// Sends events whose type is to , with their data read as . - public CloudEventRoutes Route(string eventType) + public CloudEventRoutes Route(string eventType) where THandler : class, ICloudEventHandler { ArgumentException.ThrowIfNullOrEmpty(eventType); diff --git a/src/AzureStorage.QueueService/SdkQueueClientFactory.cs b/src/AzureStorage.QueueService/SdkQueueClientFactory.cs index e29a156..7e2c345 100644 --- a/src/AzureStorage.QueueService/SdkQueueClientFactory.cs +++ b/src/AzureStorage.QueueService/SdkQueueClientFactory.cs @@ -15,7 +15,7 @@ public SdkQueueClientFactory(ILogger? logger = null) { } - /// For tests: replaces the options copy, so the fallback can be exercised. + // copyOptions: for tests, replaces the options copy, so the fallback can be exercised. internal SdkQueueClientFactory(ILogger? logger, Func copyOptions) { _logger = logger ?? NullLogger.Instance; diff --git a/src/AzureStorage.QueueService/Telemetry/QueueServiceTelemetrySettings.cs b/src/AzureStorage.QueueService/Telemetry/QueueServiceTelemetrySettings.cs index 1b9a14c..7a8c6b0 100644 --- a/src/AzureStorage.QueueService/Telemetry/QueueServiceTelemetrySettings.cs +++ b/src/AzureStorage.QueueService/Telemetry/QueueServiceTelemetrySettings.cs @@ -1,7 +1,11 @@ -using System.Diagnostics; +using System.Diagnostics; namespace AzureStorage.QueueService.Telemetry; +/// +/// Spans that are off by default because they are noisy: one per individual send and receive. The processor's +/// Queue.Process span is always on. Set with ConfigureQueueServiceTelemetry. +/// public class QueueServiceTelemetrySettings { /// diff --git a/src/AzureStorage.QueueService/Telemetry/TracerProviderBuilderExtensions.cs b/src/AzureStorage.QueueService/Telemetry/TracerProviderBuilderExtensions.cs index 789b04b..ba280b1 100644 --- a/src/AzureStorage.QueueService/Telemetry/TracerProviderBuilderExtensions.cs +++ b/src/AzureStorage.QueueService/Telemetry/TracerProviderBuilderExtensions.cs @@ -1,9 +1,11 @@ -using OpenTelemetry.Trace; +using OpenTelemetry.Trace; namespace AzureStorage.QueueService.Telemetry; +/// Subscribes an OpenTelemetry tracer provider to the library's spans. public static class TracerProviderBuilderExtensions { + /// The library's spans: Queue.Process per delivered message, and Queue.Send / Queue.Read when turned on. public static TracerProviderBuilder AddAzureStorageQueueTracing(this TracerProviderBuilder builder) { builder.AddSource(QueueServiceDiagnostics.ServiceName); From e5ebd22f5a18ac74deb49d5c09cdb40597acce7d Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:38:39 -0600 Subject: [PATCH 4/8] Remove accidentally added worktree links --- .claude/worktrees/agent-abe49847f2bdc9f48 | 1 - .claude/worktrees/agent-afb7e0b0c74b2bbe0 | 1 - 2 files changed, 2 deletions(-) delete mode 160000 .claude/worktrees/agent-abe49847f2bdc9f48 delete mode 160000 .claude/worktrees/agent-afb7e0b0c74b2bbe0 diff --git a/.claude/worktrees/agent-abe49847f2bdc9f48 b/.claude/worktrees/agent-abe49847f2bdc9f48 deleted file mode 160000 index a0524da..0000000 --- a/.claude/worktrees/agent-abe49847f2bdc9f48 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a0524da7e0904053ca75b4e595c9a4500dae8b07 diff --git a/.claude/worktrees/agent-afb7e0b0c74b2bbe0 b/.claude/worktrees/agent-afb7e0b0c74b2bbe0 deleted file mode 160000 index b225c3c..0000000 --- a/.claude/worktrees/agent-afb7e0b0c74b2bbe0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b225c3cd600ff29a8ea17b94f2c2bac877a91660 From 324fc1357b952da9e4d6aa33babf2c9cd9faaff4 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:41:12 -0600 Subject: [PATCH 5/8] Add AzureStorage.QueueService.Blobs: large message bodies in Blob storage New package implementing ILargeMessageStore over a BlobContainerClient (claim-check for messages over 64 KiB), with UseBlobStorageForLargeMessages builder extensions (container instance, factory, or container name derived from the queue client's own account and credentials). Download and delete only accept blobs inside the store's own container; references never carry a SAS token. Tests: reference refusal and endpoint derivation (no IO), plus end-to-end completion and dead-letter runs against Azurite blob storage. Co-Authored-By: Claude Opus 5.5 --- .../AzureStorage.QueueService.Blobs.csproj | 53 +++++ .../BlobEndpoints.cs | 115 ++++++++++ .../BlobLargeMessageStore.cs | 205 ++++++++++++++++++ ...BlobStorageQueueClientBuilderExtensions.cs | 131 +++++++++++ src/AzureStorage.QueueService.Blobs/README.md | 44 ++++ ...ureStorage.QueueService.Blobs.Tests.csproj | 35 +++ .../BlobLargeMessageEndToEndTests.cs | 104 +++++++++ .../BlobLargeMessageStoreTests.cs | 68 ++++++ 8 files changed, 755 insertions(+) create mode 100644 src/AzureStorage.QueueService.Blobs/AzureStorage.QueueService.Blobs.csproj create mode 100644 src/AzureStorage.QueueService.Blobs/BlobEndpoints.cs create mode 100644 src/AzureStorage.QueueService.Blobs/BlobLargeMessageStore.cs create mode 100644 src/AzureStorage.QueueService.Blobs/BlobStorageQueueClientBuilderExtensions.cs create mode 100644 src/AzureStorage.QueueService.Blobs/README.md create mode 100644 tests/AzureStorage.QueueService.Blobs.Tests/AzureStorage.QueueService.Blobs.Tests.csproj create mode 100644 tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageEndToEndTests.cs create mode 100644 tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageStoreTests.cs diff --git a/src/AzureStorage.QueueService.Blobs/AzureStorage.QueueService.Blobs.csproj b/src/AzureStorage.QueueService.Blobs/AzureStorage.QueueService.Blobs.csproj new file mode 100644 index 0000000..ab7e419 --- /dev/null +++ b/src/AzureStorage.QueueService.Blobs/AzureStorage.QueueService.Blobs.csproj @@ -0,0 +1,53 @@ + + + + net10.0 + enable + enable + $(MSBuildProjectName) + $(MSBuildProjectName.Replace(" ", "_")) + True + true + true + Unifinity Software Corporation + True + MIT + Sends Azure Storage queue messages over the 64 KiB limit by keeping their bodies in Azure Blob storage (the claim-check pattern), for AzureStorage.QueueService. One line per client: UseBlobStorageForLargeMessages(). + Unifinity Software Corporation + Azure, Azure Storage, Azure Storage Queue, Storage Queue, Queue Service, Blob Storage, Claim Check, Large Messages, Jason Shave, Unifinity Software Corporation + https://github.com/jasonshave/AzureStorage.QueueService + logo.png + README.md + git + https://github.com/jasonshave/AzureStorage.QueueService + Unifinity Software Corporation + + $(VersionPrefix) + + + + + + + + + + + + + <_Parameter1>$(AssemblyName).Tests + + + + + + True + \ + + + True + \ + + + + diff --git a/src/AzureStorage.QueueService.Blobs/BlobEndpoints.cs b/src/AzureStorage.QueueService.Blobs/BlobEndpoints.cs new file mode 100644 index 0000000..6f1e062 --- /dev/null +++ b/src/AzureStorage.QueueService.Blobs/BlobEndpoints.cs @@ -0,0 +1,115 @@ +using Azure.Storage.Blobs; +using Azure.Storage.Queues; + +namespace AzureStorage.QueueService.Blobs; + +/// Works out a queue client's Blob storage from its own connection settings. +internal static class BlobEndpoints +{ + // Azurite's default ports: Blob on 10000, Queue on 10001. + private const int EmulatorQueuePort = 10001; + private const int EmulatorBlobPort = 10000; + + /// + /// A Blob service client for the storage account the queue client connects to, with the same credentials: the + /// connection string, or the endpoint's account with the client's . + /// + /// The queue client's finished settings. + /// Names the client in errors, for example "Queue client 'orders'". + /// The settings do not say enough to find the Blob endpoint. + public static BlobServiceClient ServiceClientFor(QueueClientSettings settings, string clientDescription) + { + const string Instead = "Pass a BlobContainerClient to UseBlobStorageForLargeMessages instead."; + var options = BlobOptions(settings.ClientOptions); + + if (!string.IsNullOrEmpty(settings.ConnectionString)) + { + try + { + return new BlobServiceClient(settings.ConnectionString, options); + } + catch (Exception e) when (e is FormatException or ArgumentException or InvalidOperationException) + { + throw new InvalidOperationException($"{clientDescription} keeps large messages in Blob storage, but its connection string cannot be used for Blob storage: {e.Message} {Instead}", e); + } + } + + if (settings.EndpointUri is { } endpoint) + { + var blobEndpoint = ServiceUriFor(endpoint) + ?? throw new InvalidOperationException($"{clientDescription} keeps large messages in Blob storage, but the Blob endpoint cannot be worked out from its queue endpoint '{endpoint}'. That works for '.queue.' hosts and the storage emulator. {Instead}"); + + if (settings.TokenCredential is not { } credential) + throw new InvalidOperationException($"{clientDescription} keeps large messages in Blob storage, but has an EndpointUri and no TokenCredential to connect with. Set one, or call UseTokenCredential."); + + return new BlobServiceClient(blobEndpoint, credential, options); + } + + throw new InvalidOperationException($"{clientDescription} keeps large messages in Blob storage, but has no ConnectionString or EndpointUri to find the storage account from. {Instead}"); + } + + /// + /// The account's Blob endpoint, from a queue or queue-service URI: https://acct.queue.core.windows.net/orders + /// gives https://acct.blob.core.windows.net/, and the emulator's + /// http://127.0.0.1:10001/devstoreaccount1/orders gives http://127.0.0.1:10000/devstoreaccount1/. Null + /// when the URI is neither shape, such as a custom domain. + /// + public static Uri? ServiceUriFor(Uri queueEndpoint) + { + // Host style: the service is a label after the account, in every Azure cloud ("acct.queue.core.windows.net", + // "acct.queue.core.chinacloudapi.cn", "acct-secondary.queue..."). + var labels = queueEndpoint.Host.Split('.'); + if (labels.Length > 2) + { + var service = Array.FindIndex(labels, 1, label => label.Equals("queue", StringComparison.OrdinalIgnoreCase)); + if (service > 0) + { + labels[service] = "blob"; + return new UriBuilder(queueEndpoint.Scheme, string.Join('.', labels), queueEndpoint.Port, "/").Uri; + } + } + + // Path style (the emulator): the account is the first path segment and the service is told apart by port. + if (queueEndpoint.Port == EmulatorQueuePort && new QueueUriBuilder(queueEndpoint).AccountName is { Length: > 0 } account) + return new UriBuilder(queueEndpoint.Scheme, queueEndpoint.Host, EmulatorBlobPort, $"/{account}/").Uri; + + return null; + } + + /// The queue client's retry policy and transport, applied to the Blob client. + private static BlobClientOptions BlobOptions(QueueClientOptions? queueOptions) + { + var options = new BlobClientOptions(); + if (queueOptions is null) return options; + + options.Retry.Mode = queueOptions.Retry.Mode; + options.Retry.MaxRetries = queueOptions.Retry.MaxRetries; + options.Retry.Delay = queueOptions.Retry.Delay; + options.Retry.MaxDelay = queueOptions.Retry.MaxDelay; + options.Retry.NetworkTimeout = queueOptions.Retry.NetworkTimeout; + options.Transport = queueOptions.Transport; + return options; + } +} + +/// +/// Azure's rules for a container name: 3 to 63 characters, lowercase letters, digits and single hyphens, starting and +/// ending with a letter or digit. +/// +internal static class ContainerNameRules +{ + /// Returns a description of the problem, or null when the name is valid. + public static string? Check(string name) + { + if (name.Length is < 3 or > 63) return $"'{name}' is {name.Length} characters; a container name must be 3 to 63."; + if (!char.IsAsciiLetterLower(name[0]) && !char.IsAsciiDigit(name[0])) return $"'{name}' must start with a lowercase letter or digit."; + if (!char.IsAsciiLetterLower(name[^1]) && !char.IsAsciiDigit(name[^1])) return $"'{name}' must end with a lowercase letter or digit."; + if (name.Contains("--", StringComparison.Ordinal)) return $"'{name}' must not contain consecutive hyphens."; + foreach (var c in name) + { + if (!char.IsAsciiLetterLower(c) && !char.IsAsciiDigit(c) && c != '-') return $"'{name}' may only contain lowercase letters, digits and hyphens."; + } + + return null; + } +} diff --git a/src/AzureStorage.QueueService.Blobs/BlobLargeMessageStore.cs b/src/AzureStorage.QueueService.Blobs/BlobLargeMessageStore.cs new file mode 100644 index 0000000..2371cc5 --- /dev/null +++ b/src/AzureStorage.QueueService.Blobs/BlobLargeMessageStore.cs @@ -0,0 +1,205 @@ +using System.Globalization; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; + +namespace AzureStorage.QueueService.Blobs; + +/// +/// Keeps queue message bodies that are too big for the queue (over 64 KiB) as blobs in one Blob storage container, and +/// puts only the blob's URI on the queue. Attach it to a client with UseBlobStorageForLargeMessages; you rarely +/// need to create one yourself. +/// +/// +/// +/// Each body is stored as {queue}/{yyyy}/{MM}/{dd}/{random id} in the container, dated in UTC. The library deletes +/// the blob once the message has been processed and completed, and keeps it when the message is dead-lettered, so a +/// replay from the poison queue still has its body. +/// +/// +/// Every client that reads the queue needs the same store (the same container). A receiver without one cannot +/// read these messages and dead-letters them. +/// +/// +/// Some blobs are left behind. If a send fails after the body was uploaded, the library deletes the blob, but only +/// on a best-effort basis. A message that expires on the queue (its time to live runs out) or is removed by clearing the +/// queue is never processed, so its blob is never deleted. Add an Azure Storage lifecycle management rule that deletes +/// blobs in this container older than the queue's message time to live plus a margin (for example, 7 days + 2 days for +/// the default time to live). If you dead-letter messages and replay them later, make the rule long enough to cover that +/// too, because a dead-lettered message still needs its blob. +/// +/// +/// Messages are untrusted input. Anyone who can write to the queue can put any reference in a message, so this +/// store only reads and deletes blobs inside its own container and refuses every other reference. A refused message +/// ends up dead-lettered. +/// +/// +/// The identity used by the container client needs the Storage Blob Data Contributor role on the container (or +/// the account), to create the container, write, read and delete blobs. +/// +/// +public sealed class BlobLargeMessageStore : ILargeMessageStore +{ + private const int MaxBlobNameLength = 1024; + private const int MaxReferenceInMessage = 256; + + private readonly bool _createContainer; + private readonly TimeProvider _time; + private readonly Uri _containerUri; + private readonly string _containerPath; + private readonly Lock _gate = new(); + private Task? _containerReady; + + /// + /// Creates a store that keeps bodies in . + /// + /// + /// The container to keep bodies in, with the credential to reach it. If it was created from a URI with a SAS token, + /// the token is never written to the queue: references are the blob's URI without any query string. + /// + /// + /// When true (the default), the container is created the first time a body is uploaded, if it does not already + /// exist. This happens once per store. Pass false when the container is created ahead of time and the identity is + /// not allowed to create containers. + /// + /// The clock that dates each blob's name. Defaults to the system clock. + public BlobLargeMessageStore(BlobContainerClient container, bool createContainerIfNotExists = true, TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(container); + + Container = container; + _createContainer = createContainerIfNotExists; + _time = timeProvider ?? TimeProvider.System; + _containerUri = new Uri(container.Uri.GetLeftPart(UriPartial.Path)); + _containerPath = _containerUri.AbsolutePath.TrimEnd('/') + "/"; + } + + /// The container bodies are kept in. + public BlobContainerClient Container { get; } + + /// + /// Uploads as a new blob named {queueName}/{yyyy}/{MM}/{dd}/{random id} and returns + /// the blob's URI (without any SAS token), which the library puts on the queue in place of the body. Creates the + /// container first if this store was told to and has not done so yet. + /// + public async ValueTask UploadAsync(string queueName, BinaryData body, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(queueName); + ArgumentNullException.ThrowIfNull(body); + + await EnsureContainerAsync(cancellationToken); + + var now = _time.GetUtcNow().UtcDateTime; + var name = string.Create(CultureInfo.InvariantCulture, $"{queueName}/{now:yyyy}/{now:MM}/{now:dd}/{Guid.NewGuid():N}"); + var blob = Container.GetBlobClient(name); + await blob.UploadAsync(body, overwrite: false, cancellationToken); + return blob.Uri.GetLeftPart(UriPartial.Path); + } + + /// + /// Downloads the body a message refers to. Throws without making any request when + /// is not a blob in this store's container, and the service's exception when the blob is + /// gone; the library dead-letters the message in both cases. + /// + public async ValueTask DownloadAsync(string reference, CancellationToken cancellationToken) + { + var blob = BlobFor(reference); + var result = await blob.DownloadContentAsync(cancellationToken); + return result.Value.Content; + } + + /// + /// Deletes the blob a message refers to, with any snapshots. A blob that is already gone is not an error. Throws + /// without making any request when is not a blob in + /// this store's container. + /// + public async ValueTask DeleteAsync(string reference, CancellationToken cancellationToken) + { + var blob = BlobFor(reference); + await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, conditions: null, cancellationToken); + } + + /// + /// The blob names, reached through this store's own container client, or an + /// when it is anything else: another account, endpoint or container, a query string + /// (such as a SAS token), or a name that would not come back to the same URI. + /// + internal BlobClient BlobFor(string reference) + { + ArgumentNullException.ThrowIfNull(reference); + + if (!Uri.TryCreate(reference, UriKind.Absolute, out var uri)) + throw Refused(reference, "it is not an absolute URI"); + + if (uri.Query.Length > 0 || uri.Fragment.Length > 0 || uri.UserInfo.Length > 0) + throw Refused(reference, "it has a query string, fragment or user information, which this store never writes"); + + if (!string.Equals(uri.Scheme, _containerUri.Scheme, StringComparison.OrdinalIgnoreCase) + || !string.Equals(uri.Host, _containerUri.Host, StringComparison.OrdinalIgnoreCase) + || uri.Port != _containerUri.Port) + throw Refused(reference, $"it is not on this store's Blob endpoint ({_containerUri.GetLeftPart(UriPartial.Authority)})"); + + // AbsolutePath has dot segments already removed, so "/container/../other/x" is "/other/x" here and fails this. + var path = uri.AbsolutePath; + if (!path.StartsWith(_containerPath, StringComparison.Ordinal) || path.Length == _containerPath.Length) + throw Refused(reference, $"it is not a blob in this store's container ({_containerUri})"); + + var name = Uri.UnescapeDataString(path[_containerPath.Length..]); + if (!IsPlainBlobName(name)) + throw Refused(reference, "its blob name is not one this store writes"); + + var blob = Container.GetBlobClient(name); + + // Whatever the escaping, the request must go to exactly the blob the reference names. + var target = new Uri(blob.Uri.GetLeftPart(UriPartial.Path)); + if (!string.Equals(target.AbsolutePath, path, StringComparison.Ordinal)) + throw Refused(reference, "it does not name a blob in this store's container exactly"); + + return blob; + } + + private static bool IsPlainBlobName(string name) + { + if (name.Length is 0 or > MaxBlobNameLength) return false; + foreach (var segment in name.Split('/')) + { + if (segment.Length == 0 || segment is "." or "..") return false; + } + + foreach (var c in name) + { + if (char.IsControl(c) || c == '\\') return false; + } + + return true; + } + + private static ArgumentException Refused(string reference, string why) + { + // The reference came off the queue, so it may be anything; keep the error message a sensible size. + var shown = reference.Length > MaxReferenceInMessage ? reference[..MaxReferenceInMessage] + "..." : reference; + return new ArgumentException($"Refused to use large message reference '{shown}': {why}.", nameof(reference)); + } + + private Task EnsureContainerAsync(CancellationToken cancellationToken) + { + if (!_createContainer) return Task.CompletedTask; + + var ready = Volatile.Read(ref _containerReady); + if (ready is null || ready.IsFaulted || ready.IsCanceled) + { + lock (_gate) + { + ready = _containerReady; + // A failed attempt (no permission yet, a network blip) is tried again by the next upload. + if (ready is null || ready.IsFaulted || ready.IsCanceled) + { + // Shared by every upload waiting on it, so it is not tied to any one caller's cancellation. + ready = Task.Run(() => Container.CreateIfNotExistsAsync(PublicAccessType.None, metadata: null, cancellationToken: CancellationToken.None)); + Volatile.Write(ref _containerReady, ready); + } + } + } + + return ready.IsCompletedSuccessfully ? Task.CompletedTask : ready.WaitAsync(cancellationToken); + } +} diff --git a/src/AzureStorage.QueueService.Blobs/BlobStorageQueueClientBuilderExtensions.cs b/src/AzureStorage.QueueService.Blobs/BlobStorageQueueClientBuilderExtensions.cs new file mode 100644 index 0000000..9fc2778 --- /dev/null +++ b/src/AzureStorage.QueueService.Blobs/BlobStorageQueueClientBuilderExtensions.cs @@ -0,0 +1,131 @@ +using Azure.Storage.Blobs; +using AzureStorage.QueueService.Blobs; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace AzureStorage.QueueService; + +/// +/// Lets one queue client send messages over the queue's 64 KiB limit by keeping their bodies in Azure Blob storage. +/// +/// +/// +/// A message too big for the queue has its body uploaded to a blob, and the queue message carries only a small envelope +/// with the blob's URI (the claim-check pattern). When a processor or receiver gets that message, it downloads the body +/// before the handler sees it, so handlers read large and small messages the same way. The blob is deleted once the +/// message is completed, and kept when the message is dead-lettered so a replay still has its body. Smaller messages are +/// sent as usual unless you set a threshold. +/// +/// +/// Every client that reads the queue needs the same store: call one of these methods, pointing at the same +/// container, on the sending client and on every client that receives from the queue or its poison queue. Large messages +/// travel in the library's envelope, which only version 3.1 or later of this library reads. +/// +/// +/// Some blobs are left behind. A send that fails after the upload deletes its blob on a best-effort basis, and a +/// message that expires or is cleared from the queue never gets processed, so its blob stays. Add an Azure Storage +/// lifecycle management rule that deletes blobs in the container older than the queue's message time to live plus a +/// margin (and long enough to replay dead-lettered messages). +/// +/// +/// The identity needs the Storage Blob Data Contributor role on the container or the account. +/// +/// +public static class BlobStorageQueueClientBuilderExtensions +{ + /// The container used when none is named: queue-large-messages. + public const string DefaultContainerName = "queue-large-messages"; + + /// + /// Keeps bodies too big for a queue message as blobs in , which is created on the first + /// large send if it does not exist. + /// + /// The client to attach the store to. + /// The container to keep bodies in, with the credential to reach it. + /// + /// Also send bodies of at least this many bytes to Blob storage even when they would fit on the queue. Null (the + /// default) sends only what would otherwise be over the 64 KiB limit. + /// + /// The same builder, for chaining. + public static IQueueClientBuilder UseBlobStorageForLargeMessages(this IQueueClientBuilder builder, BlobContainerClient container, int? thresholdBytes = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(container); + return builder.UseBlobStorageForLargeMessages(_ => container, thresholdBytes); + } + + /// + /// Keeps bodies too big for a queue message as blobs in the container returns. It + /// is called once, from the application's services, the first time the client is used, so the container client can + /// come from dependency injection (for example one registered with Microsoft.Extensions.Azure). The container + /// is created on the first large send if it does not exist. + /// + /// The client to attach the store to. + /// Returns the container to keep bodies in, with the credential to reach it. + /// + /// Also send bodies of at least this many bytes to Blob storage even when they would fit on the queue. Null (the + /// default) sends only what would otherwise be over the 64 KiB limit. + /// + /// The same builder, for chaining. + public static IQueueClientBuilder UseBlobStorageForLargeMessages(this IQueueClientBuilder builder, Func createContainer, int? thresholdBytes = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(createContainer); + if (thresholdBytes is { } threshold) ArgumentOutOfRangeException.ThrowIfNegative(threshold, nameof(thresholdBytes)); + + var clientName = builder.Name; + return builder.UseLargeMessageStore( + provider => + { + var container = createContainer(provider) + ?? throw new InvalidOperationException($"{Describe(clientName)} keeps large messages in Blob storage, but the delegate passed to UseBlobStorageForLargeMessages returned no container."); + return new BlobLargeMessageStore(container, createContainerIfNotExists: true, provider.GetService()); + }, + thresholdBytes); + } + + /// + /// Keeps bodies too big for a queue message as blobs in the container , in the same + /// storage account as the queue and reached with the same credentials. The container is created on the first large + /// send if it does not exist. + /// + /// + /// The Blob endpoint is worked out from the client's settings the first time the client is used: a connection + /// string is used as it is, and an such as + /// https://myaccount.queue.core.windows.net/orders becomes https://myaccount.blob.core.windows.net/, + /// used with the client's (or the one from UseTokenCredential). + /// The storage emulator's http://127.0.0.1:10001/devstoreaccount1 becomes http://127.0.0.1:10000/devstoreaccount1. + /// When it cannot be worked out (a custom domain, or in-memory queues with no connection details), the first use of + /// the client throws ; pass a instead. The + /// client's retry settings and transport from carry over. + /// + /// The client to attach the store to. + /// + /// The container to keep bodies in; defaults to . Azure's rules apply: 3 to 63 + /// lowercase letters, digits and single hyphens, starting and ending with a letter or digit. + /// + /// + /// Also send bodies of at least this many bytes to Blob storage even when they would fit on the queue. Null (the + /// default) sends only what would otherwise be over the 64 KiB limit. + /// + /// The same builder, for chaining. + /// is not a valid container name. + public static IQueueClientBuilder UseBlobStorageForLargeMessages(this IQueueClientBuilder builder, string containerName = DefaultContainerName, int? thresholdBytes = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(containerName); + if (ContainerNameRules.Check(containerName) is { } problem) throw new ArgumentException($"Invalid container name: {problem}", nameof(containerName)); + + var clientName = builder.Name; + return builder.UseBlobStorageForLargeMessages( + provider => + { + var settings = provider.GetRequiredService>().Get(clientName); + return BlobEndpoints.ServiceClientFor(settings, Describe(clientName)).GetBlobContainerClient(containerName); + }, + thresholdBytes); + } + + private static string Describe(string clientName) => + clientName == Options.DefaultName ? "The default queue client" : $"Queue client '{clientName}'"; +} diff --git a/src/AzureStorage.QueueService.Blobs/README.md b/src/AzureStorage.QueueService.Blobs/README.md new file mode 100644 index 0000000..2392de6 --- /dev/null +++ b/src/AzureStorage.QueueService.Blobs/README.md @@ -0,0 +1,44 @@ +# AzureStorage.QueueService.Blobs + +Send Azure Storage queue messages bigger than the queue's 64 KiB limit. This package plugs Azure Blob storage into [`AzureStorage.QueueService`](https://www.nuget.org/packages/AzureStorage.QueueService): a body too big for the queue is uploaded to a blob, and the queue message carries only the blob's address (the claim-check pattern). Processors and receivers download the body before your handler sees it, so handlers don't change. The blob is deleted once the message is completed, and kept when the message is dead-lettered so a replay still has its body. + +## Install + +Add the NuGet package [`AzureStorage.QueueService.Blobs`](https://www.nuget.org/packages/AzureStorage.QueueService.Blobs). It needs `AzureStorage.QueueService` 3.1 or later. + +## Register + +Attach it to each client. **Every client that reads the queue (and its poison queue) needs the same store**, or it can't read those messages and dead-letters them. + +```csharp +builder.Services.AddAzureStorageQueueClient(queues => +{ + // 1. Same storage account and credentials as the queue, container "queue-large-messages" + queues.AddClient("orders", builder.Configuration.GetSection("Queues:Orders")) + .UseBlobStorageForLargeMessages(); + + // 2. A container you build yourself + queues.AddClient("reports", builder.Configuration.GetSection("Queues:Reports")) + .UseBlobStorageForLargeMessages(new BlobContainerClient(new Uri("https://myaccount.blob.core.windows.net/reports-bodies"), new DefaultAzureCredential())); + + // 3. A container built from your services the first time the client is used + queues.AddClient("invoices", builder.Configuration.GetSection("Queues:Invoices")) + .UseBlobStorageForLargeMessages(sp => sp.GetRequiredService().GetBlobContainerClient("invoice-bodies")); +}); +``` + +The first form works out the Blob endpoint from the queue client's own settings: a connection string is used as is, `https://myaccount.queue.core.windows.net/...` becomes `https://myaccount.blob.core.windows.net/`, and the storage emulator works too. For a custom domain, use the second or third form. Every form creates the container the first time a large message is sent, and takes an optional `thresholdBytes` to send smaller bodies to Blob storage as well. + +Large messages travel in the library's envelope, which only 3.1 and later read: upgrade every consumer of the queue before turning this on. + +## Clean up blobs that are left behind + +Most blobs are deleted automatically, but some are not: a message that expires on the queue, or is removed by clearing the queue, is never processed, so its blob stays; and a send that fails after the upload deletes its blob only on a best-effort basis. Add an Azure Storage [lifecycle management rule](https://learn.microsoft.com/azure/storage/blobs/lifecycle-management-overview) that deletes blobs in the container older than the queue's message time to live plus a margin, for example 9 days for the default 7-day time to live. If you replay dead-lettered messages later, make it long enough to cover that too. + +## Permissions + +The identity the client connects with needs **Storage Blob Data Contributor** on the container (or the storage account) to create the container and to write, read and delete blobs. A connection string with the account key needs nothing more. If the container is created ahead of time, create the store yourself with `new BlobLargeMessageStore(container, createContainerIfNotExists: false)` and attach it with `UseLargeMessageStore(store)`. + +## Security + +A queue message is untrusted input: anyone who can write to the queue can put any address in it. The store only reads and deletes blobs inside its own container, refuses every other address without making a request, and the message is dead-lettered. A SAS token on the container client is never written to the queue. diff --git a/tests/AzureStorage.QueueService.Blobs.Tests/AzureStorage.QueueService.Blobs.Tests.csproj b/tests/AzureStorage.QueueService.Blobs.Tests/AzureStorage.QueueService.Blobs.Tests.csproj new file mode 100644 index 0000000..c8eb1c4 --- /dev/null +++ b/tests/AzureStorage.QueueService.Blobs.Tests/AzureStorage.QueueService.Blobs.Tests.csproj @@ -0,0 +1,35 @@ + + + + net10.0 + enable + enable + + false + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + diff --git a/tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageEndToEndTests.cs b/tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageEndToEndTests.cs new file mode 100644 index 0000000..97c879b --- /dev/null +++ b/tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageEndToEndTests.cs @@ -0,0 +1,104 @@ +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Time.Testing; + +namespace AzureStorage.QueueService.Blobs.Tests; + +/// One Azurite container for the whole class, deleted afterwards. +public sealed class AzuriteContainer : IAsyncLifetime +{ + public const string ConnectionString = "UseDevelopmentStorage=true"; + + public BlobContainerClient Container { get; } = new(ConnectionString, $"azqs-blobs-{Guid.NewGuid():N}"); + + public Task InitializeAsync() => Task.CompletedTask; // the store creates it, which is part of what is tested + + public Task DisposeAsync() => Container.DeleteIfExistsAsync(); + + public async Task> BlobNamesAsync(string prefix) + { + var names = new List(); + if (!await Container.ExistsAsync()) return names; + await foreach (var blob in Container.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix, CancellationToken.None)) names.Add(blob.Name); + return names; + } +} + +/// +/// The real path a large message takes: in-memory queues, the real processor and a real Blob container in Azurite +/// (127.0.0.1:10000). Each test uses its own queue name, so its blobs have their own prefix in the shared container. The +/// clock is fake and never moves: the message is already on the queue when the processor's first poll runs. +/// +public class BlobLargeMessageEndToEndTests(AzuriteContainer azurite) : IClassFixture +{ + private static readonly TimeSpan SafetyNet = TimeSpan.FromSeconds(10); + + public sealed record Report(string Id, string Text); + + private static Report Large(string id) => new(id, string.Concat(Enumerable.Range(0, 10_000).Select(i => $"{i:D9}|"))); + + private static IHost BuildHost(FakeTimeProvider time, Action configure) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings()); + builder.Services.AddSingleton(time); + builder.Services.AddAzureStorageQueueClient(configure); + builder.Services.AddInMemoryQueues(); + return builder.Build(); + } + + [Fact(DisplayName = "Blob storage: a 100 KB message reaches the handler intact, and its blob is deleted once the message is completed")] + public async Task Large_Message_Round_Trips_And_Blob_Is_Deleted() + { + var time = new FakeTimeProvider(new DateTimeOffset(2026, 10, 16, 12, 0, 0, TimeSpan.Zero)); + var handled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var host = BuildHost(time, queues => + queues.AddClient("orders", settings => settings.QueueName = "orders") + .UseBlobStorageForLargeMessages(azurite.Container) + .AddProcessor((Report report) => { handled.TrySetResult(report); })); + + var sent = Large("r-1"); + await host.Services.GetRequiredKeyedService("orders").SendAsync(sent); + var stored = await azurite.BlobNamesAsync("orders/"); + stored.Should().ContainSingle().Which.Should().StartWith("orders/2026/10/16/", "the body went to Blob storage, named by queue and date"); + + await host.StartAsync(); + var received = await handled.Task.WaitAsync(SafetyNet); + await host.StopAsync(); // drains: completion, and the blob delete after it, finish before this returns + + received.Should().Be(sent); + (await azurite.BlobNamesAsync("orders/")).Should().BeEmpty("the blob is deleted once the message is completed"); + } + + [Fact(DisplayName = "Blob storage: a dead-lettered large message keeps its blob, and the poison queue still reads the body")] + public async Task Dead_Lettered_Message_Keeps_Its_Blob() + { + var handled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + // Both clients find Blob storage from their own connection string, as an application configured this way would. + using var host = BuildHost(new FakeTimeProvider(), queues => + { + queues.AddClient("refunds", settings => { settings.ConnectionString = AzuriteContainer.ConnectionString; settings.QueueName = "refunds"; }) + .UseBlobStorageForLargeMessages(azurite.Container.Name) + .AddProcessor(async (QueueMessage message) => + { + await message.DeadLetterAsync("cannot refund"); + handled.TrySetResult(); + }); + queues.AddClient("refunds-poison", settings => { settings.ConnectionString = AzuriteContainer.ConnectionString; settings.QueueName = "refunds-poison"; }) + .UseBlobStorageForLargeMessages(azurite.Container.Name); + }); + + var sent = Large("r-2"); + await host.Services.GetRequiredKeyedService("refunds").SendAsync(sent); + await host.StartAsync(); + await handled.Task.WaitAsync(SafetyNet); + await host.StopAsync(); + + (await azurite.BlobNamesAsync("refunds/")).Should().ContainSingle("a dead-lettered message may be replayed, so its body is kept"); + var deadLettered = (await host.Services.GetRequiredKeyedService("refunds-poison").ReceiveAsync()).Should().ContainSingle().Subject; + deadLettered.Error.Should().BeNull(); + deadLettered.Body.Should().Be(sent); + } +} diff --git a/tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageStoreTests.cs b/tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageStoreTests.cs new file mode 100644 index 0000000..adbd6b1 --- /dev/null +++ b/tests/AzureStorage.QueueService.Blobs.Tests/BlobLargeMessageStoreTests.cs @@ -0,0 +1,68 @@ +using Azure.Core; +using Azure.Storage.Blobs; +using AzureStorage.QueueService.Blobs; +using FluentAssertions; + +namespace AzureStorage.QueueService.Blobs.Tests; + +/// Checks that need no storage: which references the store accepts, and where Blob storage is derived from. +public class BlobLargeMessageStoreTests +{ + private static readonly BlobContainerClient Bodies = new(new Uri("https://acct.blob.core.windows.net/bodies")); + + [Theory(DisplayName = "Store: a reference that is not a blob in the store's own container is refused before any request")] + [InlineData("not a uri")] + [InlineData("file:///c:/bodies/orders/x")] + [InlineData("https://evil.example.com/bodies/orders/x")] + [InlineData("https://other.blob.core.windows.net/bodies/orders/x")] + [InlineData("http://acct.blob.core.windows.net/bodies/orders/x")] + [InlineData("https://acct.blob.core.windows.net:8443/bodies/orders/x")] + [InlineData("https://acct.blob.core.windows.net/other/orders/x")] + [InlineData("https://acct.blob.core.windows.net/bodies-evil/orders/x")] + [InlineData("https://acct.blob.core.windows.net/bodies/../other/x")] + [InlineData("https://acct.blob.core.windows.net/bodies/%2e%2e/other/x")] + [InlineData("https://acct.blob.core.windows.net/bodies/orders/x?sv=2025-01-01&sig=abc")] + [InlineData("https://acct.blob.core.windows.net/bodies/")] + public async Task Refuses_References_Outside_Its_Container(string reference) + { + // No credential and an unreachable account: anything that got as far as a request would fail differently. + var store = new BlobLargeMessageStore(Bodies); + + await store.Invoking(s => s.DownloadAsync(reference, CancellationToken.None).AsTask()).Should().ThrowAsync(); + await store.Invoking(s => s.DeleteAsync(reference, CancellationToken.None).AsTask()).Should().ThrowAsync(); + store.BlobFor("https://ACCT.blob.core.windows.net/bodies/orders/2026/10/16/abc").Name.Should().Be("orders/2026/10/16/abc", "its own blobs are still accepted"); + } + + [Theory(DisplayName = "Endpoints: the Blob endpoint is derived from the queue endpoint for Azure hosts and the emulator, and not guessed otherwise")] + [InlineData("https://acct.queue.core.windows.net/orders", "https://acct.blob.core.windows.net/")] + [InlineData("https://acct.queue.core.windows.net/", "https://acct.blob.core.windows.net/")] + [InlineData("https://acct.queue.core.chinacloudapi.cn/orders", "https://acct.blob.core.chinacloudapi.cn/")] + [InlineData("http://127.0.0.1:10001/devstoreaccount1/orders", "http://127.0.0.1:10000/devstoreaccount1/")] + [InlineData("https://queues.contoso.com/orders", null)] + public void Derives_Blob_Endpoint(string queueEndpoint, string? expected) + { + var derived = BlobEndpoints.ServiceUriFor(new Uri(queueEndpoint))?.AbsoluteUri; + derived.Should().Be(expected); + } + + [Fact(DisplayName = "Endpoints: a connection string is used as is, and an underivable endpoint names the client")] + public void Service_Client_From_Settings() + { + BlobEndpoints.ServiceClientFor(new QueueClientSettings { ConnectionString = "UseDevelopmentStorage=true" }, "Queue client 'orders'") + .Uri.AbsoluteUri.TrimEnd('/').Should().Be("http://127.0.0.1:10000/devstoreaccount1"); + + var key = Convert.ToBase64String(new byte[64]); + BlobEndpoints.ServiceClientFor(new QueueClientSettings { ConnectionString = $"DefaultEndpointsProtocol=https;AccountName=acct;AccountKey={key};EndpointSuffix=core.windows.net" }, "Queue client 'orders'") + .Uri.Should().Be(new Uri("https://acct.blob.core.windows.net/")); + + var custom = new QueueClientSettings { EndpointUri = new Uri("https://queues.contoso.com/orders"), TokenCredential = new NoCredential() }; + FluentActions.Invoking(() => BlobEndpoints.ServiceClientFor(custom, "Queue client 'orders'")) + .Should().Throw().WithMessage("Queue client 'orders'*queues.contoso.com*BlobContainerClient*"); + } + + private sealed class NoCredential : TokenCredential + { + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) => throw new NotSupportedException(); + public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) => throw new NotSupportedException(); + } +} From afb9fba22af1f6dcad5c7951c0fe0661f6eaf5f4 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:42:07 -0600 Subject: [PATCH 6/8] WIP: 3.1 feature tests; processor status snapshot --- .../QueueProcessor.cs | 41 ++- .../CapturingLoggerProvider.cs | 37 +++ .../MessageEnvelopeTests.cs | 56 ++++ .../ProcessorFeatureTests.cs | 279 ++++++++++++++++++ .../QueueFeatureTests.cs | 183 ++++++++++++ .../QueueProcessorTests.cs | 69 ++++- 6 files changed, 649 insertions(+), 16 deletions(-) create mode 100644 tests/AzureStorage.QueueService.Tests/CapturingLoggerProvider.cs create mode 100644 tests/AzureStorage.QueueService.Tests/MessageEnvelopeTests.cs create mode 100644 tests/AzureStorage.QueueService.Tests/ProcessorFeatureTests.cs create mode 100644 tests/AzureStorage.QueueService.Tests/QueueFeatureTests.cs diff --git a/src/AzureStorage.QueueService/QueueProcessor.cs b/src/AzureStorage.QueueService/QueueProcessor.cs index b3f61a5..285c4ca 100644 --- a/src/AzureStorage.QueueService/QueueProcessor.cs +++ b/src/AzureStorage.QueueService/QueueProcessor.cs @@ -95,11 +95,11 @@ internal sealed class QueueProcessor : IHostedService, IHost private volatile QueueProcessorState _state = QueueProcessorState.NotStarted; private bool _pausedLogged; - // Receive health, for status. Written by the receive loop only. - private DateTimeOffset? _lastReceiveSucceededAt; - private DateTimeOffset? _lastReceiveFailedAt; - private int _consecutiveReceiveFailures; - private string? _lastReceiveError; + // Receive health, for status: written by the receive loop only, and replaced as a whole so a reader on another + // thread (a health probe) never sees half of an update. + private volatile ReceiveHealth _health = new(null, null, 0, null); + + private sealed record ReceiveHealth(DateTimeOffset? SucceededAt, DateTimeOffset? FailedAt, int ConsecutiveFailures, string? LastError); public QueueProcessor( string clientName, @@ -281,16 +281,19 @@ private async Task FinishedWithinAsync(Task task, TimeSpan timeout, Cancel public QueueProcessorStatus GetStatus() { var state = _state == QueueProcessorState.Running && _controller?.IsPaused(_clientName) == true ? QueueProcessorState.Paused : _state; - TimeSpan? oldest = _deliveries.IsEmpty ? null : _time.GetElapsedTime(_deliveries.Values.Min(d => d.Started)); + var deliveries = _deliveries.Values.ToList(); + TimeSpan? oldest = deliveries.Count == 0 ? null : _time.GetElapsedTime(deliveries.Min(d => d.Started)); + var health = _health; return new QueueProcessorStatus( - _clientName, QueueName, _handlerName, typeof(TMessage), state, _deliveries.Count, _options.MaxConcurrentMessages, - _lastReceiveSucceededAt, _lastReceiveFailedAt, _consecutiveReceiveFailures, _lastReceiveError, oldest); + _clientName, QueueName, _handlerName, typeof(TMessage), state, deliveries.Count, _options.MaxConcurrentMessages, + health.SucceededAt, health.FailedAt, health.ConsecutiveFailures, health.LastError, oldest); } private IEnumerable> ObserveOldestHandler() { // 0 when idle, so the series is there to alert on before anything hangs. - var oldest = _deliveries.IsEmpty ? 0 : _time.GetElapsedTime(_deliveries.Values.Min(d => d.Started)).TotalSeconds; + var started = _deliveries.Values.Select(d => d.Started).ToList(); + var oldest = started.Count == 0 ? 0 : _time.GetElapsedTime(started.Min()).TotalSeconds; yield return new Measurement(oldest, new KeyValuePair(QueueServiceDiagnostics.Names.QueueName, QueueName)); } @@ -299,6 +302,18 @@ private IEnumerable> ObserveOldestHandler() // ------------------------------------------------------------------------------------------------------------ private async Task ReceiverAsync(CancellationToken stop) + { + try + { + await ReceiveWithRestartsAsync(stop); + } + finally + { + if (!stop.IsCancellationRequested) _state = QueueProcessorState.Stopped; + } + } + + private async Task ReceiveWithRestartsAsync(CancellationToken stop) { while (!stop.IsCancellationRequested) { @@ -354,9 +369,7 @@ private async Task ReceiveUntilStoppedAsync(CancellationToken stop) ? await owned.ReceiveForProcessingAsync(take, _options.VisibilityTimeout, stop) : await queue.ReceiveAsync(take, _options.VisibilityTimeout, stop); - _lastReceiveSucceededAt = _time.GetUtcNow(); - _consecutiveReceiveFailures = 0; - _lastReceiveError = null; + _health = new ReceiveHealth(_time.GetUtcNow(), _health.FailedAt, 0, null); errorBackoff = _options.EmptyQueueBackoffMin; } catch (OperationCanceledException) when (stop.IsCancellationRequested) @@ -367,9 +380,7 @@ private async Task ReceiveUntilStoppedAsync(CancellationToken stop) catch (Exception ex) { slots.Release(take); - _lastReceiveFailedAt = _time.GetUtcNow(); - _consecutiveReceiveFailures++; - _lastReceiveError = ex.Message; + _health = _health with { FailedAt = _time.GetUtcNow(), ConsecutiveFailures = _health.ConsecutiveFailures + 1, LastError = ex.Message }; // Transport failures, a missing queue, no permission: log once, back off, keep going. if (ex is RequestFailedException { ErrorCode: var code } && code == QueueErrorCode.QueueNotFound) diff --git a/tests/AzureStorage.QueueService.Tests/CapturingLoggerProvider.cs b/tests/AzureStorage.QueueService.Tests/CapturingLoggerProvider.cs new file mode 100644 index 0000000..74eae34 --- /dev/null +++ b/tests/AzureStorage.QueueService.Tests/CapturingLoggerProvider.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.Logging; + +namespace AzureStorage.QueueService.Tests; + +/// Keeps every log entry, with a signal per entry, so a test can await a log line instead of sleeping. +internal sealed class CapturingLoggerProvider : ILoggerProvider +{ + private readonly Signal _logged = new(); + + public List<(LogLevel Level, int EventId, string Message)> Entries { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(this); + + public Task WaitForAsync(int eventId) => _logged.WaitUntilAsync(() => { lock (Entries) return Entries.Any(e => e.EventId == eventId); }); + + public (LogLevel Level, int EventId, string Message) Single(int eventId) + { + lock (Entries) return Entries.Single(e => e.EventId == eventId); + } + + public void Dispose() + { + } + + private sealed class Logger(CapturingLoggerProvider provider) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (provider.Entries) provider.Entries.Add((logLevel, eventId.Id, formatter(state, exception))); + provider._logged.Set(); + } + } +} diff --git a/tests/AzureStorage.QueueService.Tests/MessageEnvelopeTests.cs b/tests/AzureStorage.QueueService.Tests/MessageEnvelopeTests.cs new file mode 100644 index 0000000..6f24ccd --- /dev/null +++ b/tests/AzureStorage.QueueService.Tests/MessageEnvelopeTests.cs @@ -0,0 +1,56 @@ +using System.Diagnostics; +using FluentAssertions; + +namespace AzureStorage.QueueService.Tests; + +/// +/// The wire format's compatibility rules: what counts as an envelope, what a later version may add without breaking +/// this one, and what this version must refuse rather than misread. +/// +public class MessageEnvelopeTests +{ + private const string TraceParent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + + [Theory(DisplayName = "Envelope detection: plain payloads are left alone, later optional properties are skipped, anything else ours but unreadable is refused")] + [InlineData("{\"orderId\":7}", "Plain")] + [InlineData("not json at all", "Plain")] + [InlineData("{\"body\":{},\"$azqs-envelope\":1}", "Plain")] // marker not first + [InlineData("{\"$azqs-envelope\":1,\"body\":{\"a\":1}}", "Envelope")] // no trace context + [InlineData("{\"$azqs-envelope\":1,\"traceparent\":\"" + TraceParent + "\",\"priority\":3,\"body\":{}}", "Envelope")] // a property from a later version + [InlineData("{\"$azqs-envelope\":2,\"body\":{}}", "Unreadable")] // a later version + [InlineData("{\"$azqs-envelope\":1,\"encoding\":\"br\",\"body64\":\"AA==\"}", "Unreadable")] // an encoding this version cannot decode + [InlineData("{\"$azqs-envelope\":1,\"traceparent\":\"x\"}", "Unreadable")] // no body + [InlineData("{\"$azqs-envelope\":1,\"body\":{},\"body64\":\"AA==\"}", "Unreadable")] // two bodies + [InlineData("{\"$azqs-envelope\":1,\"body64\":\"not base64!\"}", "Unreadable")] + [InlineData("{\"$azqs-envelope\":1,\"body\":{}", "Unreadable")] // truncated + public void Detection(string text, string kind) + { + var expected = Enum.Parse(kind); + MessageEnvelope.TryRead(BinaryData.FromString(text), out _, out var problem).Should().Be(expected); + if (expected == EnvelopeKind.Unreadable) problem.Should().NotBeNullOrEmpty(); + } + + [Fact(DisplayName = "Everything an envelope carries survives a write and a read: trace context, headers, encoding, dead-letter record and binary body")] + public void Round_Trip() + { + using var caller = new Activity("caller").Start(); + byte[] bytes = [0xFF, 0x00, (byte)'{', 0x80]; + var written = new MessageEnvelope + { + Headers = new Dictionary { ["tenant"] = "contoso" }, + Encoding = MessageEnvelope.GzipEncoding, + DeadLetter = new DeadLetterInfo("gave up", DeadLetterCategories.Handler, "m1", 5, "orders", new DateTimeOffset(2026, 10, 16, 12, 0, 0, TimeSpan.Zero)), + }; + written.SetTraceContext(caller); + written.SetBody(new BinaryData(bytes)); + + MessageEnvelope.TryRead(written.Write(), out var read, out _).Should().Be(EnvelopeKind.Envelope); + + read!.Body!.ToArray().Should().Equal(bytes); + read.BodyIsJson.Should().BeFalse(); + read.Headers.Should().Equal(written.Headers); + read.Encoding.Should().Be(MessageEnvelope.GzipEncoding); + read.DeadLetter.Should().Be(written.DeadLetter); + read.CreationContext!.Value.TraceId.Should().Be(caller.TraceId); + } +} diff --git a/tests/AzureStorage.QueueService.Tests/ProcessorFeatureTests.cs b/tests/AzureStorage.QueueService.Tests/ProcessorFeatureTests.cs new file mode 100644 index 0000000..72fa888 --- /dev/null +++ b/tests/AzureStorage.QueueService.Tests/ProcessorFeatureTests.cs @@ -0,0 +1,279 @@ +using System.ComponentModel.DataAnnotations; +using Azure.Messaging; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.Metrics.Testing; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using System.Diagnostics.Metrics; +using Xunit.Abstractions; + +namespace AzureStorage.QueueService.Tests; + +/// +/// The 3.1 processor features end to end, on in-memory queues: a real host, the real processor and the real queue code. +/// Messages are sent before the host starts, so the processor's first receives find them and nothing waits on a clock; +/// the clock is a fake that never moves. Stopping the host drains the deliveries, so their settlement is finished by then. +/// +public class ProcessorFeatureTests(ITestOutputHelper output) +{ + private static readonly TimeSpan SafetyNet = TimeSpan.FromSeconds(10); + + public sealed record OrderCreated(string Id); + + public sealed record OrderCancelled(string Id); + + /// What the handlers saw, in order, with a signal per entry so a test awaits outcomes instead of sleeping. + public sealed class Recorder + { + private readonly Signal _added = new(); + public List Entries { get; } = new(); + + public void Add(string entry) + { + lock (Entries) Entries.Add(entry); + _added.Set(); + } + + public Task WaitForAsync(int count) => _added.WaitUntilAsync(() => { lock (Entries) return Entries.Count >= count; }); + } + + private sealed class CreatedHandler(Recorder recorder) : IQueueMessageHandler + { + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) + { + recorder.Add($"created {message.Body!.Id} ({message.Headers[QueueMessageHeaders.MessageType]})"); + return ValueTask.CompletedTask; + } + } + + private sealed class CancelledHandler(Recorder recorder) : IQueueMessageHandler + { + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) + { + recorder.Add($"cancelled {message.Body!.Id}"); + return ValueTask.CompletedTask; + } + } + + private sealed class EventHandler(Recorder recorder) : ICloudEventHandler + { + public ValueTask HandleAsync(QueueMessage message, OrderCreated? data, CancellationToken cancellationToken) + { + recorder.Add($"{message.Body!.Type} {data!.Id}"); + return ValueTask.CompletedTask; + } + } + + private sealed class ThrowingHandler(Recorder recorder) : IQueueMessageHandler + { + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) + { + recorder.Add($"{message.Body!.Property1} attempt {message.DequeueCount}"); + throw message.Body.Property1 == "invalid" ? new ValidationException("no") : new TimeoutException("slow downstream"); + } + } + + private sealed class RecordingHandler(Recorder recorder) : IQueueMessageHandler + { + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) + { + recorder.Add($"handler {message.Body!.Property1}"); + return ValueTask.CompletedTask; + } + } + + /// A scoped middleware that stops the pipeline for messages marked "skip". + private sealed class SkippingMiddleware(Recorder recorder) : IQueueMessageMiddleware + { + public async ValueTask InvokeAsync(QueueMessageContext context, QueueMessageDelegate next) + { + var body = ((TestObject)context.Message.Body!).Property1; + recorder.Add($"client {body}"); + if (body != "skip") await next(context); + } + } + + private IHost Build(Action configure, Action? services = null) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings()); + builder.Logging.AddProvider(new XUnitLoggerProvider(output)); + builder.Services.AddMetrics(); + builder.Services.AddSingleton(new FakeTimeProvider()); + builder.Services.AddSingleton(); + builder.Services.AddAzureStorageQueueClient(configure); + builder.Services.AddInMemoryQueues(); + services?.Invoke(builder.Services); + return builder.Build(); + } + + private static async Task> DeadLettered(IHost host, string client) => + await host.Services.GetRequiredKeyedService(client).PeekAsync(); + + [Fact(DisplayName = "A routed processor sends each message type to its own handler and dead-letters a type nothing is routed for")] + public async Task Routed_Processor() + { + using var host = Build(queues => queues + .AddClient("orders", s => { s.QueueName = "orders"; s.SendMessageType = true; }) + .AddRoutedProcessor( + routes => routes.Route().Route("cancelled"), + o => o.MaxConcurrentMessages = 3)); + var sender = host.Services.GetRequiredKeyedService("orders"); + await sender.SendAsync(new OrderCreated("1")); + await sender.SendAsync(new OrderCancelled("2"), new SendOptions { MessageType = "cancelled" }); + await sender.SendAsync(new OrderCancelled("3"), new SendOptions { MessageType = "refunded" }); + + await host.StartAsync(); + await host.Services.GetRequiredService().WaitForAsync(2); + await host.StopAsync(); + + host.Services.GetRequiredService().Entries.Should().BeEquivalentTo("created 1 (OrderCreated)", "cancelled 2"); + var unroutable = (await DeadLettered(host, "orders")).Should().ContainSingle().Subject; + unroutable.Category.Should().Be(DeadLetterCategories.Unroutable); + unroutable.Reason.Should().Contain("'refunded'"); + } + + [Fact(DisplayName = "A CloudEvent router sends each event to the handler for its type, with its data typed")] + public async Task CloudEvent_Router() + { + using var host = Build(queues => queues + .AddClient("events", s => s.QueueName = "events") + .AddCloudEventRouter(routes => routes.Route("Contoso.Orders.Created"))); + var sender = host.Services.GetRequiredKeyedService("events"); + await sender.SendAsync(new CloudEvent("/orders", "Contoso.Orders.Created", new OrderCreated("7"))); + await sender.SendAsync(new CloudEvent("/orders", "Contoso.Orders.Shipped", new OrderCreated("8"))); + + await host.StartAsync(); + await host.Services.GetRequiredService().WaitForAsync(1); + await host.StopAsync(); + + host.Services.GetRequiredService().Entries.Should().Equal("Contoso.Orders.Created 7"); + (await DeadLettered(host, "events")).Should().ContainSingle().Which.Category.Should().Be(DeadLetterCategories.Unroutable); + } + + [Fact(DisplayName = "A lambda handler gets the body, the message, the token and services from the message's scope")] + public async Task Lambda_Handler() + { + using var host = Build(queues => queues + .AddClient("orders", s => s.QueueName = "orders") + .AddProcessor((TestObject body, IQueueMessage message, Recorder recorder, CancellationToken token) => + recorder.Add($"{body.Property1} from {message.QueueName}, cancellable {token.CanBeCanceled}"))); + await host.Services.GetRequiredKeyedService("orders").SendAsync(new TestObject { Property1 = "hi" }); + + await host.StartAsync(); + await host.Services.GetRequiredService().WaitForAsync(1); + await host.StopAsync(); + + host.Services.GetRequiredService().Entries.Should().Equal("hi from orders, cancellable True"); + (await host.Services.GetRequiredKeyedService("orders").GetPropertiesAsync()).ApproximateMessagesCount.Should().Be(0); + } + + [Fact(DisplayName = "Middleware runs in registration order, global first here; one that does not call next completes the message without the handler")] + public async Task Middleware_Order_And_Short_Circuit() + { + using var host = Build(queues => + { + queues.Use(async (context, next) => + { + context.Services.GetRequiredService().Add($"global {((TestObject)context.Message.Body!).Property1}"); + await next(context); + }); + queues.AddClient("orders", s => s.QueueName = "orders") + .UseMiddleware() + .AddProcessor(); + }); + var sender = host.Services.GetRequiredKeyedService("orders"); + await sender.SendAsync(new TestObject { Property1 = "run" }); + await sender.SendAsync(new TestObject { Property1 = "skip" }); + + await host.StartAsync(); + await host.Services.GetRequiredService().WaitForAsync(5); + await host.StopAsync(); + + host.Services.GetRequiredService().Entries.Should().Equal("global run", "client run", "handler run", "global skip", "client skip"); + (await host.Services.GetRequiredKeyedService("orders").GetPropertiesAsync()).ApproximateMessagesCount.Should().Be(0, "the skipped message counts as handled"); + } + + [Fact(DisplayName = "Error policy: DeadLetterOn dead-letters without a retry; OnHandlerError can complete; the handler sees IsLastAttempt")] + public async Task Error_Policy() + { + using var host = Build(queues => queues + .AddClient("orders", s => s.QueueName = "orders") + .AddProcessor(o => + { + o.MaxConcurrentMessages = 2; + o.DeadLetterOn(); + o.OnHandlerError = context => context.Exception is TimeoutException ? HandlerErrorAction.Complete : HandlerErrorAction.Retry; + })); + var sender = host.Services.GetRequiredKeyedService("orders"); + await sender.SendAsync(new TestObject { Property1 = "invalid" }); + await sender.SendAsync(new TestObject { Property1 = "timeout" }); + + await host.StartAsync(); + await host.Services.GetRequiredService().WaitForAsync(2); + await host.StopAsync(); + + host.Services.GetRequiredService().Entries.Should().BeEquivalentTo("invalid attempt 1", "timeout attempt 1"); + var deadLettered = (await DeadLettered(host, "orders")).Should().ContainSingle().Subject; + deadLettered.Category.Should().Be(DeadLetterCategories.Handler); + deadLettered.Reason.Should().Contain("ValidationException"); + (await host.Services.GetRequiredKeyedService("orders").GetPropertiesAsync()).ApproximateMessagesCount.Should().Be(0, "the timeout was completed by the policy"); + } + + [Fact(DisplayName = "A client paused before the host starts receives nothing until it is resumed")] + public async Task Pause_And_Resume() + { + using var host = Build(queues => queues.AddClient("orders", s => s.QueueName = "orders").AddProcessor()); + var controller = host.Services.GetRequiredService(); + var orders = host.Services.GetRequiredKeyedService("orders"); + await orders.SendAsync(new TestObject { Property1 = "held" }); + controller.Pause("orders"); + + await host.StartAsync(); + + controller.GetStatus().Should().ContainSingle().Which.State.Should().Be(QueueProcessorState.Paused); + (await orders.PeekAsync()).Should().ContainSingle("a paused processor has not taken the message"); + controller.Invoking(c => c.Pause("typo")).Should().Throw().WithMessage("*'orders'*"); + + controller.Resume("orders"); + await host.Services.GetRequiredService().WaitForAsync(1); + await host.StopAsync(); + host.Services.GetRequiredService().Entries.Should().Equal("handler held"); + } + + [Fact(DisplayName = "Metrics: time in queue, receive requests by result, dead-letters by reason, and the messaging-convention duration")] + public async Task Metrics() + { + using var host = Build(queues => queues + .AddClient("orders", s => s.QueueName = "orders") + .AddProcessor(o => o.DeadLetterOn())); + var meters = host.Services.GetRequiredService(); + using var timeInQueue = new MetricCollector(meters, "AzureStorage.QueueService", "queue_message_time_in_queue_seconds"); + using var receives = new MetricCollector(meters, "AzureStorage.QueueService", "queue_receive_requests_total"); + using var deadLettered = new MetricCollector(meters, "AzureStorage.QueueService", "queue_messages_dead_lettered_total"); + using var processDuration = new MetricCollector(meters, "AzureStorage.QueueService.Messaging", "messaging.process.duration"); + await host.Services.GetRequiredKeyedService("orders").SendAsync(new TestObject { Property1 = "invalid" }); + + await host.StartAsync(); + await host.Services.GetRequiredService().WaitForAsync(1); + await host.StopAsync(); + + timeInQueue.GetMeasurementSnapshot().Should().ContainSingle().Which.Value.Should().Be(0, "the fake clock never moved"); + receives.GetMeasurementSnapshot().Should().Contain(m => (string?)m.Tags["queue.receive_result"] == "message"); + deadLettered.GetMeasurementSnapshot().Should().ContainSingle().Which.Tags["queue.dead_letter_reason"].Should().Be(DeadLetterCategories.Handler); + processDuration.GetMeasurementSnapshot().Should().ContainSingle().Which.Tags["error.type"].Should().Be(typeof(ValidationException).FullName); + } + + [Fact(DisplayName = "Registering the same handler on the same client twice runs one processor with the merged options")] + public void Duplicate_Registration_Runs_One_Processor() + { + using var host = Build(queues => + { + queues.AddClient("orders", s => s.QueueName = "orders").AddProcessor(o => o.MaxConcurrentMessages = 2); + queues.AddClient("orders", _ => { }).AddProcessor(o => o.RenewVisibility = false); + }); + + host.Services.GetServices().Should().ContainSingle(); + } +} diff --git a/tests/AzureStorage.QueueService.Tests/QueueFeatureTests.cs b/tests/AzureStorage.QueueService.Tests/QueueFeatureTests.cs new file mode 100644 index 0000000..c32e5ee --- /dev/null +++ b/tests/AzureStorage.QueueService.Tests/QueueFeatureTests.cs @@ -0,0 +1,183 @@ +using Azure.Storage.Queues; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; + +namespace AzureStorage.QueueService.Tests; + +/// +/// The 3.1 queue features through the public API on in-memory queues, which run the same queue code as the service: +/// guardrails before a send, what travels with a message, the poison queue's record and replay, and reading. +/// +public class QueueFeatureTests +{ + private static ServiceProvider Services(Action configure) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(new FakeTimeProvider()); + services.AddAzureStorageQueueClient(configure); + services.AddInMemoryQueues(); + return services.BuildServiceProvider(); + } + + private static IStorageQueue Orders(ServiceProvider provider) => provider.GetRequiredKeyedService("orders"); + + [Fact(DisplayName = "Send refuses, before calling the service, a message over 64 KiB as it would go on the queue, Base64 growth included, and a delay the message would expire in")] + public async Task Send_Guardrails() + { + using var provider = Services(queues => + { + queues.AddClient("orders", s => s.QueueName = "orders"); + queues.AddClient("base64", s => { s.QueueName = "base64"; s.ClientOptions = new QueueClientOptions { MessageEncoding = QueueMessageEncoding.Base64 }; }); + }); + var fitsPlain = new TestObject { Property1 = new string('x', 55_000) }; + + await Orders(provider).Invoking(q => q.SendAsync(new TestObject { Property1 = new string('x', 70_000) }).AsTask()) + .Should().ThrowAsync().WithMessage("*UseBlobStorageForLargeMessages*"); + await Orders(provider).SendAsync(fitsPlain); + await provider.GetRequiredKeyedService("base64").Invoking(q => q.SendAsync(fitsPlain).AsTask()) + .Should().ThrowAsync("Base64 makes it a third bigger"); + await Orders(provider).Invoking(q => q.SendAsync(fitsPlain, new SendOptions(TimeToLive: TimeSpan.FromHours(1)) { Delay = TimeSpan.FromHours(2) }).AsTask()) + .Should().ThrowAsync().WithMessage("*expire before*"); + } + + [Fact(DisplayName = "Receive, abandon and renew check their timeouts before calling the service")] + public async Task Timeouts_Are_Checked() + { + using var provider = Services(queues => queues.AddClient("orders", s => s.QueueName = "orders")); + await Orders(provider).SendAsync(new TestObject()); + + await Orders(provider).Invoking(q => q.ReceiveAsync(visibilityTimeout: TimeSpan.FromMilliseconds(500)).AsTask()).Should().ThrowAsync(); + var message = (await Orders(provider).ReceiveAsync())[0]; + await message.Invoking(m => m.AbandonAsync(TimeSpan.FromDays(8)).AsTask()).Should().ThrowAsync(); + await message.Invoking(m => m.RenewLockAsync(TimeSpan.FromDays(8)).AsTask()).Should().ThrowAsync(); + message.IsSettled.Should().BeFalse("nothing reached the queue"); + } + + [Fact(DisplayName = "Headers and a compressed body travel with the message and come back as sent, whatever the receiving client's settings")] + public async Task Headers_And_Compression_Round_Trip() + { + using var provider = Services(queues => + { + queues.AddClient("sender", s => s.QueueName = "orders").UseCompression(thresholdBytes: 100); + queues.AddClient("orders", s => s.QueueName = "orders"); + }); + var body = new TestObject { Property1 = new string('a', 5_000) }; + + await provider.GetRequiredKeyedService("sender").SendAsync(body, new SendOptions { Headers = new Dictionary { ["tenant"] = "contoso" } }); + var received = (await Orders(provider).ReceiveAsync())[0]; + + received.Body!.Property1.Should().Be(body.Property1); + received.Headers.Should().Contain("tenant", "contoso"); + received.WireBody.ToString().Should().Contain("\"encoding\":\"gzip\""); + received.WireBody.ToMemory().Length.Should().BeLessThan(1_000, "5,000 repeated characters compress well"); + } + + [Fact(DisplayName = "The default serializer reads camelCase JSON into PascalCase properties instead of leaving them empty")] + public void Default_Serializer_Reads_CamelCase() => + new JsonQueueMessageSerializer().Deserialize(BinaryData.FromString("{\"property1\":\"from JavaScript\",\"property3\":3}")) + .Should().BeEquivalentTo(new { Property1 = "from JavaScript", Property3 = 3 }); + + [Fact(DisplayName = "A message in a later version's envelope comes back with Error set, so a processor dead-letters it intact for replay after upgrading")] + public async Task Later_Version_Envelope_Is_Not_Misread() + { + using var provider = Services(queues => queues.AddClient("orders", s => s.QueueName = "orders")); + await Orders(provider).SendAsync(System.Text.Json.Nodes.JsonNode.Parse("{\"$azqs-envelope\":2,\"body\":{\"Property1\":\"x\"}}")!); + + var message = (await Orders(provider).ReceiveAsync())[0]; + + message.Body.Should().BeNull(); + message.Error!.Message.Should().Contain("newer version"); + } + + [Fact(DisplayName = "The poison queue records why each message was dead-lettered, and a filtered replay moves only the matching ones back as they were sent")] + public async Task Dead_Letter_Queue_Peek_And_Replay() + { + using var provider = Services(queues => queues.AddClient("orders", s => s.QueueName = "orders")); + var orders = Orders(provider); + await orders.SendAsync(new TestObject { Property1 = "retry me" }); + await orders.SendAsync(new TestObject { Property1 = "leave me" }); + foreach (var message in await orders.ReceiveAsync(maxMessages: 2)) + await message.DeadLetterAsync(message.Body!.Property1 == "retry me" ? "downstream was down" : "bad data"); + var deadLetters = provider.GetRequiredKeyedService("orders"); + + var peeked = await deadLetters.PeekAsync(); + peeked.Should().HaveCount(2).And.OnlyContain(m => m.SourceQueueName == "orders" && m.DequeueCount == 1 && m.OriginalMessageId != null); + peeked.Select(m => m.GetBody()!.Property1).Should().BeEquivalentTo("retry me", "leave me"); + + (await deadLetters.ReplayAsync(filter: m => m.Reason == "downstream was down")).Should().Be(1); + + var replayed = (await orders.ReceiveAsync()).Should().ContainSingle().Subject; + replayed.Body!.Property1.Should().Be("retry me"); + replayed.DequeueCount.Should().Be(1, "a replayed message starts again"); + replayed.WireBody.ToString().Should().NotContain("deadLetter", "the record is stripped on replay"); + (await deadLetters.PeekAsync()).Should().ContainSingle("the skipped one stays, visible").Which.Reason.Should().Be("bad data"); + } + + [Fact(DisplayName = "ReadAllAsync reports a failed receive, backs off on errors and empty polls, and yields messages as they arrive")] + public async Task ReadAllAsync_Backs_Off_And_Yields() + { + var time = new TimerCountingClock(); + var receiver = new ScriptedReceiver(); + var errors = new List(); + using var stop = new CancellationTokenSource(); + await using var reading = receiver.ReadAllAsync(new ReadAllOptions { TimeProvider = time, OnReceiveError = errors.Add }, stop.Token).GetAsyncEnumerator(); + + var next = reading.MoveNextAsync(); // the first receive throws, then the loop waits + await time.WaitForTimersAsync(1); + errors.Should().ContainSingle(); + time.Advance(TimeSpan.FromSeconds(1)); // the second receive is empty, then the loop waits again + await time.WaitForTimersAsync(2); + time.Advance(TimeSpan.FromSeconds(1)); // the third has a message + + (await next.AsTask().WaitAsync(TimeSpan.FromSeconds(10))).Should().BeTrue(); + reading.Current.MessageId.Should().Be("m1"); + stop.Cancel(); + (await reading.MoveNextAsync()).Should().BeFalse("cancellation ends the loop without throwing"); + } + + private sealed class ScriptedReceiver : IQueueReceiver + { + private int _calls; + + public string QueueName => "orders"; + + public ValueTask>> ReceiveAsync(int maxMessages = 1, TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Interlocked.Increment(ref _calls) switch + { + 1 => throw new TimeoutException("storage is down"), + 3 => new([new QueueMessage(new QueueMessageInfo("m1", "r", 1, null, null, null), BinaryData.FromString("{}"), default, null, new NoOperations())]), + _ => new(Array.Empty>()), + }; + } + + public ValueTask>> PeekAsync(int maxMessages = 1, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + private sealed class NoOperations : IQueueMessageOperations + { + public ValueTask DeleteAsync(string messageId, string popReceipt, CancellationToken cancellationToken) => ValueTask.CompletedTask; + public ValueTask UpdateVisibilityAsync(string messageId, string popReceipt, TimeSpan visibilityTimeout, CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeadLetterAsync(string messageId, string popReceipt, long dequeueCount, BinaryData rawBody, string? reason, CancellationToken cancellationToken) => throw new NotSupportedException(); + } + + /// A fake clock that reports timer registrations, so the test advances only once the loop is waiting. + private sealed class TimerCountingClock : FakeTimeProvider + { + private int _timers; + private readonly Signal _created = new(); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = base.CreateTimer(callback, state, dueTime, period); + Interlocked.Increment(ref _timers); + _created.Set(); + return timer; + } + + public Task WaitForTimersAsync(int count) => _created.WaitUntilAsync(() => Volatile.Read(ref _timers) >= count); + } +} diff --git a/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs b/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs index 43580e1..bef6f8e 100644 --- a/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs +++ b/tests/AzureStorage.QueueService.Tests/QueueProcessorTests.cs @@ -169,15 +169,19 @@ private sealed class Harness : IAsyncDisposable public ObservableTimeProvider Time { get; } = new(); public QueueProcessor Processor { get; } + public CapturingLoggerProvider Logs { get; } = new(); + public Harness(ITestOutputHelper output, Action? configure = null, IStorageQueue? queue = null) { var services = new ServiceCollection(); - services.AddLogging(logging => logging.AddProvider(new XUnitLoggerProvider(output))); + services.AddLogging(logging => logging.AddProvider(new XUnitLoggerProvider(output)).AddProvider(Logs)); services.AddSingleton(State); services.AddScoped(); services.AddScoped(); services.Configure("opts", o => { + // Off, so the timers a test counts are the processor's own waits and renewals. + o.LongRunningHandlerWarning = null; o.EmptyQueueBackoffMin = TimeSpan.FromSeconds(1); o.EmptyQueueBackoffMax = TimeSpan.FromSeconds(8); configure?.Invoke(o); @@ -829,4 +833,67 @@ public async Task CloudEvent_With_Unreadable_Data_Reaches_Handler_When_Dead_Lett message.Error.Should().NotBeNull(); h.Ops.Count("deadletter").Should().Be(0); } + + // --------------------------------------------------------------------------------------------------------- + // 3.1: log levels, the last attempt, rate limiting, hung handlers + // --------------------------------------------------------------------------------------------------------- + + [Fact(DisplayName = "A failing handler is a warning while retries remain and an error on the last attempt, which the handler can see")] + public async Task Handler_Failure_Log_Level_Follows_The_Last_Attempt() + { + await using var h = new Harness(output, o => o.MaxDequeueCount = 5); + var lastAttempts = new List(); + h.State.Behaviour = (message, _, _) => + { + lock (lastAttempts) lastAttempts.Add(message.IsLastAttempt); + throw new InvalidOperationException("boom"); + }; + h.Queue.Then(h.Message("early", dequeueCount: 1)).Then(h.Message("last", dequeueCount: 5)); + + await h.StartAsync(); + await h.Ops.WaitForAsync("update", count: 2); + + lastAttempts.Should().Equal(false, true); + h.Logs.Single(7).Level.Should().Be(LogLevel.Warning); + h.Logs.Single(27).Level.Should().Be(LogLevel.Error); + } + + [Fact(DisplayName = "With a rate limiter, a receive asks only for as many messages as there are permits")] + public async Task Rate_Limiter_Caps_The_Receive() + { + using var limiter = new System.Threading.RateLimiting.TokenBucketRateLimiter(new System.Threading.RateLimiting.TokenBucketRateLimiterOptions + { + TokenLimit = 2, + TokensPerPeriod = 1, + ReplenishmentPeriod = TimeSpan.FromHours(1), + AutoReplenishment = false, + }); + await using var h = new Harness(output, o => { o.MaxConcurrentMessages = 5; o.RateLimiter = limiter; }); + h.Queue.Then(h.Message("1")); + + await h.StartAsync(); + await h.Queue.WaitForReceivesAsync(1); + + h.Queue.Receives[0].Take.Should().Be(2, "five handlers are free but only two permits are"); + } + + [Fact(DisplayName = "MaxRenewalDuration stops renewing a handler's lease and says so, so a hung handler cannot hide its message for ever")] + public async Task Renewal_Stops_At_MaxRenewalDuration() + { + await using var h = new Harness(output, o => { o.VisibilityTimeout = TimeSpan.FromSeconds(10); o.MaxRenewalDuration = TimeSpan.FromSeconds(8); }); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + h.State.Behaviour = async (_, _, _) => await release.Task; + h.Queue.Then(h.Message("1")); + + await h.StartAsync(); + await h.WaitForHandlerStartAsync(); + await h.AdvancePastTimerAsync(1, TimeSpan.FromSeconds(5)); // 5 s in: renewed + await h.Ops.WaitForAsync("update"); + await h.AdvancePastTimerAsync(2, TimeSpan.FromSeconds(5)); // 10 s in: past the limit + await h.Logs.WaitForAsync(32); + + release.SetResult(); + await h.Ops.WaitForAsync("delete"); + h.Ops.Count("update").Should().Be(1); + } } From 839754eb9d74c94f775d45b2ae7aea1d46e94553 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:43:35 -0600 Subject: [PATCH 7/8] WIP: registration tests, CI blob port, solution entries --- .github/workflows/dotnet.yml | 5 +- AzureStorage.QueueService.sln | 98 +++++++++++++++++++ .../QueueClientBuilderExtensions.cs | 7 +- .../RegistrationFeatureTests.cs | 95 ++++++++++++++++++ 4 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 tests/AzureStorage.QueueService.Tests/RegistrationFeatureTests.cs diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index f2d41d1..3874f02 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -17,10 +17,11 @@ jobs: # live storage account. It is started with docker run rather than as a # service container because it needs --skipApiVersionCheck: the Azure SDK # sends an API version newer than Azurite knows about and Azurite rejects - # it with a 400 otherwise. Only the queue port (10001) is mapped. + # it with a 400 otherwise. The queue (10001) and blob (10000) ports are + # mapped; the blob service backs the large-message package's tests. - name: Start Azurite run: | - docker run -d --name azurite -p 10001:10001 \ + docker run -d --name azurite -p 10000:10000 -p 10001:10001 \ mcr.microsoft.com/azure-storage/azurite \ azurite --queueHost 0.0.0.0 --blobHost 0.0.0.0 --tableHost 0.0.0.0 --skipApiVersionCheck --silent - name: Setup .NET diff --git a/AzureStorage.QueueService.sln b/AzureStorage.QueueService.sln index aaee5b1..97ada62 100644 --- a/AzureStorage.QueueService.sln +++ b/AzureStorage.QueueService.sln @@ -13,28 +13,122 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureStorage.QueueClient.In EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{F7562E40-A3C7-45FB-8DF2-3228F6F09488}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureStorage.QueueService.Blobs", "src\AzureStorage.QueueService.Blobs\AzureStorage.QueueService.Blobs.csproj", "{66CBCB52-CB4E-40A5-A020-D7F7758F2D87}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureStorage.QueueService.HealthChecks", "src\AzureStorage.QueueService.HealthChecks\AzureStorage.QueueService.HealthChecks.csproj", "{873B8113-8E38-4E15-B10A-A1A7E4D5592D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureStorage.QueueService.Blobs.Tests", "tests\AzureStorage.QueueService.Blobs.Tests\AzureStorage.QueueService.Blobs.Tests.csproj", "{480E270B-BDAF-466E-A16B-272369A058F3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureStorage.QueueService.HealthChecks.Tests", "tests\AzureStorage.QueueService.HealthChecks.Tests\AzureStorage.QueueService.HealthChecks.Tests.csproj", "{33AFC688-A141-43C6-A1F7-F991132A3BBD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {F18BA53F-94A4-49C2-A389-56479794D255}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F18BA53F-94A4-49C2-A389-56479794D255}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Debug|x64.ActiveCfg = Debug|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Debug|x64.Build.0 = Debug|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Debug|x86.ActiveCfg = Debug|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Debug|x86.Build.0 = Debug|Any CPU {F18BA53F-94A4-49C2-A389-56479794D255}.Release|Any CPU.ActiveCfg = Release|Any CPU {F18BA53F-94A4-49C2-A389-56479794D255}.Release|Any CPU.Build.0 = Release|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Release|x64.ActiveCfg = Release|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Release|x64.Build.0 = Release|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Release|x86.ActiveCfg = Release|Any CPU + {F18BA53F-94A4-49C2-A389-56479794D255}.Release|x86.Build.0 = Release|Any CPU {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Debug|x64.ActiveCfg = Debug|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Debug|x64.Build.0 = Debug|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Debug|x86.ActiveCfg = Debug|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Debug|x86.Build.0 = Debug|Any CPU {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Release|Any CPU.ActiveCfg = Release|Any CPU {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Release|Any CPU.Build.0 = Release|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Release|x64.ActiveCfg = Release|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Release|x64.Build.0 = Release|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Release|x86.ActiveCfg = Release|Any CPU + {4F14CBE5-50BB-47EF-A51D-D9C202B1C509}.Release|x86.Build.0 = Release|Any CPU {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Debug|x64.ActiveCfg = Debug|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Debug|x64.Build.0 = Debug|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Debug|x86.ActiveCfg = Debug|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Debug|x86.Build.0 = Debug|Any CPU {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Release|Any CPU.ActiveCfg = Release|Any CPU {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Release|Any CPU.Build.0 = Release|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Release|x64.ActiveCfg = Release|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Release|x64.Build.0 = Release|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Release|x86.ActiveCfg = Release|Any CPU + {4797C356-BA6D-4E03-AD91-F4C66D6799B5}.Release|x86.Build.0 = Release|Any CPU {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Debug|x64.ActiveCfg = Debug|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Debug|x64.Build.0 = Debug|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Debug|x86.ActiveCfg = Debug|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Debug|x86.Build.0 = Debug|Any CPU {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Release|Any CPU.ActiveCfg = Release|Any CPU {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Release|Any CPU.Build.0 = Release|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Release|x64.ActiveCfg = Release|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Release|x64.Build.0 = Release|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Release|x86.ActiveCfg = Release|Any CPU + {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD}.Release|x86.Build.0 = Release|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Debug|Any CPU.Build.0 = Debug|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Debug|x64.ActiveCfg = Debug|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Debug|x64.Build.0 = Debug|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Debug|x86.ActiveCfg = Debug|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Debug|x86.Build.0 = Debug|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Release|Any CPU.ActiveCfg = Release|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Release|Any CPU.Build.0 = Release|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Release|x64.ActiveCfg = Release|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Release|x64.Build.0 = Release|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Release|x86.ActiveCfg = Release|Any CPU + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87}.Release|x86.Build.0 = Release|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Debug|x64.ActiveCfg = Debug|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Debug|x64.Build.0 = Debug|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Debug|x86.ActiveCfg = Debug|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Debug|x86.Build.0 = Debug|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Release|Any CPU.Build.0 = Release|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Release|x64.ActiveCfg = Release|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Release|x64.Build.0 = Release|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Release|x86.ActiveCfg = Release|Any CPU + {873B8113-8E38-4E15-B10A-A1A7E4D5592D}.Release|x86.Build.0 = Release|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Debug|x64.ActiveCfg = Debug|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Debug|x64.Build.0 = Debug|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Debug|x86.ActiveCfg = Debug|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Debug|x86.Build.0 = Debug|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Release|Any CPU.Build.0 = Release|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Release|x64.ActiveCfg = Release|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Release|x64.Build.0 = Release|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Release|x86.ActiveCfg = Release|Any CPU + {480E270B-BDAF-466E-A16B-272369A058F3}.Release|x86.Build.0 = Release|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Debug|x64.ActiveCfg = Debug|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Debug|x64.Build.0 = Debug|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Debug|x86.ActiveCfg = Debug|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Debug|x86.Build.0 = Debug|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Release|Any CPU.Build.0 = Release|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Release|x64.ActiveCfg = Release|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Release|x64.Build.0 = Release|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Release|x86.ActiveCfg = Release|Any CPU + {33AFC688-A141-43C6-A1F7-F991132A3BBD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -43,6 +137,10 @@ Global {4F14CBE5-50BB-47EF-A51D-D9C202B1C509} = {F7562E40-A3C7-45FB-8DF2-3228F6F09488} {4797C356-BA6D-4E03-AD91-F4C66D6799B5} = {F7562E40-A3C7-45FB-8DF2-3228F6F09488} {191F48D2-EA72-42A6-8E6E-8CAE5F5225DD} = {F7562E40-A3C7-45FB-8DF2-3228F6F09488} + {66CBCB52-CB4E-40A5-A020-D7F7758F2D87} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {873B8113-8E38-4E15-B10A-A1A7E4D5592D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {480E270B-BDAF-466E-A16B-272369A058F3} = {F7562E40-A3C7-45FB-8DF2-3228F6F09488} + {33AFC688-A141-43C6-A1F7-F991132A3BBD} = {F7562E40-A3C7-45FB-8DF2-3228F6F09488} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7FFBEC97-1EFF-4CB6-8790-5B264434E3AE} diff --git a/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs b/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs index b27c44e..0c60554 100644 --- a/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs +++ b/src/AzureStorage.QueueService/QueueClientBuilderExtensions.cs @@ -92,7 +92,12 @@ public static IQueueClientBuilder UseLargeMessageStore(this IQueueClientBuilder ArgumentNullException.ThrowIfNull(createStore); builder.Services.AddKeyedSingleton(QueueClientFactory.LargeMessageStoreKey(builder.Name), (provider, _) => createStore(provider)); - return builder.Configure(settings => settings.LargeMessageThreshold = thresholdBytes); + // Clears a store set earlier, so whichever UseLargeMessageStore call comes last is the one that applies. + return builder.Configure(settings => + { + settings.LargeMessageStore = null; + settings.LargeMessageThreshold = thresholdBytes; + }); } /// diff --git a/tests/AzureStorage.QueueService.Tests/RegistrationFeatureTests.cs b/tests/AzureStorage.QueueService.Tests/RegistrationFeatureTests.cs new file mode 100644 index 0000000..f5112f7 --- /dev/null +++ b/tests/AzureStorage.QueueService.Tests/RegistrationFeatureTests.cs @@ -0,0 +1,95 @@ +using Azure.Identity; +using Azure.Storage.Queues; +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace AzureStorage.QueueService.Tests; + +/// 3.1 registration: one storage account for every client, configuration binding, and clear errors for a client nobody registered. +public class RegistrationFeatureTests +{ + private sealed class Handler : IQueueMessageHandler + { + public ValueTask HandleAsync(QueueMessage message, CancellationToken cancellationToken) => ValueTask.CompletedTask; + } + + private static QueueClientSettings Settings(IServiceProvider provider, string name) => + provider.GetRequiredService>().Get(name); + + [Fact(DisplayName = "UseStorageAccount connects clients that only name their queue: a connection string, or an account endpoint with the shared credential")] + public void UseStorageAccount() + { + var credential = new DefaultAzureCredential(); + var services = new ServiceCollection(); + services.AddAzureStorageQueueClient(queues => + { + queues.AddClient("orders", s => s.QueueName = "orders"); + queues.AddClient("own", s => { s.ConnectionString = "UseDevelopmentStorage=true"; s.QueueName = "own"; }); + queues.UseStorageAccount("https://acct.queue.core.windows.net/").UseTokenCredential(credential); + }); + using var provider = services.BuildServiceProvider(); + + var orders = Settings(provider, "orders"); + orders.Validate().Should().BeNull(); + orders.QueueUri.Should().Be(new Uri("https://acct.queue.core.windows.net/orders")); + orders.TokenCredential.Should().BeSameAs(credential); + orders.EffectivePoisonQueueName.Should().Be("orders-poison"); + QueueClientFactory.ForSiblingQueue(orders, "orders-poison").EndpointUri.Should().Be(new Uri("https://acct.queue.core.windows.net/orders-poison")); + Settings(provider, "own").EndpointUri.Should().BeNull("a client with its own connection keeps it"); + + var withConnectionString = new ServiceCollection(); + withConnectionString.AddAzureStorageQueueClient(queues => queues.UseStorageAccount("UseDevelopmentStorage=true").AddClient("orders", s => s.QueueName = "orders")); + using var second = withConnectionString.BuildServiceProvider(); + Settings(second, "orders").ConnectionString.Should().Be("UseDevelopmentStorage=true"); + } + + [Theory(DisplayName = "An EndpointUri that is the account's endpoint takes the queue from QueueName, emulator path-style URIs included")] + [InlineData("https://acct.queue.core.windows.net", "https://acct.queue.core.windows.net/orders")] + [InlineData("http://127.0.0.1:10001/devstoreaccount1", "http://127.0.0.1:10001/devstoreaccount1/orders")] + public void Account_Endpoint_With_QueueName(string endpoint, string expected) + { + var settings = new QueueClientSettings { EndpointUri = new Uri(endpoint), QueueName = "orders", TokenCredential = new DefaultAzureCredential() }; + + settings.Validate().Should().BeNull(); + settings.QueueUri.Should().Be(new Uri(expected)); + } + + [Fact(DisplayName = "Injecting a client nobody registered fails with the factory's message naming the registered clients")] + public void Unregistered_Clients_Fail_Clearly() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddAzureStorageQueueClient(queues => queues.AddClient("orders", s => { s.ConnectionString = "UseDevelopmentStorage=true"; s.QueueName = "orders"; })); + using var provider = services.BuildServiceProvider(); + + provider.Invoking(p => p.GetRequiredKeyedService("ordres")).Should().Throw().WithMessage("No queue client named 'ordres'*'orders'*"); + provider.Invoking(p => p.GetRequiredService()).Should().Throw().WithMessage("No default queue client*'orders'*"); + } + + [Fact(DisplayName = "Processor options and the SDK's ClientOptions bind from configuration")] + public void Configuration_Binding() + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Queues:Orders:QueueName"] = "orders", + ["Queues:Orders:ConnectionString"] = "UseDevelopmentStorage=true", + ["Queues:Orders:ClientOptions:MessageEncoding"] = "Base64", + ["Queues:Orders:PoisonTimeToLive"] = "30.00:00:00", + ["Queues:Orders:Processor:MaxConcurrentMessages"] = "7", + ["Queues:Orders:Processor:MaxRenewalDuration"] = "00:10:00", + }).Build(); + var services = new ServiceCollection(); + services.AddAzureStorageQueueClient(queues => queues + .AddClient("orders", configuration.GetSection("Queues:Orders")) + .AddProcessor(configuration.GetSection("Queues:Orders:Processor"), o => o.MaxDequeueCount = 9)); + using var provider = services.BuildServiceProvider(); + + var settings = Settings(provider, "orders"); + settings.ClientOptions!.MessageEncoding.Should().Be(QueueMessageEncoding.Base64); + settings.PoisonTimeToLive.Should().Be(TimeSpan.FromDays(30)); + var options = provider.GetRequiredService>().Create($"orders|{typeof(Handler).FullName}"); + options.Should().BeEquivalentTo(new { MaxConcurrentMessages = 7, MaxRenewalDuration = TimeSpan.FromMinutes(10), MaxDequeueCount = 9 }); + } +} From 794b0058ebe4f0d6f045fd1d24f6626b386e47b5 Mon Sep 17 00:00:00 2001 From: Jason Shave Date: Fri, 16 Oct 2026 14:46:35 -0600 Subject: [PATCH 8/8] WIP: docs --- README.md | 382 ++++++++++++++++++++++++++++++++++++++-------- docs/v3-design.md | 18 ++- 2 files changed, 334 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index c6f3439..f92c96c 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,21 @@ Typed sending, receiving and processing for Azure Storage queues on .NET 10 and Still on .NET 8? Use the 2.x package line, which stays on .NET 8 and keeps receiving fixes. Its documentation is in the [2.x README](https://github.com/jasonshave/AzureStorage.QueueService/tree/2.x#readme). -- **Send** with `IQueueSender`: your object goes in, JSON goes on the queue. +- **Send** with `IQueueSender`: your object goes in, JSON goes on the queue, with optional headers, a delay and compression. - **Receive** with `IQueueReceiver`: each message comes back typed, with its identity, and you complete, abandon or dead-letter it. -- **Process** with `AddProcessor()`: the library runs the loop — backoff, lease renewal, retries, a poison queue, cancellation and graceful shutdown — and hands each message to a handler class resolved from its own dependency-injection scope. +- **Process** with `AddProcessor()`: the library runs the loop — backoff, lease renewal, retries, a poison queue, cancellation and graceful shutdown — and hands each message to a handler resolved from its own dependency-injection scope. Handlers can be classes or lambdas, share middleware, and share one queue between several message types. +- **Look after the poison queue** with `IDeadLetterQueue`: see what was dead-lettered and why, and move messages back once the cause is fixed. -Upgrading from 2.x? 3.0 is a clean break — see [Upgrading from 2.x](#upgrading-from-2x) for the mapping. +Upgrading? See [Upgrading from 3.0 to 3.1](#upgrading-from-30-to-31) for the behaviour changes, or [Upgrading from 2.x](#upgrading-from-2x) for the mapping from the old API. ## Install -Add the NuGet package [`AzureStorage.QueueService`](https://www.nuget.org/packages/AzureStorage.QueueService). +Add the NuGet package [`AzureStorage.QueueService`](https://www.nuget.org/packages/AzureStorage.QueueService). Two optional packages build on it: + +| Package | Adds | +|---|---| +| [`AzureStorage.QueueService.Blobs`](https://www.nuget.org/packages/AzureStorage.QueueService.Blobs) | [Messages over 64 KiB](#messages-over-64-kib), kept in Blob storage | +| [`AzureStorage.QueueService.HealthChecks`](https://www.nuget.org/packages/AzureStorage.QueueService.HealthChecks) | ASP.NET Core [health checks](#health-checks) for processors and queues | You need an Azure Storage account and either its connection string or a queue URI plus an identity that can read and write the queue. The library can create the queue for you (`CreateIfNotExists`); the processor also creates a `-poison` queue the first time it dead-letters a message. @@ -64,18 +70,28 @@ builder.Services.AddAzureStorageQueueClient(queues => | Setting | What it does | |---|---| | `ConnectionString` + `QueueName` | Connect with a storage account connection string. | -| `EndpointUri` + `TokenCredential` | Connect with an identity instead. The URI includes the queue name; a `QueueName` that says something different is rejected. | +| `EndpointUri` + `TokenCredential` | Connect with an identity instead. `EndpointUri` is either the queue's URI (`https://account.queue.core.windows.net/orders`) or the account's endpoint (`https://account.queue.core.windows.net/`) with `QueueName`. A `QueueName` that contradicts the queue in the URI is rejected. | | `CreateIfNotExists` | Create the queue on first use if it does not exist. Happens on the first operation, not at startup. | | `VisibilityTimeout` | How long a message received with `ReceiveAsync` stays hidden when the call does not say. Default 30 seconds. A processor uses its own setting. | | `PoisonQueueName` | Where dead-lettered messages go. Default: the queue name plus `-poison`. Set it when the queue name is longer than 56 characters (Azure's limit is 63) or to share one poison queue. | -| `ClientOptions` | `QueueClientOptions` for the Azure SDK client: retry policy, diagnostics, and `MessageEncoding`. | -| `PropagateTraceContext` | Wrap each sent message in an envelope that carries the current trace, so the processor's span joins the sender's trace. Off by default; see [Telemetry](#telemetry) before turning it on. | +| `PoisonTimeToLive` | How long a dead-lettered message is kept. Default: for ever, until you delete or replay it. | +| `ClientOptions` | `QueueClientOptions` for the Azure SDK client: retry policy, diagnostics, `MessageEncoding`, client-side encryption. | +| `PropagateTraceContext` | Carry the current trace with each sent message, so the processor's span joins the sender's trace. Off by default; see [what travels with a message](#what-travels-with-a-message). | +| `CompressionThreshold` | Gzip bodies of at least this many bytes. Off by default; set with `UseCompression()`. | +| `SendMessageType` | Send each message's type as a header, for a [routed processor](#several-message-types-on-one-queue). | +| `LargeMessageStore` / `LargeMessageThreshold` | Where bodies over 64 KiB go; see [Messages over 64 KiB](#messages-over-64-kib). | + +Registering the same name twice adds to its configuration. `AddClient` and `AddDefaultClient` return a builder for that one client with `Configure(...)`, `UseSerializer(...)`, `UseJsonSerializer(...)`, `UseCompression(...)`, `UseLargeMessageStore(...)`, `UseMiddleware()`, and the processor registrations; other packages add their own extension methods to it. + +Everything a setting can hold in configuration binds from it, `ClientOptions` included: -Registering the same name twice adds to its configuration. `AddClient` and `AddDefaultClient` return a builder for that one client with `Configure(...)`, `UseSerializer(...)`, `UseJsonSerializer(...)`, `AddProcessor<,>(...)` and `AddCloudEventProcessor<,>(...)`; other packages can add their own extension methods to it. +```json +{ "Queues": { "Legacy": { "ConnectionString": "...", "QueueName": "legacy", "ClientOptions": { "MessageEncoding": "Base64" } } } } +``` -### One identity for every client +### One storage account and identity for every client -A credential can't go in `appsettings.json`, so set it once in code. `UseTokenCredential` gives it to every client that has an `EndpointUri` but no `TokenCredential` of its own, default and named, registered before or after the call; configuration then only needs the endpoint. A client's own `TokenCredential` still wins, clients with a `ConnectionString` are left alone, and calling it again replaces the credential. `DefaultAzureCredential` comes from the `Azure.Identity` package, which your app references; this library doesn't. +A credential can't go in `appsettings.json`, so set it once in code. `UseTokenCredential` gives it to every client that has an `EndpointUri` but no `TokenCredential` of its own. `UseStorageAccount` does the same for the account itself: every client with neither a `ConnectionString` nor an `EndpointUri` connects to it, so configuration only needs queue names. Both apply to default and named clients registered before or after the call; a client's own settings still win; calling either again replaces what it set. `DefaultAzureCredential` comes from the `Azure.Identity` package, which your app references; this library doesn't. ```csharp builder.Services.AddAzureStorageQueueClient(queues => @@ -83,19 +99,24 @@ builder.Services.AddAzureStorageQueueClient(queues => queues.AddDefaultClient(builder.Configuration.GetSection("Queues:Notifications")); queues.AddClient("orders", builder.Configuration.GetSection("Queues:Orders")); - queues.UseTokenCredential(new DefaultAzureCredential()).ValidateOnStart(); + queues.UseStorageAccount(builder.Configuration.GetConnectionString("queues")!) // a connection string or https://account.queue.core.windows.net/ + .UseTokenCredential(new DefaultAzureCredential()) + .ValidateOnStart(); }); ``` ```json { + "ConnectionStrings": { "queues": "https://myaccount.queue.core.windows.net/" }, "Queues": { - "Notifications": { "EndpointUri": "https://myaccount.queue.core.windows.net/notifications" }, - "Orders": { "EndpointUri": "https://myaccount.queue.core.windows.net/orders", "CreateIfNotExists": true } + "Notifications": { "QueueName": "notifications" }, + "Orders": { "QueueName": "orders", "CreateIfNotExists": true } } } ``` +That is the shape .NET Aspire hands an app: `ConnectionStrings:` holds the account's queue endpoint (or, with the emulator, a connection string), and `UseStorageAccount` takes either. + ### Queues shared with Azure Functions or older SDKs Those write Base64-encoded messages. Tell the SDK, and everything else is unchanged. A message on such a queue that is *not* valid Base64 does not fail the receive: it comes back with `Error` set alongside the healthy messages, so it can be dead-lettered like any other bad message. @@ -109,9 +130,25 @@ queues.AddClient("legacy", settings => }); ``` +Those senders usually write camelCase JSON. The library's default serializer reads property names case-insensitively, so `{"orderId": 7}` fills `OrderId`. It still writes PascalCase; use `UseJsonSerializer(new JsonSerializerOptions(JsonSerializerDefaults.Web))` to write camelCase too. + +### Client-side encryption + +The SDK's client-side encryption works as it is: give the client `SpecializedQueueClientOptions` with `ClientSideEncryption` set, and every message is encrypted before it leaves the process and decrypted on receive. Encryption makes each message bigger, which the library's 64 KiB check before sending does not account for, so leave headroom. + +```csharp +settings.ClientOptions = new SpecializedQueueClientOptions +{ + ClientSideEncryption = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V2_0) + { + KeyEncryptionKey = key, KeyResolver = resolver, KeyWrapAlgorithm = "RSA-OAEP", + }, +}; +``` + ## Get a client -Inject the default client as `IStorageQueue` (everything), `IQueueSender` (send only) or `IQueueReceiver` (receive only). Named clients are keyed services: +Inject the default client as `IStorageQueue` (everything), `IQueueSender` (send only), `IQueueReceiver` (receive only) or `IDeadLetterQueue` (its poison queue). Named clients are keyed services: ```csharp public class OrderService( @@ -121,16 +158,34 @@ public class OrderService( } ``` -Or ask the factory: `IQueueClientFactory.GetQueue()` / `GetQueue("orders")`. Clients are created once and shared for the life of the process; the SDK client underneath is thread-safe. +Or ask the factory: `IQueueClientFactory.GetQueue()` / `GetQueue("orders")`, and `GetDeadLetterQueue("orders")`. Clients are created once and shared for the life of the process; the SDK client underneath is thread-safe. Asking for a name nobody registered — a typo in `[FromKeyedServices]`, or the default client when only named ones exist — fails with a message listing the clients that are registered. ## Send ```csharp await notifications.SendAsync(new Notification("hello")); -await notifications.SendAsync(new Notification("later"), new SendOptions(VisibilityTimeout: TimeSpan.FromMinutes(5))); +await notifications.SendAsync(new Notification("later"), new SendOptions { Delay = TimeSpan.FromMinutes(5) }); +await notifications.SendAsync(new Notification("tagged"), new SendOptions +{ + Headers = new Dictionary { [QueueMessageHeaders.CorrelationId] = correlationId }, +}); ``` -The return value is the SDK's `SendReceipt`. Failures are logged and thrown. +`SendOptions` takes a `Delay` (also called `VisibilityTimeout`: 0 to 7 days), a `TimeToLive` (`Timeout.InfiniteTimeSpan` for never; the service default is 7 days), `Headers`, and a `MessageType`. A delay the message would expire within, or a message over the service's 64 KiB limit, throws before anything is sent: `ArgumentOutOfRangeException` and `QueueMessageTooLargeException` respectively. The size is checked as the message would go on the queue, so Base64 encoding and the library's envelope count. The return value is the SDK's `SendReceipt`. Failures are logged and thrown. + +`UseCompression()` on a client gzips bodies of 1 KiB or more (pass another threshold if you like), skipping any message compression would not make smaller. Repetitive JSON often shrinks by 80% or more, which also keeps more messages under the 64 KiB limit. + +### What travels with a message + +A storage queue message is only a body: no properties, no headers. Anything that has to travel with it — the trace context, headers, a compressed body, a body kept in Blob storage — goes in a small JSON envelope around the body. A message that needs none of these goes out exactly as the serializer wrote it, as in 3.0. + +```json +{"$azqs-envelope":1,"traceparent":"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01","headers":{"tenant":"contoso"},"body":{"OrderId":42}} +``` + +Every client from 3.1 on reads the envelope when it receives, whatever its own settings, so `Body` and `RawBody` are the message you sent and `Headers` holds the headers. The full format is under [Wire format](#wire-format). The catch is older readers: a consumer on 3.0 or earlier, or other code reading the queue (an Azure Function, a script), sees the envelope as the message and fails to deserialize it, or worse, deserializes it into an object with empty fields. So headers, compression, `PropagateTraceContext` and `SendMessageType` are all off until you use them; turn them on only once every reader of the queue is on 3.1 or later. + +**The envelope makes a message bigger:** roughly 100 bytes for the trace context, plus the headers, and binary data (a compressed body, or a custom serializer's non-JSON output) is Base64-encoded, a third bigger. A message close to the 64 KiB limit can tip over it; the send then throws rather than going out plain. ## Process messages (recommended) @@ -155,29 +210,127 @@ builder.Services.AddAzureStorageQueueClient(queues => })); ``` -A processor is `MaxConcurrentMessages` workers; each receives one message, delivers it, and goes back for the next. What it does for you: +Or as a lambda: parameters of the message type get the body, `QueueMessage` or `IQueueMessage` the message, `CancellationToken` the shutdown token, and anything else is resolved from the message's scope (`[FromKeyedServices]` works too). Give every parameter a type. -- **Polls with backoff.** An idle worker polls at a growing interval (1 s doubling to 30 s by default) that resets as soon as a message arrives. A failed receive — storage down, the queue missing, no permission — is logged and backed off the same way. Nothing at runtime stops the host. -- **Keeps the lease alive.** While your handler runs, the message's visibility timeout is renewed, which closes the avoidable window where a slow handler lets another consumer take the same message. Delivery is still at-least-once — a lease can lapse, a completion can fail after the work is done — so handlers should tolerate seeing a message twice. -- **Retries with a delay.** A handler that throws abandons the message with a growing delay (5 s, 10 s, 20 s, ... up to 5 minutes), logged with the message id and attempt count. A handler whose constructor throws is treated the same way, not swallowed. -- **Dead-letters poison messages.** A message delivered more than `MaxDequeueCount` times, or one whose body cannot be decoded or deserialized, is moved to the poison queue — body unchanged, so it can be moved straight back to replay it — without calling the handler. If that move fails, the message is abandoned instead, never dropped. +```csharp +queues.AddClient("orders", builder.Configuration.GetSection("Queues:Orders")) + .AddProcessor(async (Order order, OrdersDbContext db, CancellationToken ct) => await db.Fulfil(order, ct)); +``` + +What the processor does for you: + +- **Takes only what it can handle.** One receive loop per processor asks for as many messages as it has free handlers (up to 32 per request, at most `MaxConcurrentMessages` in flight) and starts each on its own task with its own scope. No message waits behind another's handler, and the processor never holds a message no handler is free for. When messages arrive it asks again straight away; an empty queue is polled at a growing interval (1 s doubling to 30 s by default) that resets as soon as a message arrives. A failed receive — storage down, the queue missing, no permission — is logged once and backed off the same way. Nothing at runtime stops the host. +- **Keeps the lease alive.** While your handler runs, the message's visibility timeout is renewed, which closes the avoidable window where a slow handler lets another consumer take the same message. Delivery is still at-least-once — a lease can lapse, a completion can fail after the work is done — so handlers should tolerate seeing a message twice. `MaxRenewalDuration` caps the renewal for a handler that may hang; a handler running longer than `LongRunningHandlerWarning` (10 minutes) is logged once. +- **Retries with a delay.** A handler that throws abandons the message with a growing delay (5 s, 10 s, 20 s, ... up to 5 minutes). That is a warning in the log while retries remain and an error on the last attempt; `message.IsLastAttempt` tells the handler which it is on. A handler whose constructor throws is treated the same way, not swallowed. An [error policy](#error-policy) can dead-letter or complete instead. +- **Dead-letters poison messages.** A message delivered more than `MaxDequeueCount` times, or one whose body cannot be read, is moved to the poison queue without calling the handler, with a record of why. If that move fails, the message is abandoned instead, never dropped. See [The poison queue](#the-poison-queue). - **Lets the handler take over.** Call `message.CompleteAsync()`, `AbandonAsync()` or `DeadLetterAsync()` yourself and the processor leaves it alone; `message.Settlement` says what happened. - **Shuts down cleanly.** On host shutdown it stops receiving, gives running handlers `ShutdownDrainTimeout` (30 s) to finish, then cancels their token and stops renewing their leases. A handler that ignores its token gets the same time again, after which shutdown stops waiting for it. All of that happens even if the host's own shutdown deadline cuts the drain short. Unfinished messages become visible again when their lease lapses. - **Starts safely.** A processor whose client is misconfigured fails host startup, on purpose. Polling begins only when the host reports that the whole application has started, so a slow startup does not hold leases and a startup that fails part-way never polls at all. | Option | Default | What it does | |---|---|---| -| `MaxConcurrentMessages` | 1 | Workers, each handling one message at a time in its own scope. | +| `MaxConcurrentMessages` | 1 | Messages handled at once, each in its own scope. | | `VisibilityTimeout` | 1 minute | How long each received message is hidden (1 second to 7 days, whole seconds); renewed while the handler runs. | | `RenewVisibility` | true | Renew the lease at half the timeout while the handler runs. | +| `MaxRenewalDuration` | none | Stop renewing after this long, so a hung handler's message is redelivered. | +| `LongRunningHandlerWarning` | 10 minutes | Log a warning, once, for a handler still running after this long. | | `AutoComplete` | true | Complete on return, abandon on throw. Off, the handler must settle every message. | | `MaxDequeueCount` | 5 | Dead-letter a message delivered more times than this. | | `DeadLetterOnDeserializationFailure` | true | Dead-letter a message whose body cannot be read. | | `RetryDelay` | 5 s × 2ⁿ, max 5 min | How long an abandoned message stays hidden before the next attempt. | +| `DeadLetterOn()`, `OnHandlerError` | none | The [error policy](#error-policy). | +| `RateLimiter` | none | A `System.Threading.RateLimiting` limiter: one permit per message, taken before each receive. | | `EmptyQueueBackoffMin` / `Max` | 1 s / 30 s | Polling interval on an empty queue. | | `ErrorBackoffMax` | 1 minute | Longest wait after a failed receive. | | `ShutdownDrainTimeout` | 30 s | How long shutdown waits for running handlers. | +Options can come from configuration, so they can differ by environment: `AddProcessor(builder.Configuration.GetSection("Queues:Orders:Processor"), options => ...)`. Configuration is applied first, then the delegate; the callbacks and the rate limiter are set in code. Registering the same handler on the same client again adds to its options rather than starting a second processor. + +While a message is being delivered, everything logged — by the library and by your handler — carries a scope with the queue name, message id and dequeue count, for log sinks that don't correlate by trace id. + +### The message + +`QueueMessage` carries `MessageId`, `QueueName`, `PopReceipt`, `DequeueCount`, `InsertedOn`, `ExpiresOn`, `NextVisibleOn`, `Headers`, `IsLastAttempt`, the `RawBody`, the typed `Body`, and `Error` when the body could not be read. `CompleteAsync` deletes it; `AbandonAsync(delay)` makes it visible again now or later; `RenewLockAsync(timeout)` extends the lease; `DeadLetterAsync(reason)` moves it to the poison queue; `Settlement` records which of those happened. Timeouts the service would refuse throw `ArgumentOutOfRangeException` before any call. `IQueueMessage` is the same message without its body's type, for code that serves every message type. + +### Error policy + +Some failures are not worth retrying. Tell the processor which, instead of wrapping every handler in a try/catch: + +```csharp +.AddProcessor(options => +{ + options.DeadLetterOn(); // straight to the poison queue, no retries + options.OnHandlerError = context => context.Exception switch + { + DuplicateOrderException => HandlerErrorAction.Complete, // already done: delete it + _ => HandlerErrorAction.Retry, + }; +}); +``` + +`DeadLetterOn` rules are checked first, then `OnHandlerError`; with neither, the message is retried. A callback that throws counts as `Retry`. + +### Middleware + +Middleware runs around every message, the way ASP.NET Core middleware runs around a request: idempotency checks, timing, a circuit breaker, a tenant scope — anything every handler would otherwise repeat. It is resolved from the message's scope, so it can take scoped services. + +```csharp +public class IdempotencyMiddleware(ProcessedMessages processed) : IQueueMessageMiddleware +{ + public async ValueTask InvokeAsync(QueueMessageContext context, QueueMessageDelegate next) + { + if (await processed.ContainsAsync(context.Message.MessageId)) return; // handled already: complete it + await next(context); + await processed.AddAsync(context.Message.MessageId); + } +} + +builder.Services.AddAzureStorageQueueClient(queues => +{ + queues.UseMiddleware(); // every client's processors + queues.AddClient("orders", ...).UseMiddleware().AddProcessor(); +}); +``` + +Middleware runs in the order registered. Not calling `next` stops there, and the message counts as handled (completed, with `AutoComplete` on). An exception counts as a handler failure, error policy included. `context.Items` passes things along; `context.Services` is the message's scope. + +### Several message types on one queue + +Two processors reading one queue take each other's messages and fail to read them. To share a queue, route by type: the sender sends each message's type as a header, and one processor hands each message to the handler for its type. + +```csharp +// Sending side: every message carries its type (typeof(T).Name), or set SendOptions.MessageType per message +queues.AddClient("orders", settings => { ...; settings.SendMessageType = true; }); + +// Receiving side +queues.AddClient("orders", ...) + .AddRoutedProcessor(routes => routes + .Route() // type "OrderCreated" + .Route("cancelled")); +``` + +A message with no type, or one nothing is routed for, is dead-lettered as `unroutable`. If you can't change the senders, `System.Text.Json`'s polymorphism works with a plain processor, as long as the sender sends the base type: + +```csharp +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(OrderCreated), "created")] +[JsonDerivedType(typeof(OrderCancelled), "cancelled")] +public abstract record OrderEvent; + +await sender.SendAsync(new OrderCreated(42)); // writes "$type":"created" +queues.AddClient("orders", ...).AddProcessor(); // switch on message.Body +``` + +### Pausing a processor + +`IQueueProcessorController` pauses and resumes a client's processors — during a downstream outage, say — and reports how each one is doing (`GetStatus()`: state, handlers running, receive failures in a row, the oldest handler's age). A paused processor stops receiving; what it already holds is delivered and settled as usual. The default client's name is the empty string. + +```csharp +controller.Pause("orders"); +// ... +controller.Resume("orders"); +``` + ### Event Grid and other CloudEvents When the queue is fed by an Event Grid subscription that uses the CloudEvents schema (or anything else that writes [CloudEvents](https://cloudevents.io) as JSON), register an `ICloudEventHandler` instead. The handler gets the event's `data` already read as `TData`, and the event itself as `message.Body`, for its type, source, id, subject and time: @@ -206,57 +359,94 @@ builder.Services.AddAzureStorageQueueClient(queues => .AddCloudEventProcessor()); ``` +A subscription that delivers several event types goes to a router, which picks the handler by the event's `type`: + +```csharp +.AddCloudEventRouter(routes => routes + .Route("Contoso.Orders.Created") + .Route("Contoso.Orders.Shipped")); +``` + It is the same processor underneath, with the same options and behaviour: return to complete, throw to retry, settle through `message` yourself if you want to. A few things are specific to events: - **Base64.** Event Grid's storage queue destination Base64-encodes each message by default. Set `MessageEncoding = QueueMessageEncoding.Base64` as above, or turn off Base64 encoding on the event subscription and leave the client as it is. - **The data is read with this client's serializer,** so its naming policy applies. An event with no data reaches the handler with `data` null. -- **An event whose data cannot be read as `TData`** is treated like any other unreadable message: moved to the poison queue without calling the handler, with the event id and type in the reason. With `DeadLetterOnDeserializationFailure` off, the handler is called with `data` null and `message.Error` set. -- **Binary data** (`data_base64`, or any data that is not JSON): use `BinaryData` as `TData` to get the bytes as they are. Any other `TData` is read with the client's serializer, so data that is not JSON is treated as unreadable. +- **An event whose data cannot be read as `TData`** is treated like any other unreadable message: moved to the poison queue without calling the handler, with the event id and type in the reason. With `DeadLetterOnDeserializationFailure` off, the handler is called with `data` null and `message.Error` set. A router dead-letters an event of a type nothing is routed for. +- **Binary data** (`data_base64`, or any data that is not JSON): use `BinaryData` as `TData` to get the bytes as they are. - **Event Grid's own schema** (`EventGridEvent`, Event Grid's default schema) is not supported; it needs the `Azure.Messaging.EventGrid` package. Choose the CloudEvents v1.0 schema when you create the subscription. `AddProcessor()` also works, if you would rather read `message.Body.Data` yourself. ## Receive in your own loop -If you want the loop, inject `IQueueReceiver` and own it. You get every message's identity and settle it yourself; backoff, renewal, retries and shutdown are yours to handle, and a failed receive throws so your loop can react. +If you want the loop, inject `IQueueReceiver` and own it. `ReadAllAsync` does the polling — a growing wait while the queue is empty or after a failed receive — and hands you messages one at a time until the token is cancelled; settling each one, renewal, retries and dead-lettering are yours. ```csharp public class OrderPoller(IQueueReceiver receiver, ILogger logger) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - while (!stoppingToken.IsCancellationRequested) + var options = new ReadAllOptions { OnReceiveError = ex => logger.LogError(ex, "Receiving orders failed.") }; + await foreach (var message in receiver.ReadAllAsync(options, stoppingToken)) { - try + if (message.Error is not null) { - foreach (var message in await receiver.ReceiveAsync(maxMessages: 10, cancellationToken: stoppingToken)) - { - if (message.Error is not null) - { - await message.DeadLetterAsync(message.Error.Message, stoppingToken); - continue; - } - - await Fulfil(message.Body!); - await message.CompleteAsync(); - } + await message.DeadLetterAsync(message.Error.Message); + continue; } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } - catch (Exception ex) { logger.LogError(ex, "Polling failed."); } - await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + await Fulfil(message.Body!); + await message.CompleteAsync(); } } } ``` -`QueueMessage` carries `MessageId`, `PopReceipt`, `DequeueCount`, `InsertedOn`, `ExpiresOn`, `NextVisibleOn`, the `RawBody`, the typed `Body`, and `Error` when the body could not be decoded or deserialized. `CompleteAsync` deletes it; `AbandonAsync(delay)` makes it visible again now or later; `RenewLockAsync(timeout)` extends the lease; `DeadLetterAsync(reason)` moves it to the poison queue; `Settlement` records which of those happened. `PeekAsync` returns `PeekedQueueMessage`, which has no pop receipt and so nothing to settle. +`ReceiveAsync(maxMessages, visibilityTimeout)` is underneath, for one receive at a time. `PeekAsync` returns `PeekedQueueMessage`, which has no pop receipt and so nothing to settle. + +## The poison queue + +A dead-lettered message is copied to the poison queue (`-poison` unless `PoisonQueueName` says otherwise) with a record of why, then deleted from its queue. It stays there until you delete or replay it; set `PoisonTimeToLive` to expire it instead. `IDeadLetterQueue` reads the record and moves messages back: + +```csharp +public class PoisonAdmin([FromKeyedServices("orders")] IDeadLetterQueue poison) +{ + public async Task RetryOutageFailures() + { + foreach (var message in await poison.PeekAsync()) + Console.WriteLine($"{message.OriginalMessageId}: {message.Category} — {message.Reason} (after {message.DequeueCount} attempts, {message.DeadLetteredOn})"); + + // Back to the orders queue, as they were sent, with a fresh dequeue count + return await poison.ReplayAsync(filter: m => m.Category == DeadLetterCategories.Handler && m.Reason!.Contains("TimeoutException")); + } +} +``` + +Each record has the reason, the category (`max_dequeue`, `deserialization`, `handler` or `unroutable`), the original message id, the dequeue count, the source queue and the time. `GetBody()` reads a dead-lettered body with the source client's serializer. A replayed message goes back exactly as it was sent — trace context, headers and all — without the record. A message the library could not read at all (a later version's format, say) is kept exactly as it arrived, so a replay after upgrading reads it. + +## Messages over 64 KiB + +The service rejects a queue message over 64 KiB. Add the `AzureStorage.QueueService.Blobs` package and a client keeps bodies that big in Azure Blob storage instead: the body is uploaded to a blob, and the queue message carries only the blob's address (the claim-check pattern). Processors and receivers download the body before your handler sees it, so handlers don't change. The blob is deleted once the message is completed, and kept when it is dead-lettered so a replay still has its body. + +```csharp +queues.AddClient("orders", builder.Configuration.GetSection("Queues:Orders")) + .UseBlobStorageForLargeMessages(); // container "queue-large-messages" in the queue's own account +``` + +With no arguments it uses the queue's own storage account and credentials: a connection string as is, or `https://myaccount.queue.core.windows.net/...` becomes `https://myaccount.blob.core.windows.net/`. You can also pass a container name, a `BlobContainerClient`, or `sp => ...` to build the container client from your services. The container is created the first time a large message is sent. Pass `thresholdBytes` to send smaller bodies to Blob storage as well. Any other store can plug in by implementing `ILargeMessageStore` and calling `UseLargeMessageStore`. + +Things to know before turning it on: + +- **Every client that reads the queue needs the same store.** One without it can't read those messages and dead-letters them. Large messages travel in the library's envelope, which only 3.1 and later read. +- **Some blobs are left behind.** A message that expires on the queue or is cleared from it is never processed, so its blob stays; a send that fails after the upload deletes its blob only on a best-effort basis. Add a [lifecycle management rule](https://learn.microsoft.com/azure/storage/blobs/lifecycle-management-overview) that deletes blobs in the container older than the queue's message time to live plus a margin (for example 9 days for the default 7), longer if you replay dead-lettered messages later. +- **Permissions:** the identity needs **Storage Blob Data Contributor** on the container or the account. One that can't create containers needs the container created beforehand and `UseLargeMessageStore(new BlobLargeMessageStore(container, createContainerIfNotExists: false))`. +- **Messages are untrusted input.** The store only reads and deletes blobs in its own container; a message pointing anywhere else is dead-lettered without a request being made. A SAS token on the container client is never written to the queue. ## Testing your code -Depend on `IQueueSender` and `IQueueReceiver` and fake them. A received message can be built in a test: `QueueMessage` has a public constructor that takes a `QueueMessageInfo`, the body and an `IQueueMessageOperations` — stub the operations to assert that your code completed or abandoned the message. +Depend on `IQueueSender` and `IQueueReceiver` and fake them. A received message can be built in a test: `QueueMessage` has a public constructor that takes a `QueueMessageInfo` (whose optional `QueueName`, `Headers` and `IsLastAttempt` set those properties), the body and an `IQueueMessageOperations` — stub the operations to assert that your code completed or abandoned the message. -To test a handler end to end — send, processor, handler, then complete, retry or dead-letter — swap the real queues for in-memory ones. No storage account, emulator or connection string is needed, even with `ValidateOnStart()`; each client still needs a `QueueName`, or an `EndpointUri` that names the queue, checked by the same rules as in production. Messages go through each client's serializer and follow the service's rules: a received message is hidden for its visibility timeout and its dequeue count goes up, an abandoned one stays hidden for its retry delay, and settling with an outdated pop receipt throws a `RequestFailedException` (404). Time comes from the registered `TimeProvider`, so a `FakeTimeProvider` drives polling, retries and expiry without real waits. Clients naming the same queue share it, so to see what was dead-lettered, register a client for the poison queue and peek it: +To test a handler end to end — send, processor, handler, then complete, retry or dead-letter — swap the real queues for in-memory ones. No storage account, emulator or connection string is needed, even with `ValidateOnStart()`; each client still needs a `QueueName`, or an `EndpointUri` that names the queue, checked by the same rules as in production. The in-memory queues run the same queue code as the real ones, so messages go through each client's serializer, envelope and size check, emit the same spans and metrics, and follow the service's rules: a received message is hidden for its visibility timeout and its dequeue count goes up, an abandoned one stays hidden for its retry delay, and settling with an outdated pop receipt throws a `RequestFailedException` (404). Time comes from the registered `TimeProvider`, so a `FakeTimeProvider` drives polling, retries and expiry without real waits. Clients naming the same queue share it. ```csharp var time = new FakeTimeProvider(); @@ -264,11 +454,8 @@ var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSetti builder.Services.AddSingleton(time); builder.Services.AddSingleton(fulfilment); // a fake the handler calls, which the test can await builder.Services.AddAzureStorageQueueClient(queues => -{ queues.AddClient("orders", settings => settings.QueueName = "orders") - .AddProcessor(); - queues.AddClient("orders-poison", settings => settings.QueueName = "orders-poison"); -}); + .AddProcessor()); builder.Services.AddInMemoryQueues(); // before or after the registration; with WebApplicationFactory, in ConfigureTestServices using var host = builder.Build(); @@ -279,12 +466,28 @@ await host.StopAsync(); // lets the delivery finish settling var orders = host.Services.GetRequiredKeyedService("orders"); var remaining = (await orders.GetPropertiesAsync()).ApproximateMessagesCount; // 0 once completed -var poisoned = await host.Services.GetRequiredKeyedService("orders-poison").PeekAsync(32); +var poisoned = await host.Services.GetRequiredKeyedService("orders").PeekAsync(); ``` -To drive retries, advance the clock while the processor waits on it: a failed message stays hidden for its retry delay (5 s, then 10 s, ...), and an idle worker polls again after `EmptyQueueBackoffMin`. An advance made before the processor has started waiting does not wake it, so step the clock in time with the polls rather than in one jump. +Messages sent before the host starts are picked up by the processor's first receive, so the example needs no clock at all. To drive retries, advance the clock while the processor waits on it: a failed message stays hidden for its retry delay (5 s, then 10 s, ...), and an idle processor polls again after `EmptyQueueBackoffMin`. An advance made before the processor has started waiting does not wake it, so step the clock in time with the polls rather than in one jump. + +The in-memory queues differ from the service in a few ways: every queue exists without being created, and `ClientOptions` is ignored apart from `MessageEncoding`, which counts toward the size check as it would in production. -The in-memory queues differ from the service in a few ways: every queue exists without being created, `ClientOptions` (including Base64 `MessageEncoding`) is ignored, the 64 KiB message size limit is not enforced, and sends and receives emit no spans or metrics of their own (the processor's still do). `PropagateTraceContext` works as it does against the service, carrying the caller's current span since there is no `Queue.Send` span. +## Health checks + +The `AzureStorage.QueueService.HealthChecks` package adds two ASP.NET Core health checks. + +```csharp +builder.Services.AddHealthChecks() + .AddAzureStorageQueueProcessors(tags: ["live"]) + .AddAzureStorageQueue("orders", tags: ["ready"], configure: o => o.DegradedWhenPoisonMessages = true); + +app.MapHealthChecks("/health/live", new() { Predicate = c => c.Tags.Contains("live") }); +app.MapHealthChecks("/health/ready", new() { Predicate = c => c.Tags.Contains("ready") }); +``` + +- **`AddAzureStorageQueueProcessors`** reports on the hosted processors from `IQueueProcessorController.GetStatus()`. It makes no storage calls, so it is free to run on every probe. It is **Unhealthy** when a processor's last 3 receives all failed (`UnhealthyAfterReceiveFailures`), and **Degraded** when a processor is paused (`PausedStatus`) or a message has been in its handler longer than `MaxHandlerAge` (off by default). A processor still waiting for the host to start counts as healthy. +- **`AddAzureStorageQueue(clientName)`** checks the queue can be reached by reading its properties. With `DegradedWhenPoisonMessages` it also reports **Degraded** while the poison queue holds messages. **Each run is a billable storage call** (two with the poison option), so keep it off high-frequency probes. A run longer than `Timeout` (10 s) counts as failed. ## Telemetry @@ -296,25 +499,76 @@ builder.Services.AddOpenTelemetry() .WithMetrics(m => m.AddAzureStorageQueueMetrics()); ``` -The processor emits a `Queue.Process` span (kind: consumer) per message, tagged with the queue name, message id, dequeue count and outcome. `Queue.Read` (client) and `Queue.Send` (producer) spans for individual receives and sends are off by default because they are noisy; turn them on with `ConfigureQueueServiceTelemetry`. A span whose operation failed is marked with an error status and `error.type`. +The processor emits a `Queue.Process` span (kind: consumer) per message, tagged with the queue name, message id, dequeue count and outcome, with events when the message is renewed, completed, abandoned or dead-lettered. `Queue.Read` (client) and `Queue.Send` (producer) spans for individual receives and sends are off by default because they are noisy; turn them on with `ConfigureQueueServiceTelemetry`. A span whose operation failed is marked with an error status and `error.type`. Spans also carry the [OpenTelemetry messaging attributes](https://opentelemetry.io/docs/specs/semconv/messaging/) — `messaging.system` (`azure.storage.queue`), `messaging.destination.name`, `messaging.operation.type` / `.name`, `messaging.message.id`, `messaging.batch.message_count`, `server.address` / `server.port` — so App Insights and Grafana's messaging views recognise them. -Metrics, all tagged with `queue.name`: `queue_messages_received_total`, `queue_messages_sent_total`, `queue_messages_processed_total`, `queue_messages_failed_total`, `queue_message_processing_duration_seconds` (also tagged with `queue.outcome`), `queue_message_size_bytes` (the serializer's output, before any trace-context envelope). +**Metrics,** all tagged with `queue.name`: -**Standard attribute names.** Alongside the tags above, spans carry the [OpenTelemetry messaging attributes](https://opentelemetry.io/docs/specs/semconv/messaging/), so App Insights and Grafana's messaging views recognise them: `messaging.system` (`azure.storage.queue`; the conventions define no value for Storage queues), `messaging.destination.name` (the queue), and `messaging.operation.type` / `messaging.operation.name` (`send`, `receive` or `process`). Spans also carry `messaging.message.id` where there is one message, `messaging.batch.message_count` on a receive of more than one, and `server.address` / `server.port` on sends and receives. Span names are unchanged, and metrics keep exactly their names and tags from 3.0, so stored history stays continuous. +| Metric | What it says | +|---|---| +| `queue_messages_received_total`, `queue_messages_sent_total`, `queue_messages_processed_total`, `queue_messages_failed_total` | Counts, as in 3.0. | +| `queue_message_processing_duration_seconds` | Histogram, tagged `queue.outcome`. Buckets from 5 ms to an hour. | +| `queue_message_size_bytes` | Histogram of what the serializer wrote, before any envelope. Buckets up to 64 KiB. | +| `queue_message_time_in_queue_seconds` | Histogram: how long a message waited before a processor started on it, measured from when it was first sent (retries included). The best "are we keeping up" signal, and it costs nothing. | +| `queue_receive_requests_total` | Receive calls, tagged `queue.receive_result`: `message`, `empty` or `error`. Every one is a billable transaction, empty polls included. | +| `queue_processor_active_handlers` | Handlers running now; compare with `MaxConcurrentMessages` for saturation. | +| `queue_processor_oldest_handler_age_seconds` | Gauge: the longest-running handler. A value that keeps growing is a hung handler. | +| `queue_lease_renewals_total`, `queue_lease_renewal_failures_total` | Renewals, and renewals that failed (that message will be redelivered). | +| `queue_messages_dead_lettered_total` | Dead-lettered messages, tagged `queue.dead_letter_reason`. | +| `queue_approximate_messages` | Gauge, tagged `queue.kind` (`main` or `poison`), when `queues.AddQueueDepthMetrics(interval)` is on. Each read is a billable call per queue, so pick the interval with that in mind (default one minute). | -**Metrics in tests.** When the container has an `IMeterFactory` (the generic host registers one; add it yourself with `services.AddMetrics()`), the library creates its meter through it, so a `MetricCollector` scoped to that factory sees exactly that host's measurements. Without one, the library uses a meter of its own, as before. +**Standard metric names (opt-in).** `m.AddAzureStorageQueueMessagingMetrics()` subscribes to a second meter, `AzureStorage.QueueService.Messaging`, with the conventions' own names: `messaging.client.sent.messages`, `messaging.client.consumed.messages`, `messaging.process.duration` and `messaging.client.operation.duration` (sends, receives and settlements). It leaves the metrics above, and their history, untouched. -**One trace from sender to handler (opt-in).** Storage queue messages have no properties or headers, so a trace context can only travel inside the body. Set `PropagateTraceContext = true` on a client and each send wraps the serialized message in a small envelope carrying the W3C trace context: that of the `Queue.Send` span when it is on, otherwise the caller's current span. A message sent with no current span goes out plain. Every client from 3.1 on unwraps the envelope when it receives, whatever its own setting, so `Body` and `RawBody` are the message you sent; plain messages read as before. The processor's `Queue.Process` span links to the sender's context and, because the processor's loop has no span of its own to belong to, also uses it as its parent, so the handler's work appears in the sender's trace. +**Metrics in tests.** When the container has an `IMeterFactory` (the generic host registers one; add it yourself with `services.AddMetrics()`), the library creates its meters through it, so a `MetricCollector` scoped to that factory sees exactly that host's measurements. -```json -{"$azqs-envelope":1,"traceparent":"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01","tracestate":"...","body":{"orderId":42}} -``` +**Exemplars** link a slow duration bucket to the trace that put it there. Turn them on in the OpenTelemetry SDK with `.WithMetrics(m => m.SetExemplarFilter(ExemplarFilterType.TraceBased))`; the processing-duration histogram is recorded inside the `Queue.Process` span, so its exemplars point at that message's trace. + +**Alerts worth having:** + +- *Falling behind:* p95 of `queue_message_time_in_queue_seconds` above what your users tolerate. +- *Poison messages:* `queue_approximate_messages{queue_kind="poison"} > 0`, or any increase in `queue_messages_dead_lettered_total`. +- *Failing handlers:* `queue_messages_failed_total` as a share of `queue_messages_received_total` over 15 minutes. +- *Wasted polls:* the share of `queue_receive_requests_total{queue_receive_result="empty"}`; raise `EmptyQueueBackoffMax` if it is high and latency allows. +- *Lost leases:* any `queue_lease_renewal_failures_total`, which means duplicates are coming. +- *Hung handlers:* `queue_processor_oldest_handler_age_seconds` above your slowest legitimate handler. + +**Scaling with KEDA.** KEDA's `azure-queue` scaler reads the same approximate count as `GetPropertiesAsync`, which includes messages that are hidden — being processed, or waiting out a retry delay. A queue whose messages keep failing therefore keeps its count up and holds replicas that have nothing to receive. Set `queueLength` with that in mind, keep `MaxDequeueCount` modest so failing messages leave for the poison queue, and give the poison queue no scaler. + +## Wire format + +A message is either exactly what the serializer wrote, or an envelope: a JSON object whose first property is the marker `"$azqs-envelope"` with the format version, `1`. The other properties are optional and may come in any order: + +| Property | Holds | +|---|---| +| `traceparent`, `tracestate` | The sender's W3C trace context. | +| `headers` | An object of string headers. | +| `encoding` | `gzip` when the body is compressed. Applies to `body64` or the blob, never to `body`. | +| `deadLetter` | On the poison queue only: `reason`, `category`, `messageId`, `dequeueCount`, `queue`, `at`. | +| `body` / `body64` / `blob` | Exactly one: the serializer's output as is when it is JSON; any other bytes, Base64-encoded; or a reference to a body kept by a large message store. | + +The queue's `MessageEncoding` applies to the whole envelope, as it does to a plain body. The rules that keep different versions from misreading each other: a reader skips any property it does not know, so later 3.x releases can add optional properties without changing the version; anything that changes how the body must be read uses a new `encoding` value, a new kind of body, or a new version number. A reader that meets a version or an encoding it does not know, or an envelope that breaks these rules, does not guess: the message comes back with `Error` set, a processor dead-letters it unchanged, and it can be replayed once the reader is upgraded. + +## Trimming and native AOT + +The library is annotated for trimming and native AOT, and an app using it publishes trimmed without warnings, the Azure SDK included. Two things need doing in such an app: + +- **JSON:** reflection-based serialization is off, so give the serializer your source-generated metadata: `UseJsonSerializer(MyJsonContext.Default)` on each client, with every message type (and `CloudEvent`, if you use the CloudEvent processor) listed in the context. +- **Configuration:** `AddClient(name, IConfiguration)` binds by reflection and is marked as not trim-safe; set the settings in code with `AddClient(name, settings => ...)` instead. Processor options bound with `AddProcessor<,>(IConfiguration)` are source-generated and safe. + +Lambda handlers are fine: their parameters are read from the delegate itself. -`tracestate` appears only when there is one. `body` is the serializer's output as-is when it is JSON; a custom serializer's non-JSON output goes in `body64`, Base64-encoded, instead. The queue's `MessageEncoding` applies to the whole envelope. Dead-lettering copies the envelope unchanged, so a message moved back from the poison queue replays with its trace context. +## Upgrading from 3.0 to 3.1 -**The envelope makes every message bigger.** It adds roughly 100 bytes, plus the length of `tracestate` when there is one. A custom serializer's non-JSON output grows by about a third on top of that, because Base64 turns every 3 bytes into 4. The service rejects a message over 64 KiB, so messages already close to that limit can start failing to send once the setting is on; check your largest messages first. The library does not fall back to sending plain. +3.1 adds to the API without removing anything. Recompile and these are the behaviour changes to know about: -The setting is off by default because it changes what is on the queue. Any receiver on 3.1 or later reads both forms, so mixed senders are fine. What needs care is a consumer on 3.0 or earlier, or other code reading the queue (an Azure Function, a script): it sees the envelope as the message and fails to deserialize it, or worse, deserializes it into an object with empty fields. Upgrade those, or teach them the format above, before turning the setting on for any sender. +- **The default serializer reads JSON property names case-insensitively.** A camelCase message used to fill none of a PascalCase type's properties, and was completed as if it were fine; now it fills them. Pass your own `JsonSerializerOptions` to keep the old strictness. Output is unchanged. +- **Poison messages no longer expire after 7 days.** They used to take the service's default time to live and could disappear unseen; now they stay until deleted or replayed, or for `PoisonTimeToLive`. +- **A poison message records why it was dead-lettered**, so the copy on the poison queue is now an envelope around the original body. A 3.1 reader (and `IDeadLetterQueue`) reads it; a tool that read poison messages as plain JSON sees the envelope. Moving one back by hand still works for 3.1 receivers, but `ReplayAsync` is the clean way: it strips the record. +- **One receive loop per processor** instead of one per handler slot. An idle processor makes one request per interval rather than `MaxConcurrentMessages` of them, and a burst after a quiet spell is picked up at the next poll instead of whenever each sleeping worker woke. +- **Quieter logs.** Healthy messages log at Debug, not Information. A failed poll is logged once, not twice. A handler failure is a warning while retries remain and an error on the last attempt. +- **Arguments are checked before calling the service:** `ReceiveAsync`, `AbandonAsync`, `RenewLockAsync` and `SendOptions` throw `ArgumentOutOfRangeException` where they used to get a 400 back, and a send over 64 KiB throws `QueueMessageTooLargeException` instead of a 413. +- **In-memory queues behave more like the service:** they enforce the size limit and emit spans and metrics. +- **`IStorageQueue`, `IQueueSender` and `IQueueReceiver` are always registered,** with or without a default client, so injecting one without a default client fails with a message naming the registered clients. Code that checked `GetService() is null` to detect a default client should check its configuration instead. +- **The processing-duration and size histograms have explicit buckets** (seconds and bytes) and units. Dashboards built on the old default buckets will see the bucket boundaries change once. ## Upgrading from 2.x @@ -325,7 +579,7 @@ The setting is off by default because it changes what is on the queue. Any recei | 2.x | 3.0 | |---|---| | `IQueueClientFactory.GetQueueClient()` → `AzureStorageQueueClient` | `GetQueue()` → `IStorageQueue`, or inject `IStorageQueue` / `IQueueSender` / `IQueueReceiver` | -| `ReceiveMessagesAsync(handle, handleException, ...)` | `AddProcessor()`, or `ReceiveAsync` in your own loop | +| `ReceiveMessagesAsync(handle, handleException, ...)` | `AddProcessor()`, or `ReadAllAsync` / `ReceiveAsync` in your own loop | | `SendMessageAsync(message, visibility, ttl)` | `SendAsync(message, new SendOptions(visibility, ttl))` | | `PeekMessagesAsync(n)` → `IEnumerable` | `PeekAsync(n)` → `PeekedQueueMessage` with identity and `Error` | | `IMessageConverter` | `IQueueMessageSerializer` | diff --git a/docs/v3-design.md b/docs/v3-design.md index 13c80a3..9160d8d 100644 --- a/docs/v3-design.md +++ b/docs/v3-design.md @@ -199,9 +199,9 @@ Behaviour, per processor registration (one hosted service each): - **Poison first.** Before the handler sees a message: if `DequeueCount > MaxDequeueCount`, or `Error` is set and `DeadLetterOnDeserializationFailure`, the message is dead-lettered and the handler is not called. - **Lease renewal.** While the handler runs, the lease is renewed at half `VisibilityTimeout`. A handler that takes ten minutes no longer causes duplicate processing. The renewal timer is stopped before the message is settled, so a renew and a complete never race. - **Completion.** With `AutoComplete`, a handler that returns completes the message; one that throws abandons it with `RetryDelay(DequeueCount)`. A handler may settle the message itself (`IsSettled`), and the processor then does nothing further for it. Completing after a successful handler is **not** tied to the shutdown token (the 2.1 lesson: cancelling it only redelivers work that is already done). -- **Nothing escapes.** Transport errors on receive are logged and backed off (exponential to `ErrorBackoffMax`); a missing queue is logged with "set CreateIfNotExists" and backed off the same way; handler exceptions are logged with message id and dequeue count; an exception from dead-lettering (for example, no permission to create the poison queue) falls back to abandon with the maximum retry delay and an error-level log, so nothing is lost silently. The host never stops because of this library. +- **Nothing escapes.** Transport errors on receive are logged once and backed off (exponential to `ErrorBackoffMax`); a missing queue gets its own log line saying to create it or set `CreateIfNotExists` (added in 3.1; 3.0 logged it as an ordinary failure), and is backed off the same way; handler exceptions are logged with message id and dequeue count; an exception from dead-lettering (for example, no permission to create the poison queue) falls back to abandoning the message with its normal `RetryDelay` for its dequeue count, with an error-level log, so nothing is lost silently. (The first draft said "the maximum retry delay"; the code has always used the dequeue count's delay. The maximum, five minutes, is used only when a delivery fails outside the handler.) The host never stops because of this library. - **Empty queue.** Polling backs off exponentially from `EmptyQueueBackoffMin` to `EmptyQueueBackoffMax`, resetting on the first non-empty receive. -- **Concurrency.** Up to `MaxConcurrentMessages` handlers in flight, each with its own scope and renewal timer. +- **Concurrency.** Up to `MaxConcurrentMessages` handlers in flight, each with its own scope and renewal timer. In 3.0 that was one worker per slot, each with its own receive loop; 3.1 replaced it with one receive loop per processor (see "3.1" below). - **Lifecycle.** Implements `IHostedLifecycleService`: `StartAsync` validates options and resolves the queue (a misconfigured client fails startup on purpose); polling starts in `StartedAsync`, after the application is up, so a slow startup does not hold leases. On shutdown it stops receiving, waits up to `ShutdownDrainTimeout` for in-flight handlers to finish on their own, and only then cancels their token (drain, then cancel — the first draft had these the other way round). A cancelled handler's message is abandoned with no delay so another instance can take it. Messages still unsettled become visible again when their lease lapses. - **Telemetry.** A `Consumer` span per message, the existing counters and histogram tagged with queue name and processor outcome (`completed`, `abandoned`, `dead_lettered`). @@ -274,3 +274,17 @@ Planned as four pull requests; delivered as one (#42) at the owner's request. Th 4. Removals, README rewrite, migration guide, `3.0.0-preview.1` tag. Previews publish from `v3.0.0-preview.N` tags with the existing `nuget.yml`; nuget.org marks them pre-release automatically. + +## 3.1 (issue #51) + +Issue #51 collected a review of 3.0 against Azure Functions queue triggers, the Service Bus processor, MassTransit, Wolverine, Rebus and SQS libraries. 3.1 delivered all of it except the 4.0 candidates (breaking changes) and the items the review chose to skip. The decisions that shape the code: + +- **One envelope format, with forward-compatibility rules.** 3.0's trace-context wrapper became the general envelope (`MessageEnvelope`): trace context, headers, compression, a reference to a body kept elsewhere, and the dead-letter record, each optional. A reader skips properties it does not know; anything that changes how the body is read uses a new `encoding` value, a new body kind or a new version, which an older reader refuses (the message gets `Error` and is dead-lettered intact) instead of reading it as a plain message, which System.Text.Json would turn into an object with empty fields. The marker must still be the first property, so an ordinary payload is never mistaken for an envelope. +- **One queue implementation over two transports.** `StorageQueue` now sits on `IQueueTransport`, implemented over the SDK client and over the in-memory queue. Encoding, size checks, dead-lettering, replay and telemetry exist once, and a test on in-memory queues runs the production path. This replaced `InMemoryStorageQueue` and `StorageQueueCommon`. +- **One receive loop per processor.** It waits for a free handler slot, takes every other free slot (up to 32), receives that many, and starts each delivery on its own task, which arms lease renewal first. This keeps F4 (no message waits behind another's handler) and adds "never hold a message no handler is free for". A non-empty receive goes straight back for more, which also fixes the slow pick-up after an idle spell. Batch handlers remain a possible follow-on. +- **Typed views share one lease.** `QueueMessage`'s receipt and settlement moved to an internal `MessageLease`, so a router, a CloudEvent adapter or a lambda handler can hand the application a message of its own type that settles the same delivery the processor tracks. +- **Compression lives in the envelope, not in a serializer wrapper.** The issue suggested a serializer wrapper; putting it in the envelope means every 3.1 receiver decompresses without configuration, the same way it reads trace context. +- **Dead-letter categories are a bounded set** (`max_dequeue`, `deserialization`, `handler`, `unroutable`) because they are a metric tag and a replay filter. +- **Configuration binding is split.** Processor options use the source-generated binder (trim-safe). Client settings keep reflection binding, because the generator cannot create or fill `QueueClientOptions`, which would silently drop `"ClientOptions": { "MessageEncoding": "Base64" }`; those overloads are marked as not trim-safe. +- **Behaviour change taken:** the default serializer reads property names case-insensitively. Writing camelCase by default stays a 4.0 candidate. +- **Separate packages** for Blob storage (`AzureStorage.QueueService.Blobs`, the claim-check store) and health checks (`AzureStorage.QueueService.HealthChecks`), so the core takes neither dependency. Both build on public seams in the core: `ILargeMessageStore` and `IQueueProcessorController`.