Fix ApiConnection authorization header thread safety (#9610)

* Fix ApiConnection authorization header thread safety

SetAuthenticator(HttpRequestMessage) was writing the Authorization
header to the shared HttpClient.DefaultRequestHeaders on every request.
HttpHeaders is not thread safe: under concurrent requests one thread
mutates the collection while another enumerates it during request
serialization, causing NullReferenceException failures, and requests
could also be serialized with a mismatched Authorization/Timestamp
pair causing intermittent authentication failures.

Set the Authorization header on the request message itself instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Include stack trace when logging unexpected api request exceptions

Network failures are already described by their HttpRequestError and
SocketErrorCode details and are frequent during outages, so they stay
on a single line. Any other exception type is unexpected, include its
stack trace so the source can be located directly from the logs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Harden ApiConnection.TryRequestAsync request path

Guard against a null RequestUri, which is valid when the client has a
BaseAddress set, and handle response deserialization failures with the
http status code and raw response content instead of surfacing a parse
exception, for example an html error page returned by a proxy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: Martin-Molinero

Api/ApiConnection.cs

@@ -245,7 +245,7 @@ public async Task<Tuple<bool, T>> TryRequestAsync<T>(HttpRequestMessage request,
             var stopwatch = Stopwatch.StartNew();
             try
             {
-                if (request.RequestUri.OriginalString.StartsWith('/'))
+                if (request.RequestUri != null && request.RequestUri.OriginalString.StartsWith('/'))
                 {
                     request.RequestUri = new Uri(request.RequestUri.ToString().TrimStart('/'), UriKind.Relative);
                 }
@@ -256,7 +256,18 @@ public async Task<Tuple<bool, T>> TryRequestAsync<T>(HttpRequestMessage request,
                 response = await _httpClient.SendAsync(request, cancellationTokenSource.Token).ConfigureAwait(false);
                 responseContentStream = await response.Content.ReadAsStreamAsync(cancellationTokenSource.Token).ConfigureAwait(false);
 
-                result = responseContentStream.DeserializeJson<T>(leaveOpen: true);
+                try
+                {
+                    result = responseContentStream.DeserializeJson<T>(leaveOpen: true);
+                }
+                catch (Exception err)
+                {
+                    // a non json payload, for example an html error page from a proxy or load balancer,
+                    // the http status and raw content describe the failure better than the parse exception
+                    Log.Error($"ApiConnection.TryRequest({request.RequestUri}): failed to deserialize response: {err.GetType().Name}: {err.Message}." +
+                        $" HTTP {(int)response.StatusCode} {response.ReasonPhrase}. Content: {GetRawResponseContent(responseContentStream)}");
+                    return new Tuple<bool, T>(false, null);
+                }
 
                 if (!response.IsSuccessStatusCode)
                 {
@@ -323,6 +334,12 @@ private static string DescribeError(Exception exception, bool timeoutElapsed, lo
                     }
                 }
                 builder.Append(Invariant($" after {elapsedMilliseconds}ms. ThreadPool: {ThreadPool.ThreadCount} threads, {ThreadPool.PendingWorkItemCount} pending work items"));
+                if (exception is not HttpRequestException and not OperationCanceledException)
+                {
+                    // network failures are described by their error codes above and are frequent during outages,
+                    // any other exception type is unexpected so include the stack trace to locate its source
+                    builder.Append(Environment.NewLine).Append(exception.StackTrace);
+                }
                 return builder.ToString();
             }
             catch (Exception)
@@ -360,11 +377,12 @@ private void SetAuthenticator(RestRequest request)
 
         private void SetAuthenticator(HttpRequestMessage request)
         {
-            request.Headers.Remove("Authorization");
             request.Headers.Remove("Timestamp");
 
             var base64EncodedAuthenticationString = GetAuthenticatorHeader(out var timeStamp);
-            _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString);
+            // set the authorization on the request itself, the client default headers are shared across
+            // concurrent requests and mutating them while requests are in flight is not thread safe
+            request.Headers.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString);
             request.Headers.Add("Timestamp", timeStamp);
         }