ObjectDisposedException, The query specified in the URI is not valid. IFeatureCollection has been disposed #2407
Description
Under production load (200 to 300 requests per second). Our OData endpoint returns about 1% responses return with status Code 400. Second part, There are no traces being emitted from ILogger, having some trace statements here could help triage this quicker.
{
"error": {
"code": "",
"message": "The query specified in the URI is not valid. IFeatureCollection has been disposed.\r\nObject name: 'Collection'.",
"details": [],
"innererror": {
"message": "IFeatureCollection has been disposed.\r\nObject name: 'Collection'.",
"type": "System.ObjectDisposedException",
"stacktrace": " at Microsoft.AspNetCore.Http.Features.FeatureReferences`1.ThrowContextDisposed()\r\n at Microsoft.AspNetCore.Http.DefaultHttpRequest.get_Query()\r\n at Microsoft.AspNet.OData.Adapters.WebApiRequestMessage.get_QueryParameters()\r\n at Microsoft.AspNet.OData.Query.ODataQueryOptions.GetODataQueryParameters()\r\n at Microsoft.AspNet.OData.Query.ODataQueryOptions.AddAutoSelectExpandProperties()\r\n at Microsoft.AspNet.OData.Query.ODataQueryOptions.ApplyTo(IQueryable query, ODataQuerySettings querySettings)\r\n at Microsoft.AspNet.OData.EnableQueryAttribute.ExecuteQuery(Object responseValue, IQueryable singleResultCollection, IWebApiActionDescriptor actionDescriptor, Func`2 modelFunction, IWebApiRequestMessage request, Func`2 createQueryOptionFunction)\r\n at Microsoft.AspNet.OData.EnableQueryAttribute.OnActionExecuted(Object responseValue, IQueryable singleResultCollection, IWebApiActionDescriptor actionDescriptor, IWebApiRequestMessage request, Func`2 modelFunction, Func`2 createQueryOptionFunction, Action`1 createResponseAction, Action`3 createErrorAction)"
}
}
}Assemblies affected
NuGet: Microsoft.AspNetCore.OData:
Version: 7.5.4
Assembly Version: 7.5.4.0
NuGet: Microsoft.OData.Core
Version: 7.8.1
Assembly Version: 7.8.1.0
Reproduce steps
This was quite hard to reproduce locally, I had to setup JMeter to issue large amount of requests to reproduce the issue outside of production.
The requests vary in 3 queries on different entity sets:
odata/accounts?partitionkey eq 'e58ec759-cdd4-42d9-af7b-807214f4a456' and clientId eq '381f7bcb-8073-470f-a54e-effe214186db'odata/eventGridFilters?partitionkey eq 'e58ec759-cdd4-42d9-af7b-807214f4a456' and parentId eq '381f7bcb-8073-470f-a54e-effe214186db'odata/creators?partitionkey eq 'e58ec759-cdd4-42d9-af7b-807214f4a456' and parentId eq '381f7bcb-8073-470f-a54e-effe214186db'
Each controller basically looks nearly the same like:
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.AspNet.OData;
using Microsoft.AspNet.OData.Query;
using Microsoft.AspNet.OData.Routing;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.LocationServices.IdentityService.CosmosDb;
[Route("odata/accounts")]
[ODataRoutePrefix("accounts")]
[ApiController]
[Authorize]
public class AccountsODataController : ODataController
{
private readonly IResourceRepository accountRepository;
public AccountsODataController(IResourceRepository accountRepository)
{
this.accountRepository = accountRepository;
}
[HttpGet("")]
[EnableQuery(
AllowedQueryOptions =
AllowedQueryOptions.Select |
AllowedQueryOptions.Count |
AllowedQueryOptions.Filter |
AllowedQueryOptions.OrderBy |
AllowedQueryOptions.Top |
AllowedQueryOptions.Skip)]
public async Task<IEnumerable<DocumentAccount>> QueryAccounts(ODataQueryOptions queryOptions)
{
if (!queryOptions.TryExtractFilter(nameof(ModelConstants.PartitionKey), out string partitionKey))
{
queryOptions.TryExtractFilter(ModelConstants.SubscriptionId, out partitionKey);
}
ScopeQuery query = new ScopeQuery()
{
PartitionKey = partitionKey,
ResourceType = ModelConstants.AccountResourceType
};
return await accountRepository.QueryResourceTypeAsync<DocumentAccount>(query, queryOptions, HttpContext.RequestAborted);
}Startup.cs
public static void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
app.UseRouting();
app.UseErrorHandling();
app.UseAuthenticationMiddleware();
app.UseEndpoints(endpoints =>
{
endpoints.EnableDependencyInjection();
endpoints.Select().Filter().OrderBy().Count().MaxTop(500);
endpoints.MapHealthChecks("/hc");
endpoints.MapODataRoute("odata", "odata", container =>
{
container.AddService(OData.ServiceLifetime.Singleton, sp => GetEdmModel());
container.AddService<ODataPayloadValueConverter, ExtendedODataConverter>(OData.ServiceLifetime.Singleton);
container.AddService(OData.ServiceLifetime.Singleton, _ => app.ApplicationServices.GetRequiredService<ODataUriResolver>());
container.AddService<IEnumerable<IODataRoutingConvention>>(OData.ServiceLifetime.Singleton,
sp => ODataRoutingConventions.CreateDefaultWithAttributeRouting("odata", endpoints.ServiceProvider));
});
endpoints.MapControllers();
});
}
private static IEdmModel GetEdmModel()
{
var odataBuilder = new ODataConventionModelBuilder();
odataBuilder.EnableLowerCamelCase();
odataBuilder.EntitySet<DocumentAccount>("Accounts");
var accountScopedQuery = odataBuilder.Action("AccountsScopeQuery");
accountScopedQuery.Parameter<string>("partitionKey").Optional();
accountScopedQuery.Parameter<string>(ModelConstants.ResourceGroup).Optional();
accountScopedQuery.Parameter<string>(ModelConstants.ResourceName).Optional();
accountScopedQuery.Parameter<string>(ModelConstants.ClientId).Optional();
accountScopedQuery.Parameter<string>(ModelConstants.ParentId).Optional();
accountScopedQuery.ReturnsCollectionFromEntitySet<DocumentAccount>("Accounts");
odataBuilder.EntitySet<DocumentCreatorResource>("Creators");
var creatorScopedQuery = odataBuilder.Action("CreatorsScopeQuery");
creatorScopedQuery.Parameter<string>("partitionKey").Optional();
creatorScopedQuery.Parameter<string>(ModelConstants.ResourceName).Optional();
creatorScopedQuery.Parameter<string>(ModelConstants.Id).Optional();
creatorScopedQuery.Parameter<string>(ModelConstants.ParentId).Optional();
creatorScopedQuery.ReturnsCollectionFromEntitySet<DocumentCreatorResource>("Creators");
odataBuilder.EntitySet<DocumentEventGridFilter>("EventGridFilters");
var eventGridScopedQuery = odataBuilder.Action("EventGridFiltersScopeQuery");
eventGridScopedQuery.Parameter<string>("partitionKey").Optional();
eventGridScopedQuery.Parameter<string>(ModelConstants.ResourceName).Optional();
eventGridScopedQuery.Parameter<string>(ModelConstants.Id).Optional();
eventGridScopedQuery.Parameter<string>(ModelConstants.ParentId).Optional();
eventGridScopedQuery.ReturnsCollectionFromEntitySet<DocumentEventGridFilter>("EventGridFilters");
odataBuilder.EntitySet<DocumentSubscription>("Subscriptions");
var subscriptionsScopeQuery = odataBuilder.Action("SubscriptionsScopeQuery");
subscriptionsScopeQuery.Parameter<string>("partitionKey").Optional();
subscriptionsScopeQuery.Parameter<string>(ModelConstants.Id).Optional();
subscriptionsScopeQuery.ReturnsCollectionFromEntitySet<DocumentSubscription>("Subscriptions");
var model = odataBuilder.GetEdmModel();
return model;
}The exact call stack is provided, think this failure point is on this line:
public IDictionary<string, string> QueryParameters
{
get
{
IDictionary<string, string> result = new Dictionary<string, string>();
foreach (var pair in this.innerRequest.Query) // <--- Failing object disposed exception.
{
if (!result.ContainsKey(pair.Key))
{
result.Add(pair.Key, pair.Value);
}
}
return result;
}
}Expected result
- I would expect 500 internal server error in the case this happens.
- I would expect ILogger Trace statements so debugging is easier
- I would expect library to not access disposed http context and instead return 200 with the result set.
If there is an work around like downgrading version of NuGet that would be appreciated. Currently this is a blocking issue.
Actual result
What's actually happening is 1% of responses are returning back with 400 response code.