We have a use case where we allow users to scan market data for instruments that meet certain criteria. These scans can request historical data snapshots across 1000-2000 instruments or more. Memory profiling shows that after performing a scan of almost 2000 instruments the OMMConnection object was using 158MB of memory (see attached screenshot from dotMemory).
Is there a way to not cache OMMStreamCache objects if it is a snapshot request, or could the amount of data held in memory be reduced? Here is our code when making a historical snapshot request...
public IObservable<WorkspaceSnapShotStreamingResponse> GetTimeSeriesDataSnapShot( ISession workspaceSession, WorkspaceSnapShotStreamingRequest request, WorkspaceMetadataResponse metadataResponse) { return Observable.Create<WorkspaceSnapShotStreamingResponse>(observer => { SerialDisposable activeSubscriptionToken = new SerialDisposable(); if (workspaceSession.OpenState == Session.State.Opened) { // Build the request var summariesDefinition = BuildRequest(request); // Make the request activeSubscriptionToken.Disposable = summariesDefinition .GetDataAsync(workspaceSession) .ToObservable() .Subscribe(OnSubscribeAction, observer.OnError); // Deal with the response void OnSubscribeAction(IDataSetResponse workspaceResponse) { if (workspaceResponse.Data != null) { if (workspaceResponse.Data.Table != null) { observer.OnNext(ConvertDataTableToWorkspacePriceRecords( workspaceResponse.Data.Table, request.Interval, request.View, TimeSeriesUpdateType.Update)); observer.OnCompleted(); } else { Logger.Warn(workspaceResponse.Data.Errors); observer.OnNext(new WorkspaceSnapShotStreamingResponse()); } } } } return activeSubscriptionToken; }); } private ISummariesDefinition BuildRequest(WorkspaceSnapShotStreamingRequest request) { var summariesDefinition = Summaries .Definition(request.Symbol) .Interval(ResponseHelper.ConvertInterval(request.Interval, Logger)) .Fields(request.View.MapTo.ToArray()); if (request.Count > 0) { summariesDefinition.Count(request.Count); } else if (request.RequestRange.StartDateTime.HasValue) { summariesDefinition.Start((DateTime)request.RequestRange.StartDateTime); } else if (request.RequestRange.EndDateTime.HasValue) { summariesDefinition.End((DateTime)request.RequestRange.EndDateTime); } return summariesDefinition; }