feat(ourlogs): Add auto-refresh to logs

This adds an auto-refresh toggle behind the 'ourlogs-live-refresh' flag.

Auto-refresh can be enabled with filters so long as the sort is by time and descending. It works on 5 second polling, with a virtual time using RAF to emulate streaming. Filters are applied so data is queried in a ~30s second window behind our ingest delay.

Auto-refresh will automatically disable itself under the following conditions:
- You've hit a rate limit consistently over 3 polls (>1k logs/s)
- You've hit an api error for any reason
- You've hit the absolute timeout cap (10 minutes), you can re-enable it if you're actively using it
- You change sort / filter etc.

Author: k-fish

static/app/components/searchQueryBuilder/context.tsx

@@ -104,15 +104,29 @@ export function SearchQueryBuilderProvider({
     disabled,
   });
 
+  const stableFieldDefinitionGetter = useMemo(
+    () => fieldDefinitionGetter,
+    [fieldDefinitionGetter]
+  );
+
+  const stableFilterKeys = useMemo(() => filterKeys, [filterKeys]);
+
+  const stableGetSuggestedFilterKey = useCallback(
+    (key: string) => {
+      return getSuggestedFilterKey ? getSuggestedFilterKey(key) : key;
+    },
+    [getSuggestedFilterKey]
+  );
+
   const parseQuery = useCallback(
     (query: string) =>
-      parseQueryBuilderValue(query, fieldDefinitionGetter, {
+      parseQueryBuilderValue(query, stableFieldDefinitionGetter, {
         getFilterTokenWarning,
         disallowFreeText,
         disallowLogicalOperators,
         disallowUnsupportedFilters,
         disallowWildcard,
-        filterKeys,
+        filterKeys: stableFilterKeys,
         invalidMessages,
         filterKeyAliases,
       }),
@@ -121,8 +135,8 @@ export function SearchQueryBuilderProvider({
       disallowLogicalOperators,
       disallowUnsupportedFilters,
       disallowWildcard,
-      fieldDefinitionGetter,
-      filterKeys,
+      stableFieldDefinitionGetter,
+      stableFilterKeys,
       getFilterTokenWarning,
       invalidMessages,
       filterKeyAliases,
@@ -150,10 +164,10 @@ export function SearchQueryBuilderProvider({
       parsedQuery,
       filterKeySections: filterKeySections ?? [],
       filterKeyMenuWidth,
-      filterKeys,
-      getSuggestedFilterKey: getSuggestedFilterKey ?? ((key: string) => key),
+      filterKeys: stableFilterKeys,
+      getSuggestedFilterKey: stableGetSuggestedFilterKey,
       getTagValues,
-      getFieldDefinition: fieldDefinitionGetter,
+      getFieldDefinition: stableFieldDefinitionGetter,
       dispatch,
       wrapperRef,
       actionBarRef,
@@ -177,10 +191,10 @@ export function SearchQueryBuilderProvider({
     parsedQuery,
     filterKeySections,
     filterKeyMenuWidth,
-    filterKeys,
-    getSuggestedFilterKey,
+    stableFilterKeys,
+    stableGetSuggestedFilterKey,
     getTagValues,
-    fieldDefinitionGetter,
+    stableFieldDefinitionGetter,
     dispatch,
     handleSearch,
     placeholder,

static/app/utils/timeSeries/markDelayedData.tsx

@@ -2,6 +2,7 @@
  * Given a timeseries and a delay in seconds, goes through the timeseries data, and marks each point as either delayed (data bucket ended before the delay threshold) or not
  */
 
+import {defined} from 'sentry/utils';
 import type {TimeSeries} from 'sentry/views/dashboards/widgets/common/types';
 
 export function markDelayedData(timeSeries: TimeSeries, delay: number): TimeSeries {
@@ -22,6 +23,10 @@ export function markDelayedData(timeSeries: TimeSeries, delay: number): TimeSeri
       const bucketEndTimestamp = new Date(datum.timestamp).getTime() + bucketSize;
       const delayed = bucketEndTimestamp >= ingestionDelayTimestamp;
 
+      if (defined(datum.incomplete)) {
+        return datum;
+      }
+
       if (!delayed) {
         return datum;
       }

static/app/views/explore/contexts/logs/logsPageParams.tsx

@@ -237,7 +237,15 @@ function setLogsPageParams(location: Location, pageParams: LogPageParamsUpdate)
   updateNullableLocation(target, LOGS_AGGREGATE_PARAM_KEY, pageParams.aggregateParam);
   if (!pageParams.isTableFrozen) {
     updateLocationWithLogSortBys(target, pageParams.sortBys);
-    updateLocationWithAggregateSortBys(target, pageParams.aggregateSortBys);
+    if (
+      pageParams.sortBys ||
+      pageParams.search ||
+      pageParams.aggregateFn ||
+      pageParams.aggregateParam ||
+      pageParams.groupBy
+    ) {
+      updateLocationWithAggregateSortBys(target, pageParams.aggregateSortBys);
+    }
     if (pageParams.sortBys || pageParams.aggregateSortBys || pageParams.search) {
       // make sure to clear the cursor every time the query is updated
       delete target.query[LOGS_CURSOR_KEY];
@@ -511,14 +519,17 @@ export function useLogsAutoRefresh() {
 
 export function useSetLogsAutoRefresh() {
   const setPageParams = useSetLogsPageParams();
-  const {queryKey} = useLogsQueryKeyWithInfinite({referrer: 'api.explore.logs-table'});
+  const {queryKey} = useLogsQueryKeyWithInfinite({
+    referrer: 'api.explore.logs-table',
+    autoRefresh: false,
+  });
   const queryClient = useQueryClient();
   return useCallback(
     (autoRefresh: boolean) => {
-      setPageParams({autoRefresh});
       if (autoRefresh) {
         queryClient.removeQueries({queryKey});
       }
+      setPageParams({autoRefresh});
     },
     [setPageParams, queryClient, queryKey]
   );

static/app/views/explore/logs/constants.tsx

@@ -13,7 +13,10 @@ export const LogAttributesHumanLabel: Partial<Record<OurLogFieldKey, string>> =
   [OurLogKnownFieldKey.TRACE_ID]: t('Trace'),
 };
 
-export const LOG_INGEST_DELAY = 10_000;
+export const MAX_LOG_INGEST_DELAY = 40_000;
+export const MAX_LOG_INGEST_QUERY_DELAY = 15_000;
+export const QUERY_PAGE_LIMIT = 5000; // If this does not equal the limit with auto-refresh, the query keys will diverge and they will have separate caches. We may want to make this change in the future.
+export const QUERY_PAGE_LIMIT_WITH_AUTO_REFRESH = 5000;
 
 /**
  * These are required fields are always added to the query when fetching the log table.
@@ -66,6 +69,6 @@ export const LOGS_INSTRUCTIONS_URL =
 
 export const LOGS_FILTER_KEY_SECTIONS: FilterKeySection[] = [LOGS_FILTERS];
 
-export const VIRTUAL_STREAMED_INTERVAL_MS = 333;
+export const VIRTUAL_STREAMED_INTERVAL_MS = 250;
 
 export const LOGS_GRID_SCROLL_MIN_ITEM_THRESHOLD = 100; // Items from bottom of table to trigger table fetch.

static/app/views/explore/logs/logsAutoRefresh.spec.tsx

@@ -139,4 +139,106 @@ describe('LogsAutoRefresh', () => {
       expect(mockApi).toHaveBeenCalledTimes(5);
     });
   });
+
+  it('disables auto-refresh after 3 consecutive requests with more data', async () => {
+    const mockApi = MockApiClient.addMockResponse({
+      url: `/organizations/${organization.slug}/events/`,
+      method: 'GET',
+      body: {data: mockLogsData},
+      headers: {
+        Link: '<http://localhost/api/0/organizations/org-slug/events/?cursor=0:5000:0>; rel="next"; results="true"',
+      },
+    });
+
+    jest.spyOn(logsPageParams, 'useLogsAutoRefresh').mockReturnValue(true);
+    jest.spyOn(logsPageParams, 'useLogsRefreshInterval').mockReturnValue(1); // Faster interval for testing
+
+    renderWithProviders(<AutorefreshToggle />);
+
+    const toggleSwitch = screen.getByRole('checkbox', {name: 'Auto-refresh'});
+    expect(toggleSwitch).toBeChecked();
+
+    await waitFor(() => {
+      expect(mockApi).toHaveBeenCalledTimes(3);
+    });
+
+    // After 3 consecutive requests with more data, auto-refresh should be disabled
+    await waitFor(() => {
+      expect(setAutoRefresh).toHaveBeenCalledWith(false);
+    });
+  });
+
+  it('continues auto-refresh when there is no more data', async () => {
+    const mockApi = MockApiClient.addMockResponse({
+      url: `/organizations/${organization.slug}/events/`,
+      method: 'GET',
+      body: {data: mockLogsData},
+      headers: {
+        Link: '', // No Link header means no more data
+      },
+    });
+
+    jest.spyOn(logsPageParams, 'useLogsAutoRefresh').mockReturnValue(true);
+    jest.spyOn(logsPageParams, 'useLogsRefreshInterval').mockReturnValue(1); // Faster interval for testing
+
+    renderWithProviders(<AutorefreshToggle />);
+
+    const toggleSwitch = screen.getByRole('checkbox', {name: 'Auto-refresh'});
+    expect(toggleSwitch).toBeChecked();
+
+    await waitFor(() => {
+      expect(mockApi).toHaveBeenCalledTimes(5);
+    });
+
+    // Auto-refresh should NOT be disabled
+    expect(setAutoRefresh).not.toHaveBeenCalledWith(false);
+  });
+
+  it('disables auto-refresh when API request fails', async () => {
+    const initialMock = MockApiClient.addMockResponse({
+      url: `/organizations/${organization.slug}/events/`,
+      method: 'GET',
+      body: {data: mockLogsData},
+    });
+
+    jest.spyOn(logsPageParams, 'useLogsAutoRefresh').mockReturnValue(true);
+    jest.spyOn(logsPageParams, 'useLogsRefreshInterval').mockReturnValue(1); // Faster interval for testing
+
+    renderWithProviders(<AutorefreshToggle />);
+
+    const toggleSwitch = screen.getByRole('checkbox', {name: 'Auto-refresh'});
+    expect(toggleSwitch).toBeChecked();
+
+    await waitFor(() => {
+      expect(initialMock).toHaveBeenCalled();
+    });
+
+    expect(setAutoRefresh).not.toHaveBeenCalled();
+
+    initialMock.mockClear();
+    const errorMock = MockApiClient.addMockResponse({
+      url: `/organizations/${organization.slug}/events/`,
+      method: 'GET',
+      statusCode: 500,
+      body: {
+        detail: 'Internal Server Error',
+      },
+    });
+
+    await waitFor(() => {
+      expect(errorMock).toHaveBeenCalled();
+    });
+
+    await waitFor(() => {
+      expect(setAutoRefresh).toHaveBeenCalledWith(false);
+    });
+
+    await userEvent.hover(toggleSwitch);
+
+    expect(
+      await screen.findByText(
+        'Auto-refresh was disabled due to an error fetching logs. If the issue persists, please contact support.'
+      )
+    ).toBeInTheDocument();
+  });
 });