Skip to content

Pagination cursor initial value has next page #1201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Pagination cursor initial value has next page #1201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
04303d3
pagination curstor for Query QL added support for initial cursor valu…
milanajdinovic Mar 13, 2025
746f0a8
yarn changeset
micko189 Mar 27, 2025
58995e0
Merge branch 'main' into pagination-cursor-initial-value-has-next-page
micko189 Mar 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/gorgeous-lies-applaud.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'grafana-infinity-datasource': patch
---

Added initial cursor value support, next page retrieval, and iteration exit handling for QueryQL pagination.
52 changes: 33 additions & 19 deletions pkg/infinity/remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func GetFrameForURLSources(ctx context.Context, pCtx *backend.PluginContext, que
if query.Type == models.QueryTypeJSON && query.Parser == models.InfinityParserBackend && query.PageMode != models.PaginationModeNone && query.PageMode != "" {
return GetPaginatedResults(ctx, pCtx, query, infClient, requestHeaders)
}
frame, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true)
frame, _, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true)
return frame, err
}

Expand Down Expand Up @@ -64,30 +64,32 @@ func GetPaginatedResults(ctx context.Context, pCtx *backend.PluginContext, query
case models.PaginationModeCursor:
queries = append(queries, query)
default:
frame, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true)
frame, _, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true)
return frame, err
}
if query.PageMode != models.PaginationModeCursor {
for _, currentQuery := range queries {
frame, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false)
frame, _, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false)
frames = append(frames, frame)
errs = errors.Join(errs, err)
}
}
if query.PageMode == models.PaginationModeCursor {
i := 0
oCursor := ""
oCursor := query.PageParamCursorInitialValue
oHasNextPage := "true"
for {
currentQuery := query
if i > 0 && oCursor != "" {
currentQuery = ApplyPaginationItemToQuery(currentQuery, query.PageParamCursorFieldType, query.PageParamCursorFieldName, oCursor)
}
if i > query.PageMaxPages || (i > 0 && oCursor == "") {
currentQuery = ApplyPaginationItemToQuery(currentQuery, query.PageParamCursorFieldType, query.PageParamCursorFieldName, oCursor)

if i > query.PageMaxPages || oCursor == "" || strings.EqualFold(oHasNextPage, query.PageParamNoNextPageValue) {
break
}
i++
frame, cursor, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false)
frame, cursor, hasNextPage, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false)
oCursor = cursor
oHasNextPage = hasNextPage

frames = append(frames, frame)
errs = errors.Join(errs, err)
}
Expand Down Expand Up @@ -134,13 +136,15 @@ func ApplyPaginationItemToQuery(currentQuery models.Query, fieldType models.Pagi
return currentQuery
}

func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend.PluginContext, query models.Query, infClient Client, requestHeaders map[string]string, postProcessingRequired bool) (*data.Frame, string, error) {
func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend.PluginContext, query models.Query, infClient Client, requestHeaders map[string]string, postProcessingRequired bool) (*data.Frame, string, string, error) {
ctx, span := tracing.DefaultTracer().Start(ctx, "GetFrameForURLSourcesWithPostProcessing")
logger := backend.Logger.FromContext(ctx)
defer span.End()
frame := GetDummyFrame(query)
cursor := ""
hasNextPage := ""
urlResponseObject, statusCode, duration, err := infClient.GetResults(ctx, pCtx, query, requestHeaders)

frame.Meta.ExecutedQueryString = infClient.GetExecutedURL(ctx, query)
if infClient.IsMock {
duration = 123
Expand All @@ -155,30 +159,30 @@ func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend.
Query: query,
Error: err.Error(),
}
return frame, cursor, err
return frame, cursor, hasNextPage, err
}
if query.Type == models.QueryTypeGSheets {
if frame, err = GetGoogleSheetsResponse(ctx, urlResponseObject, query); err != nil {
return frame, cursor, err
return frame, cursor, hasNextPage, err
}
}
if query.Parser == "backend" {
if query.Type == models.QueryTypeJSON || query.Type == models.QueryTypeGraphQL {
if frame, err = GetJSONBackendResponse(ctx, urlResponseObject, query); err != nil {
return frame, cursor, err
return frame, cursor, hasNextPage, err
}
}
if query.Type == models.QueryTypeCSV || query.Type == models.QueryTypeTSV {
if responseString, ok := urlResponseObject.(string); ok {
if frame, err = GetCSVBackendResponse(ctx, responseString, query); err != nil {
return frame, cursor, err
return frame, cursor, hasNextPage, err
}
}
}
if query.Type == models.QueryTypeXML || query.Type == models.QueryTypeHTML {
if responseString, ok := urlResponseObject.(string); ok {
if frame, err = GetXMLBackendResponse(ctx, responseString, query); err != nil {
return frame, cursor, err
return frame, cursor, hasNextPage, err
}
}
}
Expand Down Expand Up @@ -208,17 +212,27 @@ func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend.
Query: query,
Error: err.Error(),
}
return frame, cursor, err
return frame, cursor, hasNextPage, err
}
if query.PageMode == models.PaginationModeCursor && strings.TrimSpace(query.PageParamCursorFieldExtractionPath) != "" {
body, err := json.Marshal(urlResponseObject)
if err != nil {
return frame, cursor, backend.PluginError(errors.New("error while finding the cursor value"))
return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while finding the cursor value"))
}
cursor, err = jsonframer.GetRootData(string(body), query.PageParamCursorFieldExtractionPath)
if err != nil {
return frame, cursor, backend.PluginError(errors.New("error while extracting the cursor value"))
return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while extracting the cursor value"))
}
}
if query.PageMode == models.PaginationModeCursor && strings.TrimSpace(query.PageParamHasNextPageFieldExtractionPath) != "" {
body, err := json.Marshal(urlResponseObject)
if err != nil {
return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while finding the has next page value"))
}
hasNextPage, err = jsonframer.GetRootData(string(body), query.PageParamHasNextPageFieldExtractionPath)
if err != nil {
return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while extracting the has next page value"))
}
}
return frame, cursor, nil
return frame, cursor, hasNextPage, nil
}
3 changes: 3 additions & 0 deletions pkg/models/query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ type Query struct {
PageParamCursorFieldName string `json:"pagination_param_cursor_field_name,omitempty"`
PageParamCursorFieldType PaginationParamType `json:"pagination_param_cursor_field_type,omitempty"`
PageParamCursorFieldExtractionPath string `json:"pagination_param_cursor_extraction_path,omitempty"`
PageParamCursorInitialValue string `json:"pagination_param_cursor_initial_value,omitempty"`
PageParamNoNextPageValue string `json:"pagination_param_no_next_page_value,omitempty"`
PageParamHasNextPageFieldExtractionPath string `json:"pagination_param_has_next_page_extraction_path,omitempty"`
PageParamListFieldName string `json:"pagination_param_list_field_name,omitempty"`
PageParamListFieldType PaginationParamType `json:"pagination_param_list_field_type,omitempty"`
PageParamListFieldValue string `json:"pagination_param_list_value,omitempty"`
Expand Down
85 changes: 58 additions & 27 deletions src/editors/query/query.pagination.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,33 +141,64 @@ export const PaginationEditor = (props: PaginationEditorProps) => {
</EditorField>
)}
{query.pagination_mode === 'cursor' && (
<EditorField label="Cursor field">
<Stack>
<InlineLabel width={12}>Field name</InlineLabel>
<Input
width={30}
value={query.pagination_param_cursor_field_name || ''}
onChange={(e) => onChange({ ...query, pagination_param_cursor_field_name: e.currentTarget.value })}
placeholder="cursor"
/>
<InlineLabel width={12}>Field type</InlineLabel>
<Select<PaginationParamType>
width={20}
options={paginationParamTypes}
value={query.pagination_param_cursor_field_type || 'query'}
onChange={(e) => onChange({ ...query, pagination_param_cursor_field_type: e.value || 'query' })}
/>
<InlineLabel width={20} tooltip="selector to extract the cursor">
Extraction path
</InlineLabel>
<Input
width={20}
value={query.pagination_param_cursor_extraction_path}
onChange={(e) => onChange({ ...query, pagination_param_cursor_extraction_path: e.currentTarget.value || '' })}
placeholder="selector to extract the cursor"
></Input>
</Stack>
</EditorField>
<>
<EditorField label="Cursor field">
<Stack>
<InlineLabel width={12}>Field name</InlineLabel>
<Input
width={30}
value={query.pagination_param_cursor_field_name || ''}
onChange={(e) => onChange({ ...query, pagination_param_cursor_field_name: e.currentTarget.value })}
placeholder="cursor"
/>
<InlineLabel width={12}>Initial value</InlineLabel>
<Input
width={20}
value={query.pagination_param_cursor_initial_value || 'null'}
onChange={(e) => onChange({ ...query, pagination_param_cursor_initial_value: e.currentTarget.value })}
placeholder="cursor initial value"
/>
<InlineLabel width={12}>Field type</InlineLabel>
<Select<PaginationParamType>
width={20}
options={paginationParamTypes}
value={query.pagination_param_cursor_field_type || 'query'}
onChange={(e) => onChange({ ...query, pagination_param_cursor_field_type: e.value || 'query' })}
/>
<InlineLabel width={20} tooltip="selector to extract the cursor">
Extraction path
</InlineLabel>
<Input
width={30}
value={query.pagination_param_cursor_extraction_path}
onChange={(e) => onChange({ ...query, pagination_param_cursor_extraction_path: e.currentTarget.value || '' })}
placeholder="selector to extract the cursor"
></Input>
</Stack>
</EditorField>
<EditorField label="Has next page field">
<Stack>
<InlineLabel width={25} tooltip="value that will be matched with received have next page field to stop iterating">
No next page match value
</InlineLabel>
<Input
width={20}
value={query.pagination_param_no_next_page_value || 'false'}
onChange={(e) => onChange({ ...query, pagination_param_no_next_page_value: e.currentTarget.value })}
placeholder="No next page value"
/>
<InlineLabel width={20} tooltip="selector to extract the have next page">
Extraction path
</InlineLabel>
<Input
width={30}
value={query.pagination_param_has_next_page_extraction_path}
onChange={(e) => onChange({ ...query, pagination_param_has_next_page_extraction_path: e.currentTarget.value || '' })}
placeholder="selector to extract the has next page"
></Input>
</Stack>
</EditorField>
</>
)}
</Stack>
</>
Expand Down
3 changes: 3 additions & 0 deletions src/types/query.types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ export type PaginationCursor = {
pagination_param_cursor_field_name?: string;
pagination_param_cursor_field_type?: PaginationParamType;
pagination_param_cursor_extraction_path?: string;
pagination_param_cursor_initial_value?: string;
pagination_param_no_next_page_value?: string;
pagination_param_has_next_page_extraction_path?: string;
} & PaginationBase<'cursor'>;
export type PaginationList = {
pagination_param_list_field_name?: string;
Expand Down