From 04303d3c07a14b81c1749c72bc4ff5db701f098f Mon Sep 17 00:00:00 2001 From: Milan AJDINOVIC Date: Thu, 13 Mar 2025 13:58:45 +0100 Subject: [PATCH 1/2] pagination curstor for Query QL added support for initial cursor value and has next page --- pkg/infinity/remote.go | 52 ++++++++++------ pkg/models/query.go | 3 + src/editors/query/query.pagination.tsx | 85 ++++++++++++++++++-------- src/types/query.types.ts | 3 + 4 files changed, 97 insertions(+), 46 deletions(-) diff --git a/pkg/infinity/remote.go b/pkg/infinity/remote.go index c5b553353..4e7bed711 100644 --- a/pkg/infinity/remote.go +++ b/pkg/infinity/remote.go @@ -21,7 +21,7 @@ func GetFrameForURLSources(ctx context.Context, pCtx *backend.PluginContext, que if query.Type == models.QueryTypeJSON && query.Parser == models.InfinityParserBackend && query.PageMode != models.PaginationModeNone && query.PageMode != "" { return GetPaginatedResults(ctx, pCtx, query, infClient, requestHeaders) } - frame, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true) + frame, _, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true) return frame, err } @@ -64,30 +64,32 @@ func GetPaginatedResults(ctx context.Context, pCtx *backend.PluginContext, query case models.PaginationModeCursor: queries = append(queries, query) default: - frame, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true) + frame, _, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, query, infClient, requestHeaders, true) return frame, err } if query.PageMode != models.PaginationModeCursor { for _, currentQuery := range queries { - frame, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false) + frame, _, _, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false) frames = append(frames, frame) errs = errors.Join(errs, err) } } if query.PageMode == models.PaginationModeCursor { i := 0 - oCursor := "" + oCursor := query.PageParamCursorInitialValue + oHasNextPage := "true" for { currentQuery := query - if i > 0 && oCursor != "" { - currentQuery = ApplyPaginationItemToQuery(currentQuery, query.PageParamCursorFieldType, query.PageParamCursorFieldName, oCursor) - } - if i > query.PageMaxPages || (i > 0 && oCursor == "") { + currentQuery = ApplyPaginationItemToQuery(currentQuery, query.PageParamCursorFieldType, query.PageParamCursorFieldName, oCursor) + + if i > query.PageMaxPages || oCursor == "" || strings.EqualFold(oHasNextPage, query.PageParamNoNextPageValue) { break } i++ - frame, cursor, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false) + frame, cursor, hasNextPage, err := GetFrameForURLSourcesWithPostProcessing(ctx, pCtx, currentQuery, infClient, requestHeaders, false) oCursor = cursor + oHasNextPage = hasNextPage + frames = append(frames, frame) errs = errors.Join(errs, err) } @@ -134,13 +136,15 @@ func ApplyPaginationItemToQuery(currentQuery models.Query, fieldType models.Pagi return currentQuery } -func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend.PluginContext, query models.Query, infClient Client, requestHeaders map[string]string, postProcessingRequired bool) (*data.Frame, string, error) { +func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend.PluginContext, query models.Query, infClient Client, requestHeaders map[string]string, postProcessingRequired bool) (*data.Frame, string, string, error) { ctx, span := tracing.DefaultTracer().Start(ctx, "GetFrameForURLSourcesWithPostProcessing") logger := backend.Logger.FromContext(ctx) defer span.End() frame := GetDummyFrame(query) cursor := "" + hasNextPage := "" urlResponseObject, statusCode, duration, err := infClient.GetResults(ctx, pCtx, query, requestHeaders) + frame.Meta.ExecutedQueryString = infClient.GetExecutedURL(ctx, query) if infClient.IsMock { duration = 123 @@ -155,30 +159,30 @@ func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend. Query: query, Error: err.Error(), } - return frame, cursor, err + return frame, cursor, hasNextPage, err } if query.Type == models.QueryTypeGSheets { if frame, err = GetGoogleSheetsResponse(ctx, urlResponseObject, query); err != nil { - return frame, cursor, err + return frame, cursor, hasNextPage, err } } if query.Parser == "backend" { if query.Type == models.QueryTypeJSON || query.Type == models.QueryTypeGraphQL { if frame, err = GetJSONBackendResponse(ctx, urlResponseObject, query); err != nil { - return frame, cursor, err + return frame, cursor, hasNextPage, err } } if query.Type == models.QueryTypeCSV || query.Type == models.QueryTypeTSV { if responseString, ok := urlResponseObject.(string); ok { if frame, err = GetCSVBackendResponse(ctx, responseString, query); err != nil { - return frame, cursor, err + return frame, cursor, hasNextPage, err } } } if query.Type == models.QueryTypeXML || query.Type == models.QueryTypeHTML { if responseString, ok := urlResponseObject.(string); ok { if frame, err = GetXMLBackendResponse(ctx, responseString, query); err != nil { - return frame, cursor, err + return frame, cursor, hasNextPage, err } } } @@ -208,17 +212,27 @@ func GetFrameForURLSourcesWithPostProcessing(ctx context.Context, pCtx *backend. Query: query, Error: err.Error(), } - return frame, cursor, err + return frame, cursor, hasNextPage, err } if query.PageMode == models.PaginationModeCursor && strings.TrimSpace(query.PageParamCursorFieldExtractionPath) != "" { body, err := json.Marshal(urlResponseObject) if err != nil { - return frame, cursor, backend.PluginError(errors.New("error while finding the cursor value")) + return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while finding the cursor value")) } cursor, err = jsonframer.GetRootData(string(body), query.PageParamCursorFieldExtractionPath) if err != nil { - return frame, cursor, backend.PluginError(errors.New("error while extracting the cursor value")) + return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while extracting the cursor value")) + } + } + if query.PageMode == models.PaginationModeCursor && strings.TrimSpace(query.PageParamHasNextPageFieldExtractionPath) != "" { + body, err := json.Marshal(urlResponseObject) + if err != nil { + return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while finding the has next page value")) + } + hasNextPage, err = jsonframer.GetRootData(string(body), query.PageParamHasNextPageFieldExtractionPath) + if err != nil { + return frame, cursor, hasNextPage, backend.PluginError(errors.New("error while extracting the has next page value")) } } - return frame, cursor, nil + return frame, cursor, hasNextPage, nil } diff --git a/pkg/models/query.go b/pkg/models/query.go index 3f37f7c30..b0b262123 100644 --- a/pkg/models/query.go +++ b/pkg/models/query.go @@ -133,6 +133,9 @@ type Query struct { PageParamCursorFieldName string `json:"pagination_param_cursor_field_name,omitempty"` PageParamCursorFieldType PaginationParamType `json:"pagination_param_cursor_field_type,omitempty"` PageParamCursorFieldExtractionPath string `json:"pagination_param_cursor_extraction_path,omitempty"` + PageParamCursorInitialValue string `json:"pagination_param_cursor_initial_value,omitempty"` + PageParamNoNextPageValue string `json:"pagination_param_no_next_page_value,omitempty"` + PageParamHasNextPageFieldExtractionPath string `json:"pagination_param_has_next_page_extraction_path,omitempty"` PageParamListFieldName string `json:"pagination_param_list_field_name,omitempty"` PageParamListFieldType PaginationParamType `json:"pagination_param_list_field_type,omitempty"` PageParamListFieldValue string `json:"pagination_param_list_value,omitempty"` diff --git a/src/editors/query/query.pagination.tsx b/src/editors/query/query.pagination.tsx index da783727b..5694efaef 100644 --- a/src/editors/query/query.pagination.tsx +++ b/src/editors/query/query.pagination.tsx @@ -141,33 +141,64 @@ export const PaginationEditor = (props: PaginationEditorProps) => { )} {query.pagination_mode === 'cursor' && ( - - - Field name - onChange({ ...query, pagination_param_cursor_field_name: e.currentTarget.value })} - placeholder="cursor" - /> - Field type - - width={20} - options={paginationParamTypes} - value={query.pagination_param_cursor_field_type || 'query'} - onChange={(e) => onChange({ ...query, pagination_param_cursor_field_type: e.value || 'query' })} - /> - - Extraction path - - onChange({ ...query, pagination_param_cursor_extraction_path: e.currentTarget.value || '' })} - placeholder="selector to extract the cursor" - > - - + <> + + + Field name + onChange({ ...query, pagination_param_cursor_field_name: e.currentTarget.value })} + placeholder="cursor" + /> + Initial value + onChange({ ...query, pagination_param_cursor_initial_value: e.currentTarget.value })} + placeholder="cursor initial value" + /> + Field type + + width={20} + options={paginationParamTypes} + value={query.pagination_param_cursor_field_type || 'query'} + onChange={(e) => onChange({ ...query, pagination_param_cursor_field_type: e.value || 'query' })} + /> + + Extraction path + + onChange({ ...query, pagination_param_cursor_extraction_path: e.currentTarget.value || '' })} + placeholder="selector to extract the cursor" + > + + + + + + No next page match value + + onChange({ ...query, pagination_param_no_next_page_value: e.currentTarget.value })} + placeholder="No next page value" + /> + + Extraction path + + onChange({ ...query, pagination_param_has_next_page_extraction_path: e.currentTarget.value || '' })} + placeholder="selector to extract the has next page" + > + + + )} diff --git a/src/types/query.types.ts b/src/types/query.types.ts index 3c92beb5b..8fa249fa3 100644 --- a/src/types/query.types.ts +++ b/src/types/query.types.ts @@ -132,6 +132,9 @@ export type PaginationCursor = { pagination_param_cursor_field_name?: string; pagination_param_cursor_field_type?: PaginationParamType; pagination_param_cursor_extraction_path?: string; + pagination_param_cursor_initial_value?: string; + pagination_param_no_next_page_value?: string; + pagination_param_has_next_page_extraction_path?: string; } & PaginationBase<'cursor'>; export type PaginationList = { pagination_param_list_field_name?: string; From 746f0a8f1e4fbf91f14f6f5dcac083d1aa46dd15 Mon Sep 17 00:00:00 2001 From: micko189 Date: Thu, 27 Mar 2025 09:31:05 +0100 Subject: [PATCH 2/2] yarn changeset --- .changeset/gorgeous-lies-applaud.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/gorgeous-lies-applaud.md diff --git a/.changeset/gorgeous-lies-applaud.md b/.changeset/gorgeous-lies-applaud.md new file mode 100644 index 000000000..eed5d9142 --- /dev/null +++ b/.changeset/gorgeous-lies-applaud.md @@ -0,0 +1,5 @@ +--- +'grafana-infinity-datasource': patch +--- + +Added initial cursor value support, next page retrieval, and iteration exit handling for QueryQL pagination.