Skip to content

Commit 95830c2

Browse files
fix(clients): reduce chances of Push rate limiting (generated)
algolia/api-clients-automation#5153 Co-authored-by: algolia-bot <[email protected]> Co-authored-by: Clément Vannicatte <[email protected]>
1 parent ee151b1 commit 95830c2

File tree

1 file changed

+94
-76
lines changed

1 file changed

+94
-76
lines changed

algoliasearch/ingestion/client.py

Lines changed: 94 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,12 @@ async def chunked_push(
217217
"""
218218
Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
219219
"""
220+
offset = 0
220221
records: List[PushTaskRecords] = []
221222
responses: List[WatchResponse] = []
223+
wait_batch_size = batch_size // 10
224+
if wait_batch_size < 1:
225+
wait_batch_size = batch_size
222226
for i, obj in enumerate(objects):
223227
records.append(obj) # pyright: ignore
224228
if len(records) == batch_size or i == len(objects) - 1:
@@ -234,44 +238,49 @@ async def chunked_push(
234238
)
235239
)
236240
records = []
237-
if wait_for_tasks:
238-
for response in responses:
239-
240-
async def _func(_: Optional[Event]) -> Event:
241-
if response.event_id is None:
242-
raise ValueError(
243-
"received unexpected response from the push endpoint, eventID must not be undefined"
244-
)
245-
try:
246-
return await self.get_event(
247-
run_id=response.run_id,
248-
event_id=response.event_id,
249-
request_options=request_options,
250-
)
251-
except RequestException as e:
252-
if e.status_code == 404:
253-
return None # pyright: ignore
254-
raise e
255-
256-
_retry_count = 0
257-
258-
def _aggregator(_: Event | None) -> None:
259-
nonlocal _retry_count
260-
_retry_count += 1
261-
262-
def _validate(_resp: Event | None) -> bool:
263-
return _resp is not None
264-
265-
timeout = RetryTimeout()
266-
267-
await create_iterable(
268-
func=_func,
269-
validate=_validate,
270-
aggregator=_aggregator,
271-
timeout=lambda: timeout(_retry_count),
272-
error_validate=lambda _: _retry_count >= 50,
273-
error_message=lambda _: f"The maximum number of retries exceeded. (${_retry_count}/${50})",
274-
)
241+
if (
242+
wait_for_tasks
243+
and len(responses) > 0
244+
and (len(responses) % wait_batch_size == 0 or i == len(objects) - 1)
245+
):
246+
for response in responses[offset : offset + wait_batch_size]:
247+
248+
async def _func(_: Optional[Event]) -> Event:
249+
if response.event_id is None:
250+
raise ValueError(
251+
"received unexpected response from the push endpoint, eventID must not be undefined"
252+
)
253+
try:
254+
return await self.get_event(
255+
run_id=response.run_id,
256+
event_id=response.event_id,
257+
request_options=request_options,
258+
)
259+
except RequestException as e:
260+
if e.status_code == 404:
261+
return None # pyright: ignore
262+
raise e
263+
264+
_retry_count = 0
265+
266+
def _aggregator(_: Event | None) -> None:
267+
nonlocal _retry_count
268+
_retry_count += 1
269+
270+
def _validate(_resp: Event | None) -> bool:
271+
return _resp is not None
272+
273+
timeout = RetryTimeout()
274+
275+
await create_iterable(
276+
func=_func,
277+
validate=_validate,
278+
aggregator=_aggregator,
279+
timeout=lambda: timeout(_retry_count),
280+
error_validate=lambda _: _retry_count >= 50,
281+
error_message=lambda _: f"The maximum number of retries exceeded. (${_retry_count}/${50})",
282+
)
283+
offset += wait_batch_size
275284
return responses
276285

277286
async def create_authentication_with_http_info(
@@ -5326,8 +5335,12 @@ def chunked_push(
53265335
"""
53275336
Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
53285337
"""
5338+
offset = 0
53295339
records: List[PushTaskRecords] = []
53305340
responses: List[WatchResponse] = []
5341+
wait_batch_size = batch_size // 10
5342+
if wait_batch_size < 1:
5343+
wait_batch_size = batch_size
53315344
for i, obj in enumerate(objects):
53325345
records.append(obj) # pyright: ignore
53335346
if len(records) == batch_size or i == len(objects) - 1:
@@ -5343,44 +5356,49 @@ def chunked_push(
53435356
)
53445357
)
53455358
records = []
5346-
if wait_for_tasks:
5347-
for response in responses:
5348-
5349-
def _func(_: Optional[Event]) -> Event:
5350-
if response.event_id is None:
5351-
raise ValueError(
5352-
"received unexpected response from the push endpoint, eventID must not be undefined"
5353-
)
5354-
try:
5355-
return self.get_event(
5356-
run_id=response.run_id,
5357-
event_id=response.event_id,
5358-
request_options=request_options,
5359-
)
5360-
except RequestException as e:
5361-
if e.status_code == 404:
5362-
return None # pyright: ignore
5363-
raise e
5364-
5365-
_retry_count = 0
5366-
5367-
def _aggregator(_: Event | None) -> None:
5368-
nonlocal _retry_count
5369-
_retry_count += 1
5370-
5371-
def _validate(_resp: Event | None) -> bool:
5372-
return _resp is not None
5373-
5374-
timeout = RetryTimeout()
5375-
5376-
create_iterable_sync(
5377-
func=_func,
5378-
validate=_validate,
5379-
aggregator=_aggregator,
5380-
timeout=lambda: timeout(_retry_count),
5381-
error_validate=lambda _: _retry_count >= 50,
5382-
error_message=lambda _: f"The maximum number of retries exceeded. (${_retry_count}/${50})",
5383-
)
5359+
if (
5360+
wait_for_tasks
5361+
and len(responses) > 0
5362+
and (len(responses) % wait_batch_size == 0 or i == len(objects) - 1)
5363+
):
5364+
for response in responses[offset : offset + wait_batch_size]:
5365+
5366+
def _func(_: Optional[Event]) -> Event:
5367+
if response.event_id is None:
5368+
raise ValueError(
5369+
"received unexpected response from the push endpoint, eventID must not be undefined"
5370+
)
5371+
try:
5372+
return self.get_event(
5373+
run_id=response.run_id,
5374+
event_id=response.event_id,
5375+
request_options=request_options,
5376+
)
5377+
except RequestException as e:
5378+
if e.status_code == 404:
5379+
return None # pyright: ignore
5380+
raise e
5381+
5382+
_retry_count = 0
5383+
5384+
def _aggregator(_: Event | None) -> None:
5385+
nonlocal _retry_count
5386+
_retry_count += 1
5387+
5388+
def _validate(_resp: Event | None) -> bool:
5389+
return _resp is not None
5390+
5391+
timeout = RetryTimeout()
5392+
5393+
create_iterable_sync(
5394+
func=_func,
5395+
validate=_validate,
5396+
aggregator=_aggregator,
5397+
timeout=lambda: timeout(_retry_count),
5398+
error_validate=lambda _: _retry_count >= 50,
5399+
error_message=lambda _: f"The maximum number of retries exceeded. (${_retry_count}/${50})",
5400+
)
5401+
offset += wait_batch_size
53845402
return responses
53855403

53865404
def create_authentication_with_http_info(

0 commit comments

Comments
 (0)