Skip to content

Commit e43bca2

Browse files
committed
chore: determine wait batch size on batch size
1 parent 1eb3ac2 commit e43bca2

File tree

4 files changed

+15
-10
lines changed

4 files changed

+15
-10
lines changed

templates/go/ingestion_helpers.mustache

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ func (c *APIClient) ChunkedPush(indexName string, objects []map[string]any, acti
1616
batchSize: 1000,
1717
}
1818

19+
waitBatchSize := conf.batchSize / 10
20+
1921
for _, opt := range opts {
2022
opt.apply(&conf)
2123
}
@@ -60,11 +62,11 @@ func (c *APIClient) ChunkedPush(indexName string, objects []map[string]any, acti
6062
}
6163

6264

63-
if conf.waitForTasks && (len(responses) % 50 == 0 || i == len(objects)-1) {
65+
if conf.waitForTasks && (len(responses) % waitBatchSize == 0 || i == len(objects)-1) {
6466
var waitableResponses []WatchResponse
6567
66-
if len(responses) > 50 {
67-
waitableResponses = responses[len(responses)-50:]
68+
if len(responses) > waitBatchSize {
69+
waitableResponses = responses[len(responses)-waitBatchSize:]
6870
} else {
6971
waitableResponses = responses
7072
}

templates/java/api_helpers.mustache

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,10 @@ public <T> List<WatchResponse> chunkedPush(
4545
List<WatchResponse> responses = new ArrayList<>();
4646
List<T> records = new ArrayList<>();
4747
Int count = 0;
48+
Int waitBatchSize = batchSize / 10;
4849
4950
for (T item : objects) {
50-
if (records.size() == batchSize || count == objects.length() - 1) {
51+
if (records.size() == batchSize || count == objects.size() - 1) {
5152
count++;
5253
5354
WatchResponse watch =
@@ -64,8 +65,8 @@ public <T> List<WatchResponse> chunkedPush(
6465

6566
records.add(item);
6667

67-
if (waitForTasks && (responses.size() % 50 == 0 || count == objects.length() - 1)) {
68-
responses.subList(Math.max(responses.size() - 50, 0), responses.size()).forEach(response -> {
68+
if (waitForTasks && (responses.size() % waitBatchSize == 0 || count == objects.size() - 1)) {
69+
responses.subList(Math.max(responses.size() - waitBatchSize, 0), responses.size()).forEach(response -> {
6970
TaskUtils.retryUntil(
7071
() -> {
7172
try {

templates/php/api.mustache

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,7 @@ use Algolia\AlgoliaSearch\Exceptions\NotFoundException;
337337
$responses = [];
338338
$records = [];
339339
$count = 0;
340+
$waitBatchSize = (int)($batchSize / 10);
340341
341342
foreach ($objects as $object) {
342343
$records[] = $object;
@@ -349,10 +350,10 @@ use Algolia\AlgoliaSearch\Exceptions\NotFoundException;
349350

350351
++$count;
351352

352-
if ($waitForTasks && (0 === count($responses) % 50 || $count === sizeof($objects) - 1)) {
353+
if ($waitForTasks && (0 === count($responses) % $waitBatchSize || $count === sizeof($objects) - 1)) {
353354
$timeoutCalculation = 'Algolia\AlgoliaSearch\Support\Helpers::linearTimeout';
354355
355-
foreach ($responses as $response) {
356+
foreach (array_slice($responses, -waitBatchSize) as $response) {
356357
$retry = 0;
357358
358359
while ($retry < 50) {

templates/python/ingestion_helpers.mustache

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"""
1414
records: List[PushTaskRecords] = []
1515
responses: List[WatchResponse] = []
16+
wait_batch_size = batch_size // 10
1617
for i, obj in enumerate(objects):
1718
records.append(obj) # pyright: ignore
1819
if len(records) == batch_size or i == len(objects) - 1:
@@ -28,8 +29,8 @@
2829
)
2930
)
3031
records = []
31-
if wait_for_tasks and (len(responses) % 50 == 0 or i == len(objects) - 1):
32-
for response in responses:
32+
if wait_for_tasks and (len(responses) % wait_batch_size == 0 or i == len(objects) - 1):
33+
for response in responses[-wait_batch_size:]:
3334
{{^isSyncClient}}async {{/isSyncClient}}def _func(_: Optional[Event]) -> Event:
3435
if response.event_id is None:
3536
raise ValueError(

0 commit comments

Comments
 (0)