@@ -422,8 +422,8 @@ static QINLINE void qthread_gotlock_empty_inner(qthread_shepherd_t *shep,
422
422
/* op */
423
423
if (maddr && (maddr != X -> addr )) {
424
424
* (aligned_t * )maddr = * (X -> addr );
425
- MACHINE_FENCE ;
426
425
}
426
+ MACHINE_FENCE ;
427
427
/* requeue */
428
428
qthread_debug (FEB_DETAILS , "m(%p), maddr(%p), recursive(%u): dQ 1 EFQ (%u releasing tid %u with %u), will fill\n" , m , maddr , recursive , qthread_id (), X -> waiter -> thread_id , * (X -> addr ));
429
429
qt_feb_schedule (X -> waiter , shep );
@@ -478,8 +478,8 @@ static QINLINE void qthread_gotlock_fill_inner(qthread_shepherd_t *shep,
478
478
/* op */
479
479
if (maddr && (maddr != X -> addr )) {
480
480
* (aligned_t * )maddr = * (X -> addr );
481
- MACHINE_FENCE ;
482
481
}
482
+ MACHINE_FENCE ;
483
483
/* schedule */
484
484
qthread_t * waiter = X -> waiter ;
485
485
qthread_debug (FEB_DETAILS , "shep(%u), m(%p), maddr(%p), recursive(%u): dQ one from FFWQ (%u releasing tid %u with %u)\n" , shep -> shepherd_id , m , maddr , recursive , qthread_id (), waiter -> thread_id , * (aligned_t * )maddr );
@@ -517,8 +517,8 @@ static QINLINE void qthread_gotlock_fill_inner(qthread_shepherd_t *shep,
517
517
/* op */
518
518
if (X -> addr && (X -> addr != maddr )) {
519
519
* (aligned_t * )(X -> addr ) = * (aligned_t * )maddr ;
520
- MACHINE_FENCE ;
521
520
}
521
+ MACHINE_FENCE ;
522
522
/* schedule */
523
523
qthread_t * waiter = X -> waiter ;
524
524
qthread_debug (FEB_DETAILS , "shep(%u), m(%p), maddr(%p), recursive(%u): dQ one from FFQ (%u releasing tid %u with %u)\n" , shep -> shepherd_id , m , maddr , recursive , qthread_id (), waiter -> thread_id , * (aligned_t * )maddr );
@@ -542,8 +542,8 @@ static QINLINE void qthread_gotlock_fill_inner(qthread_shepherd_t *shep,
542
542
/* op */
543
543
if (X -> addr && (X -> addr != maddr )) {
544
544
* (aligned_t * )(X -> addr ) = * (aligned_t * )maddr ;
545
- MACHINE_FENCE ;
546
545
}
546
+ MACHINE_FENCE ;
547
547
qthread_debug (FEB_DETAILS , "m(%p), maddr(%p), recursive(%u): dQ 1 EFQ (%u releasing tid %u with %u), will empty\n" , m , maddr , recursive , qthread_id (), X -> waiter -> thread_id , * (aligned_t * )maddr );
548
548
qt_feb_schedule (X -> waiter , shep );
549
549
FREE_ADDRRES (X );
@@ -776,8 +776,9 @@ int API_FUNC qthread_writeF(aligned_t *dest,
776
776
#endif /* ifdef LOCK_FREE_FEBS */
777
777
/* we have the lock on m, so... */
778
778
if (dest && (dest != src )) {
779
- memcpy ( dest , src , sizeof (aligned_t )) ;
779
+ * ( aligned_t * ) dest = * (aligned_t * ) src ;
780
780
}
781
+ MACHINE_FENCE ;
781
782
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p\n" , (shep -> current ) ? (shep -> current -> thread_id ) : UINT_MAX , dest , src );
782
783
if (m ) {
783
784
qthread_gotlock_fill (shep , m , alignedaddr );
@@ -877,9 +878,9 @@ int API_FUNC qthread_purge_to(aligned_t *restrict dest,
877
878
} /* END CRITICAL SECTION */
878
879
qt_hash_unlock (FEBbin );
879
880
#endif /* ifdef LOCK_FREE_FEBS */
881
+ MACHINE_FENCE ;
880
882
if (dest && (dest != src )) {
881
883
* (aligned_t * )dest = * (aligned_t * )src ;
882
- MACHINE_FENCE ;
883
884
}
884
885
if (m ) {
885
886
qthread_debug (FEB_BEHAVIOR , "dest=%p src=%p (tid=%i): waking waiters\n" , dest , src , qthread_id ());
@@ -1006,6 +1007,7 @@ int API_FUNC qthread_writeEF(aligned_t *restrict dest,
1006
1007
atomic_store_explicit (& me -> thread_state , QTHREAD_STATE_FEB_BLOCKED , memory_order_relaxed );
1007
1008
me -> rdata -> blockedon .addr = m ;
1008
1009
QTHREAD_WAIT_TIMER_START ();
1010
+ MACHINE_FENCE ;
1009
1011
qthread_back_to_master (me );
1010
1012
QTHREAD_WAIT_TIMER_STOP (me , febwait );
1011
1013
#ifdef QTHREAD_USE_EUREKAS
@@ -1015,8 +1017,8 @@ int API_FUNC qthread_writeEF(aligned_t *restrict dest,
1015
1017
} else {
1016
1018
if (dest && (dest != src )) {
1017
1019
* (aligned_t * )dest = * (aligned_t * )src ;
1018
- MACHINE_FENCE ;
1019
1020
}
1021
+ MACHINE_FENCE ;
1020
1022
qthread_debug (FEB_BEHAVIOR , "dest=%p, src=%p (tid=%i): succeeded! waking waiters...\n" , dest , src , me -> thread_id );
1021
1023
qthread_gotlock_fill (me -> rdata -> shepherd_ptr , m , alignedaddr );
1022
1024
}
@@ -1083,8 +1085,8 @@ int API_FUNC qthread_writeEF_nb(aligned_t *restrict dest,
1083
1085
} else {
1084
1086
if (dest && (dest != src )) {
1085
1087
* (aligned_t * )dest = * (aligned_t * )src ;
1086
- MACHINE_FENCE ;
1087
1088
}
1089
+ MACHINE_FENCE ;
1088
1090
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p\n" , me -> thread_id , dest , src );
1089
1091
qthread_gotlock_fill (me -> rdata -> shepherd_ptr , m , alignedaddr );
1090
1092
}
@@ -1153,8 +1155,8 @@ int API_FUNC qthread_writeFF(aligned_t *restrict dest,
1153
1155
if (m == NULL ) { /* already full! */
1154
1156
if (dest && (dest != src )) {
1155
1157
* (aligned_t * )dest = * (aligned_t * )src ;
1156
- MACHINE_FENCE ;
1157
1158
}
1159
+ MACHINE_FENCE ;
1158
1160
qthread_debug (FEB_BEHAVIOR , "dest=%p, src=%p (tid=%u): non-blocking success!\n" , dest , src , me -> thread_id );
1159
1161
} else if (m -> full != 1 ) { /* not full... so we must block */
1160
1162
QTHREAD_WAIT_TIMER_DECLARATION ;
@@ -1171,6 +1173,7 @@ int API_FUNC qthread_writeFF(aligned_t *restrict dest,
1171
1173
atomic_store_explicit (& me -> thread_state , QTHREAD_STATE_FEB_BLOCKED , memory_order_relaxed );
1172
1174
me -> rdata -> blockedon .addr = m ;
1173
1175
QTHREAD_WAIT_TIMER_START ();
1176
+ MACHINE_FENCE ;
1174
1177
qthread_back_to_master (me );
1175
1178
QTHREAD_WAIT_TIMER_STOP (me , febwait );
1176
1179
#ifdef QTHREAD_USE_EUREKAS
@@ -1180,8 +1183,8 @@ int API_FUNC qthread_writeFF(aligned_t *restrict dest,
1180
1183
} else { /* exists AND is empty... weird, but that's life */
1181
1184
if (dest && (dest != src )) {
1182
1185
* (aligned_t * )dest = * (aligned_t * )src ;
1183
- MACHINE_FENCE ;
1184
1186
}
1187
+ MACHINE_FENCE ;
1185
1188
qthread_debug (FEB_BEHAVIOR , "dest=%p, src=%p (tid=%u): succeeded!\n" , dest , src , me -> thread_id );
1186
1189
QTHREAD_FASTLOCK_UNLOCK (& m -> lock );
1187
1190
}
@@ -1250,9 +1253,9 @@ int API_FUNC qthread_readFF(aligned_t *restrict dest,
1250
1253
qthread_debug (FEB_DETAILS , "dest=%p, src=%p (tid=%u): data structure locked or null (m=%p)\n" , dest , src , me -> thread_id , m );
1251
1254
/* now m, if it exists, is locked - if m is NULL, then we're done! */
1252
1255
if (m == NULL ) { /* already full! */
1256
+ MACHINE_FENCE ;
1253
1257
if (dest && (dest != src )) {
1254
1258
* (aligned_t * )dest = * (aligned_t * )src ;
1255
- MACHINE_FENCE ;
1256
1259
}
1257
1260
qthread_debug (FEB_BEHAVIOR , "dest=%p, src=%p (tid=%u): non-blocking success!\n" , dest , src , me -> thread_id );
1258
1261
} else if (m -> full != 1 ) { /* not full... so we must block */
@@ -1271,15 +1274,16 @@ int API_FUNC qthread_readFF(aligned_t *restrict dest,
1271
1274
me -> rdata -> blockedon .addr = m ;
1272
1275
QTHREAD_WAIT_TIMER_START ();
1273
1276
qthread_back_to_master (me );
1277
+ MACHINE_FENCE ;
1274
1278
QTHREAD_WAIT_TIMER_STOP (me , febwait );
1275
1279
#ifdef QTHREAD_USE_EUREKAS
1276
1280
qt_eureka_check (0 );
1277
1281
#endif /* QTHREAD_USE_EUREKAS */
1278
1282
qthread_debug (FEB_BEHAVIOR , "dest=%p, src=%p (tid=%u): succeeded after waiting\n" , dest , src , me -> thread_id );
1279
1283
} else { /* exists AND is empty... weird, but that's life */
1284
+ MACHINE_FENCE ;
1280
1285
if (dest && (dest != src )) {
1281
1286
* (aligned_t * )dest = * (aligned_t * )src ;
1282
- MACHINE_FENCE ;
1283
1287
}
1284
1288
qthread_debug (FEB_BEHAVIOR , "dest=%p, src=%p (tid=%u): succeeded!\n" , dest , src , me -> thread_id );
1285
1289
QTHREAD_FASTLOCK_UNLOCK (& m -> lock );
@@ -1337,19 +1341,19 @@ int API_FUNC qthread_readFF_nb(aligned_t *restrict dest,
1337
1341
qthread_debug (FEB_DETAILS , "data structure locked\n" );
1338
1342
/* now m, if it exists, is locked - if m is NULL, then we're done! */
1339
1343
if (m == NULL ) { /* already full! */
1344
+ MACHINE_FENCE ;
1340
1345
if (dest && (dest != src )) {
1341
1346
* (aligned_t * )dest = * (aligned_t * )src ;
1342
- MACHINE_FENCE ;
1343
1347
}
1344
1348
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p\n" , me -> thread_id , dest , src );
1345
1349
} else if (m -> full != 1 ) { /* not full... so we must block */
1346
1350
qthread_debug (FEB_BEHAVIOR , "tid %u non-blocking fail\n" , me -> thread_id );
1347
1351
QTHREAD_FASTLOCK_UNLOCK (& m -> lock );
1348
1352
return QTHREAD_OPFAIL ;
1349
1353
} else { /* exists AND is empty... weird, but that's life */
1354
+ MACHINE_FENCE ;
1350
1355
if (dest && (dest != src )) {
1351
1356
* (aligned_t * )dest = * (aligned_t * )src ;
1352
- MACHINE_FENCE ;
1353
1357
}
1354
1358
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p\n" , me -> thread_id , dest , src );
1355
1359
QTHREAD_FASTLOCK_UNLOCK (& m -> lock );
@@ -1455,15 +1459,16 @@ int API_FUNC qthread_readFE(aligned_t *restrict dest,
1455
1459
me -> rdata -> blockedon .addr = m ;
1456
1460
QTHREAD_WAIT_TIMER_START ();
1457
1461
qthread_back_to_master (me );
1462
+ MACHINE_FENCE ;
1458
1463
QTHREAD_WAIT_TIMER_STOP (me , febwait );
1459
1464
#ifdef QTHREAD_USE_EUREKAS
1460
1465
qt_eureka_check (0 );
1461
1466
#endif /* QTHREAD_USE_EUREKAS */
1462
1467
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p after waiting\n" , me -> thread_id , dest , src );
1463
1468
} else { /* full, thus IT IS OURS! MUAHAHAHA! */
1469
+ MACHINE_FENCE ;
1464
1470
if (dest && (dest != src )) {
1465
1471
* (aligned_t * )dest = * (aligned_t * )src ;
1466
- MACHINE_FENCE ;
1467
1472
}
1468
1473
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p\n" , me -> thread_id , dest , src );
1469
1474
qthread_gotlock_empty (me -> rdata -> shepherd_ptr , m , (void * )alignedaddr );
@@ -1482,6 +1487,7 @@ int API_FUNC qthread_readXX(aligned_t *restrict dest,
1482
1487
{ /*{{{ */
1483
1488
qthread_debug (FEB_CALLS , "dest=%p, src=%p\n" , dest , src );
1484
1489
1490
+ MACHINE_FENCE ;
1485
1491
if (dest && (dest != src )) {
1486
1492
* (aligned_t * )dest = * (aligned_t * )src ;
1487
1493
}
@@ -1556,9 +1562,9 @@ int API_FUNC qthread_readFE_nb(aligned_t *restrict dest,
1556
1562
QTHREAD_FASTLOCK_UNLOCK (& m -> lock );
1557
1563
return QTHREAD_OPFAIL ;
1558
1564
} else { /* full, thus IT IS OURS! MUAHAHAHA! */
1565
+ MACHINE_FENCE ;
1559
1566
if (dest && (dest != src )) {
1560
1567
* (aligned_t * )dest = * (aligned_t * )src ;
1561
- MACHINE_FENCE ;
1562
1568
}
1563
1569
qthread_debug (FEB_BEHAVIOR , "tid %u succeeded on %p=%p\n" , me -> thread_id , dest , src );
1564
1570
qthread_gotlock_empty (me -> rdata -> shepherd_ptr , m , (void * )alignedaddr );
0 commit comments