12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
14#include "internal/gc.h"
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
20#ifdef HAVE_THR_STKSEGMENT
23#if defined(HAVE_FCNTL_H)
25#elif defined(HAVE_SYS_FCNTL_H)
28#ifdef HAVE_SYS_PRCTL_H
31#if defined(HAVE_SYS_TIME_H)
38#include <sys/syscall.h>
44# include <AvailabilityMacros.h>
47#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
48# define USE_EVENTFD (1)
49# include <sys/eventfd.h>
51# define USE_EVENTFD (0)
54#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
55 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
56 defined(HAVE_CLOCK_GETTIME)
57static pthread_condattr_t condattr_mono;
58static pthread_condattr_t *condattr_monotonic = &condattr_mono;
60static const void *
const condattr_monotonic = NULL;
65#ifndef HAVE_SYS_EVENT_H
66#define HAVE_SYS_EVENT_H 0
69#ifndef HAVE_SYS_EPOLL_H
70#define HAVE_SYS_EPOLL_H 0
78 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
81 #define USE_MN_THREADS 0
82 #elif HAVE_SYS_EPOLL_H
83 #include <sys/epoll.h>
84 #define USE_MN_THREADS 1
85 #elif HAVE_SYS_EVENT_H
86 #include <sys/event.h>
87 #define USE_MN_THREADS 1
89 #define USE_MN_THREADS 0
95#define NATIVE_MUTEX_LOCK_DEBUG 0
98mutex_debug(
const char *msg,
void *lock)
100 if (NATIVE_MUTEX_LOCK_DEBUG) {
102 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
104 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
105 fprintf(stdout,
"%s: %p\n", msg, lock);
106 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
114 mutex_debug(
"lock", lock);
115 if ((r = pthread_mutex_lock(lock)) != 0) {
124 mutex_debug(
"unlock", lock);
125 if ((r = pthread_mutex_unlock(lock)) != 0) {
134 mutex_debug(
"trylock", lock);
135 if ((r = pthread_mutex_trylock(lock)) != 0) {
149 int r = pthread_mutex_init(lock, 0);
150 mutex_debug(
"init", lock);
159 int r = pthread_mutex_destroy(lock);
160 mutex_debug(
"destroy", lock);
169 int r = pthread_cond_init(cond, condattr_monotonic);
178 int r = pthread_cond_destroy(cond);
199 r = pthread_cond_signal(cond);
200 }
while (r == EAGAIN);
211 r = pthread_cond_broadcast(cond);
212 }
while (r == EAGAIN);
221 int r = pthread_cond_wait(cond, mutex);
228native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex,
const rb_hrtime_t *abs)
240 rb_hrtime2timespec(&ts, abs);
241 r = pthread_cond_timedwait(cond, mutex, &ts);
242 }
while (r == EINTR);
244 if (r != 0 && r != ETIMEDOUT) {
252native_cond_timeout(rb_nativethread_cond_t *cond,
const rb_hrtime_t rel)
254 if (condattr_monotonic) {
255 return rb_hrtime_add(rb_hrtime_now(), rel);
261 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
268 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
269 native_cond_timedwait(cond, mutex, &hrmsec);
274static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
275static void rb_thread_execute_hooks(
rb_event_flag_t event, rb_thread_t *th);
296#define RB_INTERNAL_THREAD_HOOK(event, th) \
297 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
298 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
299 rb_thread_execute_hooks(event, th); \
302#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
305static rb_serial_t current_fork_gen = 1;
307#if defined(SIGVTALRM) && !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__)
308# define USE_UBF_LIST 1
311static void threadptr_trap_interrupt(rb_thread_t *);
313#ifdef HAVE_SCHED_YIELD
314#define native_thread_yield() (void)sched_yield()
316#define native_thread_yield() ((void)0)
322#define TIME_QUANTUM_MSEC (100)
323#define TIME_QUANTUM_USEC (TIME_QUANTUM_MSEC * 1000)
324#define TIME_QUANTUM_NSEC (TIME_QUANTUM_USEC * 1000)
326static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt);
327static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt);
328static void native_thread_assign(
struct rb_native_thread *nt, rb_thread_t *th);
330static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r);
331static void timer_thread_wakeup(
void);
332static void timer_thread_wakeup_locked(rb_vm_t *vm);
333static void timer_thread_wakeup_force(
void);
334static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th);
336#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s)
339th_has_dedicated_nt(
const rb_thread_t *th)
342 return th->nt->dedicated > 0;
347thread_sched_dump_(const
char *file,
int line, struct
rb_thread_sched *sched)
349 fprintf(stderr,
"@%s:%d running:%d\n", file, line, sched->running ? (
int)sched->running->serial : -1);
352 ccan_list_for_each(&sched->readyq, th, sched.node.readyq) {
353 i++;
if (i>10) rb_bug(
"too many");
354 fprintf(stderr,
" ready:%d (%sNT:%d)\n", th->serial,
355 th->nt ? (th->nt->dedicated ?
"D" :
"S") :
"x",
356 th->nt ? (
int)th->nt->serial : -1);
360#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s)
364ractor_sched_dump_(const
char *file,
int line, rb_vm_t *vm)
368 fprintf(stderr,
"ractor_sched_dump %s:%d\n", file, line);
371 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
373 if (i>10) rb_bug(
"!!");
374 fprintf(stderr,
" %d ready:%d\n", i, rb_ractor_id(r));
378#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__)
379#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__)
382thread_sched_lock_(
struct rb_thread_sched *sched, rb_thread_t *th,
const char *file,
int line)
387 RUBY_DEBUG_LOG2(file, line,
"th:%u prev_owner:%u", rb_th_serial(th), rb_th_serial(sched->lock_owner));
388 VM_ASSERT(sched->lock_owner == NULL);
389 sched->lock_owner = th;
391 RUBY_DEBUG_LOG2(file, line,
"th:%u", rb_th_serial(th));
396thread_sched_unlock_(
struct rb_thread_sched *sched, rb_thread_t *th,
const char *file,
int line)
398 RUBY_DEBUG_LOG2(file, line,
"th:%u", rb_th_serial(th));
401 VM_ASSERT(sched->lock_owner == th);
402 sched->lock_owner = NULL;
409thread_sched_set_lock_owner(
struct rb_thread_sched *sched, rb_thread_t *th)
411 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
414 sched->lock_owner = th;
419ASSERT_thread_sched_locked(
struct rb_thread_sched *sched, rb_thread_t *th)
425 VM_ASSERT(sched->lock_owner == th);
428 VM_ASSERT(sched->lock_owner != NULL);
433#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__)
434#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__)
438rb_ractor_serial(const rb_ractor_t *r) {
440 return rb_ractor_id(r);
448ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr)
451 VM_ASSERT(vm->ractor.sched.lock_owner == NULL);
452 VM_ASSERT(vm->ractor.sched.locked ==
false);
454 vm->ractor.sched.lock_owner = cr;
455 vm->ractor.sched.locked =
true;
460ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr)
463 VM_ASSERT(vm->ractor.sched.locked);
464 VM_ASSERT(vm->ractor.sched.lock_owner == cr);
466 vm->ractor.sched.locked =
false;
467 vm->ractor.sched.lock_owner = NULL;
472ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr,
const char *file,
int line)
477 RUBY_DEBUG_LOG2(file, line,
"cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner));
479 RUBY_DEBUG_LOG2(file, line,
"cr:%u", rb_ractor_serial(cr));
482 ractor_sched_set_locked(vm, cr);
486ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr,
const char *file,
int line)
488 RUBY_DEBUG_LOG2(file, line,
"cr:%u", rb_ractor_serial(cr));
490 ractor_sched_set_unlocked(vm, cr);
495ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr)
498 VM_ASSERT(vm->ractor.sched.locked);
499 VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr);
504ractor_sched_running_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
507 ccan_list_for_each(&vm->ractor.sched.running_threads, rth, sched.node.running_threads) {
508 if (rth == th)
return true;
515ractor_sched_running_threads_size(rb_vm_t *vm)
519 ccan_list_for_each(&vm->ractor.sched.running_threads, th, sched.node.running_threads) {
527ractor_sched_timeslice_threads_size(rb_vm_t *vm)
531 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
539ractor_sched_timeslice_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
542 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, rth, sched.node.timeslice_threads) {
543 if (rth == th)
return true;
548static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm);
549static void ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th);
553thread_sched_setup_running_threads(
struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm,
554 rb_thread_t *add_th, rb_thread_t *del_th, rb_thread_t *add_timeslice_th)
556#if USE_RUBY_DEBUG_LOG
557 unsigned int prev_running_cnt = vm->ractor.sched.running_cnt;
560 rb_thread_t *del_timeslice_th;
562 if (del_th && sched->is_running_timeslice) {
563 del_timeslice_th = del_th;
564 sched->is_running_timeslice =
false;
567 del_timeslice_th = NULL;
570 RUBY_DEBUG_LOG(
"+:%u -:%u +ts:%u -ts:%u",
571 rb_th_serial(add_th), rb_th_serial(del_th),
572 rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th));
574 ractor_sched_lock(vm, cr);
578 VM_ASSERT(ractor_sched_running_threads_contain_p(vm, del_th));
579 VM_ASSERT(del_timeslice_th != NULL ||
580 !ractor_sched_timeslice_threads_contain_p(vm, del_th));
582 ccan_list_del_init(&del_th->sched.node.running_threads);
583 vm->ractor.sched.running_cnt--;
585 if (UNLIKELY(vm->ractor.sched.barrier_waiting)) {
586 ractor_sched_barrier_join_signal_locked(vm);
588 sched->is_running =
false;
592 if (UNLIKELY(vm->ractor.sched.barrier_waiting)) {
593 RUBY_DEBUG_LOG(
"barrier-wait");
595 ractor_sched_barrier_join_signal_locked(vm);
596 ractor_sched_barrier_join_wait_locked(vm, add_th);
599 VM_ASSERT(!ractor_sched_running_threads_contain_p(vm, add_th));
600 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_th));
602 ccan_list_add(&vm->ractor.sched.running_threads, &add_th->sched.node.running_threads);
603 vm->ractor.sched.running_cnt++;
604 sched->is_running =
true;
607 if (add_timeslice_th) {
609 int was_empty = ccan_list_empty(&vm->ractor.sched.timeslice_threads);
610 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_timeslice_th));
611 ccan_list_add(&vm->ractor.sched.timeslice_threads, &add_timeslice_th->sched.node.timeslice_threads);
612 sched->is_running_timeslice =
true;
614 timer_thread_wakeup_locked(vm);
618 if (del_timeslice_th) {
619 VM_ASSERT(ractor_sched_timeslice_threads_contain_p(vm, del_timeslice_th));
620 ccan_list_del_init(&del_timeslice_th->sched.node.timeslice_threads);
623 VM_ASSERT(ractor_sched_running_threads_size(vm) == vm->ractor.sched.running_cnt);
624 VM_ASSERT(ractor_sched_timeslice_threads_size(vm) <= vm->ractor.sched.running_cnt);
626 ractor_sched_unlock(vm, cr);
628 if (add_th && !del_th && UNLIKELY(vm->ractor.sync.lock_owner != NULL)) {
630 rb_thread_t *lock_owner = NULL;
632 lock_owner = sched->lock_owner;
634 thread_sched_unlock(sched, lock_owner);
639 thread_sched_lock(sched, lock_owner);
645 RUBY_DEBUG_LOG(
"run:%u->%u", prev_running_cnt, vm->ractor.sched.running_cnt);
649thread_sched_add_running_thread(
struct rb_thread_sched *sched, rb_thread_t *th)
651 ASSERT_thread_sched_locked(sched, th);
652 VM_ASSERT(sched->running == th);
654 rb_vm_t *vm = th->vm;
655 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, ccan_list_empty(&sched->readyq) ? NULL : th);
659thread_sched_del_running_thread(
struct rb_thread_sched *sched, rb_thread_t *th)
661 ASSERT_thread_sched_locked(sched, th);
663 rb_vm_t *vm = th->vm;
664 thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th, NULL);
668rb_add_running_thread(rb_thread_t *th)
672 thread_sched_lock(sched, th);
674 thread_sched_add_running_thread(sched, th);
676 thread_sched_unlock(sched, th);
680rb_del_running_thread(rb_thread_t *th)
684 thread_sched_lock(sched, th);
686 thread_sched_del_running_thread(sched, th);
688 thread_sched_unlock(sched, th);
696thread_sched_set_running(
struct rb_thread_sched *sched, rb_thread_t *th)
698 RUBY_DEBUG_LOG(
"th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th));
699 VM_ASSERT(sched->running != th);
706thread_sched_readyq_contain_p(struct
rb_thread_sched *sched, rb_thread_t *th)
709 ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) {
710 if (rth == th)
return true;
722 ASSERT_thread_sched_locked(sched, NULL);
723 rb_thread_t *next_th;
725 VM_ASSERT(sched->running != NULL);
727 if (ccan_list_empty(&sched->readyq)) {
731 next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq);
733 VM_ASSERT(sched->readyq_cnt > 0);
735 ccan_list_node_init(&next_th->sched.node.readyq);
738 RUBY_DEBUG_LOG(
"next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt);
747 ASSERT_thread_sched_locked(sched, NULL);
748 RUBY_DEBUG_LOG(
"ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt);
750 VM_ASSERT(sched->running != NULL);
751 VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th));
753 if (sched->is_running) {
754 if (ccan_list_empty(&sched->readyq)) {
756 thread_sched_setup_running_threads(sched, ready_th->ractor, ready_th->vm, NULL, NULL, sched->running);
760 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(ready_th->vm, sched->running));
763 ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq);
770thread_sched_wakeup_running_thread(
struct rb_thread_sched *sched, rb_thread_t *next_th,
bool will_switch)
772 ASSERT_thread_sched_locked(sched, NULL);
773 VM_ASSERT(sched->running == next_th);
777 if (th_has_dedicated_nt(next_th)) {
778 RUBY_DEBUG_LOG(
"pinning th:%u", next_th->serial);
783 RUBY_DEBUG_LOG(
"th:%u is already running.", next_th->serial);
788 RUBY_DEBUG_LOG(
"th:%u (do nothing)", rb_th_serial(next_th));
791 RUBY_DEBUG_LOG(
"th:%u (enq)", rb_th_serial(next_th));
792 ractor_sched_enq(next_th->vm, next_th->ractor);
797 RUBY_DEBUG_LOG(
"no waiting threads%s",
"");
803thread_sched_to_ready_common(
struct rb_thread_sched *sched, rb_thread_t *th,
bool wakeup,
bool will_switch)
805 RUBY_DEBUG_LOG(
"th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt);
807 VM_ASSERT(sched->running != th);
808 VM_ASSERT(!thread_sched_readyq_contain_p(sched, th));
811 if (sched->running == NULL) {
812 thread_sched_set_running(sched, th);
813 if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch);
816 thread_sched_enq(sched, th);
828 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
830 thread_sched_lock(sched, th);
832 thread_sched_to_ready_common(sched, th,
true,
false);
834 thread_sched_unlock(sched, th);
839thread_sched_wait_running_turn(
struct rb_thread_sched *sched, rb_thread_t *th,
bool can_direct_transfer)
841 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
843 ASSERT_thread_sched_locked(sched, th);
844 VM_ASSERT(th == GET_THREAD());
846 if (th != sched->running) {
851 rb_thread_t *next_th;
852 while((next_th = sched->running) != th) {
853 if (th_has_dedicated_nt(th)) {
854 RUBY_DEBUG_LOG(
"(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running));
856 thread_sched_set_lock_owner(sched, NULL);
858 RUBY_DEBUG_LOG(
"nt:%d cond:%p", th->nt->serial, &th->nt->cond.readyq);
861 thread_sched_set_lock_owner(sched, th);
863 RUBY_DEBUG_LOG(
"(nt) wakeup %s", sched->running == th ?
"success" :
"failed");
864 if (th == sched->running) {
865 rb_ractor_thread_switch(th->ractor, th);
870 if (can_direct_transfer &&
871 (next_th = sched->running) != NULL &&
875 RUBY_DEBUG_LOG(
"th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th));
877 thread_sched_set_lock_owner(sched, NULL);
879 rb_ractor_set_current_ec(th->ractor, NULL);
880 thread_sched_switch(th, next_th);
882 thread_sched_set_lock_owner(sched, th);
887 native_thread_assign(NULL, th);
889 RUBY_DEBUG_LOG(
"th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th));
891 thread_sched_set_lock_owner(sched, NULL);
893 rb_ractor_set_current_ec(th->ractor, NULL);
894 coroutine_transfer(th->sched.context, nt->nt_context);
896 thread_sched_set_lock_owner(sched, th);
899 VM_ASSERT(GET_EC() == th->ec);
903 VM_ASSERT(th->nt != NULL);
904 VM_ASSERT(GET_EC() == th->ec);
905 VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none);
908 thread_sched_add_running_thread(sched, th);
917thread_sched_to_running_common(
struct rb_thread_sched *sched, rb_thread_t *th)
919 RUBY_DEBUG_LOG(
"th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th));
921 VM_ASSERT(sched->running != th);
922 VM_ASSERT(th_has_dedicated_nt(th));
923 VM_ASSERT(GET_THREAD() == th);
925 native_thread_dedicated_dec(th->vm, th->ractor, th->nt);
928 thread_sched_to_ready_common(sched, th,
false,
false);
930 if (sched->running == th) {
931 thread_sched_add_running_thread(sched, th);
935 thread_sched_wait_running_turn(sched, th,
false);
947 thread_sched_lock(sched, th);
949 thread_sched_to_running_common(sched, th);
951 thread_sched_unlock(sched, th);
962thread_sched_wakeup_next_thread(
struct rb_thread_sched *sched, rb_thread_t *th,
bool will_switch)
964 ASSERT_thread_sched_locked(sched, th);
966 VM_ASSERT(sched->running == th);
967 VM_ASSERT(sched->running->nt != NULL);
969 rb_thread_t *next_th = thread_sched_deq(sched);
971 RUBY_DEBUG_LOG(
"next_th:%u", rb_th_serial(next_th));
972 VM_ASSERT(th != next_th);
974 thread_sched_set_running(sched, next_th);
975 VM_ASSERT(next_th == sched->running);
976 thread_sched_wakeup_running_thread(sched, next_th, will_switch);
979 thread_sched_del_running_thread(sched, th);
992thread_sched_to_waiting_common0(
struct rb_thread_sched *sched, rb_thread_t *th,
bool to_dead)
996 if (!to_dead) native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
998 RUBY_DEBUG_LOG(
"%sth:%u", to_dead ?
"to_dead " :
"", rb_th_serial(th));
1000 bool can_switch = to_dead ? !th_has_dedicated_nt(th) : false;
1001 thread_sched_wakeup_next_thread(sched, th, can_switch);
1006thread_sched_to_dead_common(
struct rb_thread_sched *sched, rb_thread_t *th)
1008 RUBY_DEBUG_LOG(
"dedicated:%d", th->nt->dedicated);
1009 thread_sched_to_waiting_common0(sched, th,
true);
1017 thread_sched_lock(sched, th);
1019 thread_sched_to_dead_common(sched, th);
1021 thread_sched_unlock(sched, th);
1028thread_sched_to_waiting_common(
struct rb_thread_sched *sched, rb_thread_t *th)
1030 RUBY_DEBUG_LOG(
"dedicated:%d", th->nt->dedicated);
1031 thread_sched_to_waiting_common0(sched, th,
false);
1038thread_sched_to_waiting(
struct rb_thread_sched *sched, rb_thread_t *th)
1040 thread_sched_lock(sched, th);
1042 thread_sched_to_waiting_common(sched, th);
1044 thread_sched_unlock(sched, th);
1053 th->unblock.func = func;
1054 th->unblock.arg = arg;
1060ubf_waiting(
void *ptr)
1062 rb_thread_t *th = (rb_thread_t *)ptr;
1066 th->unblock.func = NULL;
1067 th->unblock.arg = NULL;
1069 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
1071 thread_sched_lock(sched, th);
1073 if (sched->running == th) {
1077 thread_sched_to_ready_common(sched, th,
true,
false);
1080 thread_sched_unlock(sched, th);
1087thread_sched_to_waiting_until_wakeup(
struct rb_thread_sched *sched, rb_thread_t *th)
1089 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
1091 RB_VM_SAVE_MACHINE_CONTEXT(th);
1092 setup_ubf(th, ubf_waiting, (
void *)th);
1096 thread_sched_lock(sched, th);
1098 if (!RUBY_VM_INTERRUPTED(th->ec)) {
1099 bool can_direct_transfer = !th_has_dedicated_nt(th);
1100 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1101 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1104 RUBY_DEBUG_LOG(
"th:%u interrupted", rb_th_serial(th));
1107 thread_sched_unlock(sched, th);
1109 setup_ubf(th, NULL, NULL);
1117 RUBY_DEBUG_LOG(
"th:%d sched->readyq_cnt:%d", (
int)th->serial, sched->readyq_cnt);
1119 thread_sched_lock(sched, th);
1121 if (!ccan_list_empty(&sched->readyq)) {
1123 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1124 bool can_direct_transfer = !th_has_dedicated_nt(th);
1125 thread_sched_to_ready_common(sched, th,
false, can_direct_transfer);
1126 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1129 VM_ASSERT(sched->readyq_cnt == 0);
1132 thread_sched_unlock(sched, th);
1141 sched->lock_owner = NULL;
1144 ccan_list_head_init(&sched->readyq);
1145 sched->readyq_cnt = 0;
1148 if (!atfork) sched->enable_mn_threads =
true;
1155 VM_ASSERT(!nt->dedicated);
1156 VM_ASSERT(next_th->nt == NULL);
1158 RUBY_DEBUG_LOG(
"next_th:%u", rb_th_serial(next_th));
1160 ruby_thread_set_native(next_th);
1161 native_thread_assign(nt, next_th);
1162 coroutine_transfer(current_cont, next_th->sched.context);
1166thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th)
1169 native_thread_assign(NULL, cth);
1170 RUBY_DEBUG_LOG(
"th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial);
1171 thread_sched_switch0(cth->sched.context, next_th, nt);
1174#if VM_CHECK_MODE > 0
1177grq_size(rb_vm_t *vm, rb_ractor_t *cr)
1179 ASSERT_ractor_sched_locked(vm, cr);
1181 rb_ractor_t *r, *prev_r = NULL;
1184 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
1187 VM_ASSERT(r != prev_r);
1195ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r)
1198 rb_ractor_t *cr = NULL;
1200 VM_ASSERT(sched->running != NULL);
1201 VM_ASSERT(sched->running->nt == NULL);
1203 ractor_sched_lock(vm, cr);
1205#if VM_CHECK_MODE > 0
1208 ccan_list_for_each(&vm->ractor.sched.grq,
tr, threads.sched.grq_node) {
1213 ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node);
1214 vm->ractor.sched.grq_cnt++;
1215 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1217 RUBY_DEBUG_LOG(
"r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt);
1223 ractor_sched_unlock(vm, cr);
1227#ifndef SNT_KEEP_SECONDS
1228#define SNT_KEEP_SECONDS 0
1233#define MINIMUM_SNT 0
1237ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr)
1241 ractor_sched_lock(vm, cr);
1243 RUBY_DEBUG_LOG(
"empty? %d", ccan_list_empty(&vm->ractor.sched.grq));
1246 VM_ASSERT(rb_current_execution_context(
false) == NULL);
1247 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1249 while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) {
1250 RUBY_DEBUG_LOG(
"wait grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1252#if SNT_KEEP_SECONDS > 0
1253 rb_hrtime_t abs = rb_hrtime_add(rb_hrtime_now(), RB_HRTIME_PER_SEC * SNT_KEEP_SECONDS);
1254 if (native_cond_timedwait(&vm->ractor.sched.cond, &vm->ractor.sched.lock, &abs) == ETIMEDOUT) {
1255 RUBY_DEBUG_LOG(
"timeout, grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1256 VM_ASSERT(r == NULL);
1257 vm->ractor.sched.snt_cnt--;
1258 vm->ractor.sched.running_cnt--;
1262 RUBY_DEBUG_LOG(
"wakeup grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1265 ractor_sched_set_unlocked(vm, cr);
1267 ractor_sched_set_locked(vm, cr);
1269 RUBY_DEBUG_LOG(
"wakeup grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1273 VM_ASSERT(rb_current_execution_context(
false) == NULL);
1276 VM_ASSERT(vm->ractor.sched.grq_cnt > 0);
1277 vm->ractor.sched.grq_cnt--;
1278 RUBY_DEBUG_LOG(
"r:%d grq_cnt:%u", (
int)rb_ractor_id(r), vm->ractor.sched.grq_cnt);
1281 VM_ASSERT(SNT_KEEP_SECONDS > 0);
1285 ractor_sched_unlock(vm, cr);
1290void rb_ractor_lock_self(rb_ractor_t *r);
1291void rb_ractor_unlock_self(rb_ractor_t *r);
1298 rb_thread_t *th = rb_ec_thread_ptr(ec);
1300 cr->sync.wait.waiting_thread = th;
1302 setup_ubf(th, ubf, (
void *)cr);
1304 thread_sched_lock(sched, th);
1306 rb_ractor_unlock_self(cr);
1308 if (RUBY_VM_INTERRUPTED(th->ec)) {
1309 RUBY_DEBUG_LOG(
"interrupted");
1311 else if (cr->sync.wait.wakeup_status != wakeup_none) {
1312 RUBY_DEBUG_LOG(
"awaken:%d", (
int)cr->sync.wait.wakeup_status);
1316 RB_VM_SAVE_MACHINE_CONTEXT(th);
1317 th->status = THREAD_STOPPED_FOREVER;
1321 bool can_direct_transfer = !th_has_dedicated_nt(th);
1322 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1323 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1324 th->status = THREAD_RUNNABLE;
1329 thread_sched_unlock(sched, th);
1331 setup_ubf(th, NULL, NULL);
1333 rb_ractor_lock_self(cr);
1334 cr->sync.wait.waiting_thread = NULL;
1338rb_ractor_sched_wakeup(rb_ractor_t *r)
1340 rb_thread_t *r_th = r->sync.wait.waiting_thread;
1344 VM_ASSERT(r->sync.wait.wakeup_status != 0);
1346 thread_sched_lock(sched, r_th);
1348 if (r_th->status == THREAD_STOPPED_FOREVER) {
1349 thread_sched_to_ready_common(sched, r_th,
true,
false);
1352 thread_sched_unlock(sched, r_th);
1356ractor_sched_barrier_completed_p(rb_vm_t *vm)
1358 RUBY_DEBUG_LOG(
"run:%u wait:%u", vm->ractor.sched.running_cnt, vm->ractor.sched.barrier_waiting_cnt);
1359 VM_ASSERT(vm->ractor.sched.running_cnt - 1 >= vm->ractor.sched.barrier_waiting_cnt);
1360 return (vm->ractor.sched.running_cnt - vm->ractor.sched.barrier_waiting_cnt) == 1;
1364rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr)
1366 VM_ASSERT(cr == GET_RACTOR());
1367 VM_ASSERT(vm->ractor.sync.lock_owner == cr);
1368 VM_ASSERT(!vm->ractor.sched.barrier_waiting);
1369 VM_ASSERT(vm->ractor.sched.barrier_waiting_cnt == 0);
1371 RUBY_DEBUG_LOG(
"start serial:%u", vm->ractor.sched.barrier_serial);
1373 unsigned int lock_rec;
1375 ractor_sched_lock(vm, cr);
1377 vm->ractor.sched.barrier_waiting =
true;
1380 lock_rec = vm->ractor.sync.lock_rec;
1381 vm->ractor.sync.lock_rec = 0;
1382 vm->ractor.sync.lock_owner = NULL;
1387 ccan_list_for_each(&vm->ractor.sched.running_threads, ith, sched.node.running_threads) {
1388 if (ith->ractor != cr) {
1389 RUBY_DEBUG_LOG(
"barrier int:%u", rb_th_serial(ith));
1390 RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec);
1395 while (!ractor_sched_barrier_completed_p(vm)) {
1396 ractor_sched_set_unlocked(vm, cr);
1398 ractor_sched_set_locked(vm, cr);
1402 ractor_sched_unlock(vm, cr);
1406 vm->ractor.sync.lock_rec = lock_rec;
1407 vm->ractor.sync.lock_owner = cr;
1409 RUBY_DEBUG_LOG(
"completed seirial:%u", vm->ractor.sched.barrier_serial);
1411 ractor_sched_lock(vm, cr);
1413 vm->ractor.sched.barrier_waiting =
false;
1414 vm->ractor.sched.barrier_serial++;
1415 vm->ractor.sched.barrier_waiting_cnt = 0;
1418 ractor_sched_unlock(vm, cr);
1422ractor_sched_barrier_join_signal_locked(rb_vm_t *vm)
1424 if (ractor_sched_barrier_completed_p(vm)) {
1430ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th)
1432 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1434 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1436 while (vm->ractor.sched.barrier_serial == barrier_serial) {
1437 RUBY_DEBUG_LOG(
"sleep serial:%u", barrier_serial);
1438 RB_VM_SAVE_MACHINE_CONTEXT(th);
1440 rb_ractor_t *cr = th->ractor;
1441 ractor_sched_set_unlocked(vm, cr);
1443 ractor_sched_set_locked(vm, cr);
1445 RUBY_DEBUG_LOG(
"wakeup serial:%u", barrier_serial);
1450rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr)
1452 VM_ASSERT(cr->threads.sched.running != NULL);
1453 VM_ASSERT(cr == GET_RACTOR());
1454 VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
1455 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1457#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0
1458 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1461 RUBY_DEBUG_LOG(
"join");
1465 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1466 VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial);
1468 ractor_sched_lock(vm, cr);
1471 vm->ractor.sched.barrier_waiting_cnt++;
1472 RUBY_DEBUG_LOG(
"waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_waiting_cnt, barrier_serial);
1474 ractor_sched_barrier_join_signal_locked(vm);
1475 ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running);
1477 ractor_sched_unlock(vm, cr);
1487static void clear_thread_cache_altstack(
void);
1500 clear_thread_cache_altstack();
1504#ifdef RB_THREAD_T_HAS_NATIVE_ID
1506get_native_thread_id(
void)
1509 return (
int)syscall(SYS_gettid);
1510#elif defined(__FreeBSD__)
1511 return pthread_getthreadid_np();
1516#if defined(HAVE_WORKING_FORK)
1521 rb_thread_sched_init(sched,
true);
1522 rb_thread_t *th = GET_THREAD();
1523 rb_vm_t *vm = GET_VM();
1525 if (th_has_dedicated_nt(th)) {
1526 vm->ractor.sched.snt_cnt = 0;
1529 vm->ractor.sched.snt_cnt = 1;
1531 vm->ractor.sched.running_cnt = 0;
1539 ccan_list_head_init(&vm->ractor.sched.grq);
1540 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1541 ccan_list_head_init(&vm->ractor.sched.running_threads);
1543 VM_ASSERT(sched->is_running);
1544 sched->is_running_timeslice =
false;
1546 if (sched->running != th) {
1547 thread_sched_to_running(sched, th);
1550 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, NULL);
1553#ifdef RB_THREAD_T_HAS_NATIVE_ID
1555 th->nt->tid = get_native_thread_id();
1562#ifdef RB_THREAD_LOCAL_SPECIFIER
1563static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
1565static pthread_key_t ruby_native_thread_key;
1577ruby_thread_from_native(
void)
1579#ifdef RB_THREAD_LOCAL_SPECIFIER
1580 return ruby_native_thread;
1582 return pthread_getspecific(ruby_native_thread_key);
1587ruby_thread_set_native(rb_thread_t *th)
1591 ccan_list_node_init(&th->sched.node.ubf);
1598 rb_ractor_set_current_ec(th->ractor, th->ec);
1600#ifdef RB_THREAD_LOCAL_SPECIFIER
1601 ruby_native_thread = th;
1604 return pthread_setspecific(ruby_native_thread_key, th) == 0;
1612Init_native_thread(rb_thread_t *main_th)
1614#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
1615 if (condattr_monotonic) {
1616 int r = pthread_condattr_init(condattr_monotonic);
1618 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
1620 if (r) condattr_monotonic = NULL;
1624#ifndef RB_THREAD_LOCAL_SPECIFIER
1625 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
1626 rb_bug(
"pthread_key_create failed (ruby_native_thread_key)");
1628 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
1629 rb_bug(
"pthread_key_create failed (ruby_current_ec_key)");
1632 ruby_posix_signal(SIGVTALRM, null_func);
1635 rb_vm_t *vm = main_th->vm;
1641 ccan_list_head_init(&vm->ractor.sched.grq);
1642 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1643 ccan_list_head_init(&vm->ractor.sched.running_threads);
1646 main_th->nt->thread_id = pthread_self();
1647 main_th->nt->serial = 1;
1648#ifdef RUBY_NT_SERIAL
1651 ruby_thread_set_native(main_th);
1652 native_thread_setup(main_th->nt);
1653 native_thread_setup_on_thread(main_th->nt);
1655 TH_SCHED(main_th)->running = main_th;
1656 main_th->has_dedicated_nt = 1;
1658 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL, NULL);
1661 main_th->nt->dedicated = 1;
1662 main_th->nt->vm = vm;
1665 vm->ractor.sched.dnt_cnt = 1;
1668extern int ruby_mn_threads_enabled;
1671ruby_mn_threads_params(
void)
1673 rb_vm_t *vm = GET_VM();
1674 rb_ractor_t *main_ractor = GET_RACTOR();
1676 const char *mn_threads_cstr = getenv(
"RUBY_MN_THREADS");
1677 bool enable_mn_threads =
false;
1679 if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) {
1681 ruby_mn_threads_enabled = 1;
1683 main_ractor->threads.sched.enable_mn_threads = enable_mn_threads;
1685 const char *max_cpu_cstr = getenv(
"RUBY_MAX_CPU");
1686 const int default_max_cpu = 8;
1687 int max_cpu = default_max_cpu;
1689 if (USE_MN_THREADS && max_cpu_cstr && (max_cpu = atoi(max_cpu_cstr)) > 0) {
1690 max_cpu = default_max_cpu;
1693 vm->ractor.sched.max_cpu = max_cpu;
1697native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt)
1699 RUBY_DEBUG_LOG(
"nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1);
1701 if (nt->dedicated == 0) {
1702 ractor_sched_lock(vm, cr);
1704 vm->ractor.sched.snt_cnt--;
1705 vm->ractor.sched.dnt_cnt++;
1707 ractor_sched_unlock(vm, cr);
1714native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt)
1716 RUBY_DEBUG_LOG(
"nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1);
1717 VM_ASSERT(nt->dedicated > 0);
1720 if (nt->dedicated == 0) {
1721 ractor_sched_lock(vm, cr);
1723 nt->vm->ractor.sched.snt_cnt++;
1724 nt->vm->ractor.sched.dnt_cnt--;
1726 ractor_sched_unlock(vm, cr);
1733#if USE_RUBY_DEBUG_LOG
1736 RUBY_DEBUG_LOG(
"th:%d nt:%d->%d", (
int)th->serial, (
int)th->nt->serial, (
int)nt->serial);
1739 RUBY_DEBUG_LOG(
"th:%d nt:NULL->%d", (
int)th->serial, (
int)nt->serial);
1744 RUBY_DEBUG_LOG(
"th:%d nt:%d->NULL", (
int)th->serial, (
int)th->nt->serial);
1747 RUBY_DEBUG_LOG(
"th:%d nt:NULL->NULL", (
int)th->serial);
1761 if (&nt->cond.readyq != &nt->cond.intr) {
1765 RB_ALTSTACK_FREE(nt->altstack);
1766 ruby_xfree(nt->nt_context);
1771#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
1772#define STACKADDR_AVAILABLE 1
1773#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
1774#define STACKADDR_AVAILABLE 1
1775#undef MAINSTACKADDR_AVAILABLE
1776#define MAINSTACKADDR_AVAILABLE 1
1777void *pthread_get_stackaddr_np(pthread_t);
1778size_t pthread_get_stacksize_np(pthread_t);
1779#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1780#define STACKADDR_AVAILABLE 1
1781#elif defined HAVE_PTHREAD_GETTHRDS_NP
1782#define STACKADDR_AVAILABLE 1
1783#elif defined __HAIKU__
1784#define STACKADDR_AVAILABLE 1
1787#ifndef MAINSTACKADDR_AVAILABLE
1788# ifdef STACKADDR_AVAILABLE
1789# define MAINSTACKADDR_AVAILABLE 1
1791# define MAINSTACKADDR_AVAILABLE 0
1794#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
1795# define get_main_stack(addr, size) get_stack(addr, size)
1798#ifdef STACKADDR_AVAILABLE
1803get_stack(
void **addr,
size_t *size)
1805#define CHECK_ERR(expr) \
1806 {int err = (expr); if (err) return err;}
1807#ifdef HAVE_PTHREAD_GETATTR_NP
1808 pthread_attr_t attr;
1810 STACK_GROW_DIR_DETECTION;
1811 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
1812# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1813 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1814 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
1816 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1817 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1819# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
1820 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
1822 guard = getpagesize();
1825 pthread_attr_destroy(&attr);
1826#elif defined HAVE_PTHREAD_ATTR_GET_NP
1827 pthread_attr_t attr;
1828 CHECK_ERR(pthread_attr_init(&attr));
1829 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
1830# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1831 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1833 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1834 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1836 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
1837 pthread_attr_destroy(&attr);
1838#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP)
1839 pthread_t th = pthread_self();
1840 *addr = pthread_get_stackaddr_np(th);
1841 *size = pthread_get_stacksize_np(th);
1842#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1844# if defined HAVE_THR_STKSEGMENT
1845 CHECK_ERR(thr_stksegment(&stk));
1847 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
1850 *size = stk.ss_size;
1851#elif defined HAVE_PTHREAD_GETTHRDS_NP
1852 pthread_t th = pthread_self();
1853 struct __pthrdsinfo thinfo;
1855 int regsiz=
sizeof(reg);
1856 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
1857 &thinfo,
sizeof(thinfo),
1859 *addr = thinfo.__pi_stackaddr;
1863 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
1864 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
1865#elif defined __HAIKU__
1867 STACK_GROW_DIR_DETECTION;
1868 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
1869 *addr = info.stack_base;
1870 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
1871 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
1873#error STACKADDR_AVAILABLE is defined but not implemented.
1881 rb_nativethread_id_t id;
1882 size_t stack_maxsize;
1884} native_main_thread;
1886#ifdef STACK_END_ADDRESS
1887extern void *STACK_END_ADDRESS;
1891 RUBY_STACK_SPACE_LIMIT = 1024 * 1024,
1892 RUBY_STACK_SPACE_RATIO = 5
1896space_size(
size_t stack_size)
1898 size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
1899 if (space_size > RUBY_STACK_SPACE_LIMIT) {
1900 return RUBY_STACK_SPACE_LIMIT;
1909reserve_stack(
volatile char *limit,
size_t size)
1912# error needs alloca()
1915 volatile char buf[0x100];
1916 enum {stack_check_margin = 0x1000};
1918 STACK_GROW_DIR_DETECTION;
1920 if (!getrlimit(RLIMIT_STACK, &rl) && rl.rlim_cur == RLIM_INFINITY)
1923 if (size < stack_check_margin)
return;
1924 size -= stack_check_margin;
1926 size -=
sizeof(buf);
1927 if (IS_STACK_DIR_UPPER()) {
1928 const volatile char *end = buf +
sizeof(buf);
1938 size_t sz = limit - end;
1953 size_t sz = buf - limit;
1960# define reserve_stack(limit, size) ((void)(limit), (void)(size))
1963#undef ruby_init_stack
1967 native_main_thread.id = pthread_self();
1969#if MAINSTACKADDR_AVAILABLE
1970 if (native_main_thread.stack_maxsize)
return;
1974 if (get_main_stack(&stackaddr, &size) == 0) {
1975 native_main_thread.stack_maxsize = size;
1976 native_main_thread.stack_start = stackaddr;
1977 reserve_stack(stackaddr, size);
1982#ifdef STACK_END_ADDRESS
1983 native_main_thread.stack_start = STACK_END_ADDRESS;
1985 if (!native_main_thread.stack_start ||
1986 STACK_UPPER((
VALUE *)(
void *)&addr,
1987 native_main_thread.stack_start > addr,
1988 native_main_thread.stack_start < addr)) {
1989 native_main_thread.stack_start = (
VALUE *)addr;
1993#if defined(HAVE_GETRLIMIT)
1994#if defined(PTHREAD_STACK_DEFAULT)
1995# if PTHREAD_STACK_DEFAULT < RUBY_STACK_SPACE*5
1996# error "PTHREAD_STACK_DEFAULT is too small"
1998 size_t size = PTHREAD_STACK_DEFAULT;
2000 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
2003 int pagesize = getpagesize();
2005 STACK_GROW_DIR_DETECTION;
2006 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
2007 size = (size_t)rlim.rlim_cur;
2009 addr = native_main_thread.stack_start;
2010 if (IS_STACK_DIR_UPPER()) {
2011 space = ((size_t)((
char *)addr + size) / pagesize) * pagesize - (size_t)addr;
2014 space = (size_t)addr - ((
size_t)((
char *)addr - size) / pagesize + 1) * pagesize;
2016 native_main_thread.stack_maxsize = space;
2020#if MAINSTACKADDR_AVAILABLE
2027 STACK_GROW_DIR_DETECTION;
2029 if (IS_STACK_DIR_UPPER()) {
2030 start = native_main_thread.stack_start;
2031 end = (
char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
2034 start = (
char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
2035 end = native_main_thread.stack_start;
2038 if ((
void *)addr < start || (
void *)addr > end) {
2040 native_main_thread.stack_start = (
VALUE *)addr;
2041 native_main_thread.stack_maxsize = 0;
2046#define CHECK_ERR(expr) \
2047 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
2050native_thread_init_stack(rb_thread_t *th)
2052 rb_nativethread_id_t curr = pthread_self();
2054 if (pthread_equal(curr, native_main_thread.id)) {
2055 th->ec->machine.stack_start = native_main_thread.stack_start;
2056 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
2059#ifdef STACKADDR_AVAILABLE
2060 if (th_has_dedicated_nt(th)) {
2064 if (get_stack(&start, &size) == 0) {
2065 uintptr_t diff = (uintptr_t)start - (uintptr_t)&curr;
2066 th->ec->machine.stack_start = (
VALUE *)&curr;
2067 th->ec->machine.stack_maxsize = size - diff;
2071 rb_raise(
rb_eNotImpError,
"ruby engine can initialize only in the main thread");
2080 struct rb_native_thread *nt;
2090 pthread_attr_t attr;
2092 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
2093 const size_t space = space_size(stack_size);
2095 nt->machine_stack_maxsize = stack_size - space;
2097#ifdef USE_SIGALTSTACK
2098 nt->altstack = rb_allocate_sigaltstack();
2101 CHECK_ERR(pthread_attr_init(&attr));
2103# ifdef PTHREAD_STACK_MIN
2104 RUBY_DEBUG_LOG(
"stack size: %lu", (
unsigned long)stack_size);
2105 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
2108# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
2109 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
2111 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
2113 err = pthread_create(&nt->thread_id, &attr, nt_start, nt);
2115 RUBY_DEBUG_LOG(
"nt:%d err:%d", (
int)nt->serial, err);
2117 CHECK_ERR(pthread_attr_destroy(&attr));
2128 if (&nt->cond.readyq != &nt->cond.intr) {
2137#ifdef RB_THREAD_T_HAS_NATIVE_ID
2138 nt->tid = get_native_thread_id();
2142 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
2146native_thread_alloc(
void)
2149 native_thread_setup(nt);
2155#if USE_RUBY_DEBUG_LOG
2163native_thread_create_dedicated(rb_thread_t *th)
2165 th->nt = native_thread_alloc();
2166 th->nt->vm = th->vm;
2167 th->nt->running_thread = th;
2168 th->nt->dedicated = 1;
2171 size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size /
sizeof(
VALUE);
2172 void *vm_stack = ruby_xmalloc(vm_stack_word_size *
sizeof(
VALUE));
2173 th->sched.malloc_stack =
true;
2174 rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size);
2175 th->sched.context_stack = vm_stack;
2178 thread_sched_to_ready(TH_SCHED(th), th);
2180 return native_thread_create0(th->nt);
2184call_thread_start_func_2(rb_thread_t *th)
2186 native_thread_init_stack(th);
2187 thread_start_func_2(th, th->ec->machine.stack_start);
2194 rb_vm_t *vm = nt->vm;
2196 native_thread_setup_on_thread(nt);
2199#ifdef RB_THREAD_T_HAS_NATIVE_ID
2200 nt->tid = get_native_thread_id();
2203#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL)
2204 ruby_nt_serial = nt->serial;
2207 RUBY_DEBUG_LOG(
"nt:%u", nt->serial);
2209 if (!nt->dedicated) {
2210 coroutine_initialize_main(nt->nt_context);
2214 if (nt->dedicated) {
2216 rb_thread_t *th = nt->running_thread;
2219 RUBY_DEBUG_LOG(
"on dedicated th:%u", rb_th_serial(th));
2220 ruby_thread_set_native(th);
2222 thread_sched_lock(sched, th);
2224 if (sched->running == th) {
2225 thread_sched_add_running_thread(sched, th);
2227 thread_sched_wait_running_turn(sched, th,
false);
2229 thread_sched_unlock(sched, th);
2232 call_thread_start_func_2(th);
2236 RUBY_DEBUG_LOG(
"check next");
2237 rb_ractor_t *r = ractor_sched_deq(vm, NULL);
2242 thread_sched_lock(sched, NULL);
2244 rb_thread_t *next_th = sched->running;
2246 if (next_th && next_th->nt == NULL) {
2247 RUBY_DEBUG_LOG(
"nt:%d next_th:%d", (
int)nt->serial, (
int)next_th->serial);
2248 thread_sched_switch0(nt->nt_context, next_th, nt);
2251 RUBY_DEBUG_LOG(
"no schedulable threads -- next_th:%p", next_th);
2254 thread_sched_unlock(sched, NULL);
2266static int native_thread_create_shared(rb_thread_t *th);
2269static void nt_free_stack(
void *mstack);
2273rb_threadptr_remove(rb_thread_t *th)
2276 if (th->sched.malloc_stack) {
2281 rb_vm_t *vm = th->vm;
2282 th->sched.finished =
false;
2286 ccan_list_add(&vm->ractor.sched.zombie_threads, &th->sched.node.zombie_threads);
2294rb_threadptr_sched_free(rb_thread_t *th)
2297 if (th->sched.malloc_stack) {
2299 ruby_xfree(th->sched.context_stack);
2300 native_thread_destroy(th->nt);
2303 nt_free_stack(th->sched.context_stack);
2307 if (th->sched.context) {
2308 ruby_xfree(th->sched.context);
2309 VM_ASSERT((th->sched.context = NULL) == NULL);
2312 ruby_xfree(th->sched.context_stack);
2313 native_thread_destroy(th->nt);
2320rb_thread_sched_mark_zombies(rb_vm_t *vm)
2322 if (!ccan_list_empty(&vm->ractor.sched.zombie_threads)) {
2323 rb_thread_t *zombie_th, *next_zombie_th;
2324 ccan_list_for_each_safe(&vm->ractor.sched.zombie_threads, zombie_th, next_zombie_th, sched.node.zombie_threads) {
2325 if (zombie_th->sched.finished) {
2326 ccan_list_del_init(&zombie_th->sched.node.zombie_threads);
2329 rb_gc_mark(zombie_th->self);
2336native_thread_create(rb_thread_t *th)
2338 VM_ASSERT(th->nt == 0);
2339 RUBY_DEBUG_LOG(
"th:%d has_dnt:%d", th->serial, th->has_dedicated_nt);
2342 if (!th->ractor->threads.sched.enable_mn_threads) {
2343 th->has_dedicated_nt = 1;
2346 if (th->has_dedicated_nt) {
2347 return native_thread_create_dedicated(th);
2350 return native_thread_create_shared(th);
2354#if USE_NATIVE_THREAD_PRIORITY
2357native_thread_apply_priority(rb_thread_t *th)
2359#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
2360 struct sched_param sp;
2362 int priority = 0 - th->priority;
2364 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
2365 max = sched_get_priority_max(policy);
2366 min = sched_get_priority_min(policy);
2368 if (min > priority) {
2371 else if (max < priority) {
2375 sp.sched_priority = priority;
2376 pthread_setschedparam(th->nt->thread_id, policy, &sp);
2387 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
2391ubf_pthread_cond_signal(
void *ptr)
2393 rb_thread_t *th = (rb_thread_t *)ptr;
2394 RUBY_DEBUG_LOG(
"th:%u on nt:%d", rb_th_serial(th), (
int)th->nt->serial);
2399native_cond_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2401 rb_nativethread_lock_t *lock = &th->interrupt_lock;
2402 rb_nativethread_cond_t *cond = &th->nt->cond.intr;
2412 const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC;
2414 THREAD_BLOCKING_BEGIN(th);
2417 th->unblock.func = ubf_pthread_cond_signal;
2418 th->unblock.arg = th;
2420 if (RUBY_VM_INTERRUPTED(th->ec)) {
2422 RUBY_DEBUG_LOG(
"interrupted before sleep th:%u", rb_th_serial(th));
2435 end = native_cond_timeout(cond, *rel);
2436 native_cond_timedwait(cond, lock, &end);
2439 th->unblock.func = 0;
2443 THREAD_BLOCKING_END(th);
2445 RUBY_DEBUG_LOG(
"done th:%u", rb_th_serial(th));
2449static CCAN_LIST_HEAD(ubf_list_head);
2450static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT;
2453ubf_list_atfork(
void)
2455 ccan_list_head_init(&ubf_list_head);
2461ubf_list_contain_p(rb_thread_t *th)
2463 rb_thread_t *list_th;
2464 ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) {
2465 if (list_th == th)
return true;
2472register_ubf_list(rb_thread_t *th)
2474 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
2475 struct ccan_list_node *node = &th->sched.node.ubf;
2477 VM_ASSERT(th->unblock.func != NULL);
2482 if (ccan_list_empty((
struct ccan_list_head*)node)) {
2483 VM_ASSERT(!ubf_list_contain_p(th));
2484 ccan_list_add(&ubf_list_head, node);
2489 timer_thread_wakeup();
2494unregister_ubf_list(rb_thread_t *th)
2496 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
2497 struct ccan_list_node *node = &th->sched.node.ubf;
2500 VM_ASSERT(th->unblock.func == NULL);
2502 if (!ccan_list_empty((
struct ccan_list_head*)node)) {
2505 VM_ASSERT(ubf_list_contain_p(th));
2506 ccan_list_del_init(node);
2517ubf_wakeup_thread(rb_thread_t *th)
2519 RUBY_DEBUG_LOG(
"th:%u thread_id:%p", rb_th_serial(th), (
void *)th->nt->thread_id);
2521 pthread_kill(th->nt->thread_id, SIGVTALRM);
2525ubf_select(
void *ptr)
2527 rb_thread_t *th = (rb_thread_t *)ptr;
2528 RUBY_DEBUG_LOG(
"wakeup th:%u", rb_th_serial(th));
2529 ubf_wakeup_thread(th);
2530 register_ubf_list(th);
2534ubf_threads_empty(
void)
2536 return ccan_list_empty(&ubf_list_head) != 0;
2540ubf_wakeup_all_threads(
void)
2542 if (!ubf_threads_empty()) {
2546 ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) {
2547 ubf_wakeup_thread(th);
2555#define register_ubf_list(th) (void)(th)
2556#define unregister_ubf_list(th) (void)(th)
2558static void ubf_wakeup_all_threads(
void) {
return; }
2559static bool ubf_threads_empty(
void) {
return true; }
2560#define ubf_list_atfork() do {} while (0)
2564#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
2567rb_thread_wakeup_timer_thread(
int sig)
2573 timer_thread_wakeup_force();
2576 if (system_working) {
2577 rb_vm_t *vm = GET_VM();
2578 rb_thread_t *main_th = vm->ractor.main_thread;
2581 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
2584 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
2586 if (vm->ubf_async_safe && main_th->unblock.func) {
2587 (main_th->unblock.func)(main_th->unblock.arg);
2594#define CLOSE_INVALIDATE_PAIR(expr) \
2595 close_invalidate_pair(expr,"close_invalidate: "#expr)
2597close_invalidate(
int *fdp,
const char *msg)
2602 if (close(fd) < 0) {
2603 async_bug_fd(msg,
errno, fd);
2608close_invalidate_pair(
int fds[2],
const char *msg)
2610 if (USE_EVENTFD && fds[0] == fds[1]) {
2612 close_invalidate(&fds[0], msg);
2615 close_invalidate(&fds[1], msg);
2616 close_invalidate(&fds[0], msg);
2626 oflags = fcntl(fd, F_GETFL);
2629 oflags |= O_NONBLOCK;
2630 err = fcntl(fd, F_SETFL, oflags);
2637setup_communication_pipe_internal(
int pipes[2])
2641 if (pipes[0] > 0 || pipes[1] > 0) {
2642 VM_ASSERT(pipes[0] > 0);
2643 VM_ASSERT(pipes[1] > 0);
2651#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
2652 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
2654 if (pipes[0] >= 0) {
2662 rb_bug(
"can not create communication pipe");
2666 set_nonblock(pipes[0]);
2667 set_nonblock(pipes[1]);
2670#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
2671# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
2676#if defined(__linux__)
2678#elif defined(__APPLE__)
2686static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
2689native_set_thread_name(rb_thread_t *th)
2691#ifdef SET_CURRENT_THREAD_NAME
2693 if (!
NIL_P(loc = th->name)) {
2694 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
2696 else if ((loc = threadptr_invoke_proc_location(th)) !=
Qnil) {
2698 char buf[THREAD_NAME_MAX];
2703 p = strrchr(name,
'/');
2711 if (
len >=
sizeof(buf)) {
2712 buf[
sizeof(buf)-2] =
'*';
2713 buf[
sizeof(buf)-1] =
'\0';
2715 SET_CURRENT_THREAD_NAME(buf);
2721native_set_another_thread_name(rb_nativethread_id_t thread_id,
VALUE name)
2723#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
2724 char buf[THREAD_NAME_MAX];
2726# if !defined SET_ANOTHER_THREAD_NAME
2727 if (!pthread_equal(pthread_self(), thread_id))
return;
2732 if (n >= (
int)
sizeof(buf)) {
2733 memcpy(buf, s,
sizeof(buf)-1);
2734 buf[
sizeof(buf)-1] =
'\0';
2738# if defined SET_ANOTHER_THREAD_NAME
2739 SET_ANOTHER_THREAD_NAME(thread_id, s);
2740# elif defined SET_CURRENT_THREAD_NAME
2741 SET_CURRENT_THREAD_NAME(s);
2746#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
2748native_thread_native_thread_id(rb_thread_t *target_th)
2750 if (!target_th->nt)
return Qnil;
2752#ifdef RB_THREAD_T_HAS_NATIVE_ID
2753 int tid = target_th->nt->tid;
2754 if (tid == 0)
return Qnil;
2756#elif defined(__APPLE__)
2762# if (!defined(MAC_OS_X_VERSION_10_6) || \
2763 (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_6) || \
2764 defined(__POWERPC__) )
2765 const bool no_pthread_threadid_np =
true;
2766# define NO_PTHREAD_MACH_THREAD_NP 1
2767# elif MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_6
2768 const bool no_pthread_threadid_np =
false;
2770# if !(defined(__has_attribute) && __has_attribute(availability))
2772 __attribute__((weak))
int pthread_threadid_np(pthread_t, uint64_t*);
2775 const bool no_pthread_threadid_np = !&pthread_threadid_np;
2777 if (no_pthread_threadid_np) {
2778 return ULL2NUM(pthread_mach_thread_np(pthread_self()));
2780# ifndef NO_PTHREAD_MACH_THREAD_NP
2781 int e = pthread_threadid_np(target_th->nt->thread_id, &tid);
2783 return ULL2NUM((
unsigned long long)tid);
2787# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
2789# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
2793 rb_serial_t created_fork_gen;
2794 pthread_t pthread_id;
2798#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
2801#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
2802#define EPOLL_EVENTS_MAX 0x10
2803 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
2804#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
2805#define KQUEUE_EVENTS_MAX 0x10
2806 struct kevent finished_events[KQUEUE_EVENTS_MAX];
2810 struct ccan_list_head waiting;
2811 pthread_mutex_t waiting_lock;
2813 .created_fork_gen = 0,
2816#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
2818static void timer_thread_check_timeslice(rb_vm_t *vm);
2819static int timer_thread_set_timeout(rb_vm_t *vm);
2820static void timer_thread_wakeup_thread(rb_thread_t *th);
2822#include "thread_pthread_mn.c"
2825timer_thread_set_timeout(rb_vm_t *vm)
2832 ractor_sched_lock(vm, NULL);
2834 if ( !ccan_list_empty(&vm->ractor.sched.timeslice_threads)
2835 || !ubf_threads_empty()
2836 || vm->ractor.sched.grq_cnt > 0
2839 RUBY_DEBUG_LOG(
"timeslice:%d ubf:%d grq:%d",
2840 !ccan_list_empty(&vm->ractor.sched.timeslice_threads),
2841 !ubf_threads_empty(),
2842 (vm->ractor.sched.grq_cnt > 0));
2845 vm->ractor.sched.timeslice_wait_inf =
false;
2848 vm->ractor.sched.timeslice_wait_inf =
true;
2851 ractor_sched_unlock(vm, NULL);
2853 if (vm->ractor.sched.timeslice_wait_inf) {
2856 rb_thread_t *th = ccan_list_top(&timer_th.waiting, rb_thread_t, sched.waiting_reason.node);
2857 if (th && (th->sched.waiting_reason.flags & thread_sched_waiting_timeout)) {
2858 rb_hrtime_t now = rb_hrtime_now();
2859 rb_hrtime_t hrrel = rb_hrtime_sub(th->sched.waiting_reason.data.timeout, now);
2861 RUBY_DEBUG_LOG(
"th:%u now:%lu rel:%lu", rb_th_serial(th), (
unsigned long)now, (
unsigned long)hrrel);
2864 timeout = (int)((hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC);
2870 RUBY_DEBUG_LOG(
"timeout:%d inf:%d", timeout, (
int)vm->ractor.sched.timeslice_wait_inf);
2878timer_thread_check_signal(rb_vm_t *vm)
2882 int signum = rb_signal_buff_size();
2883 if (UNLIKELY(signum > 0) && vm->ractor.main_thread) {
2884 RUBY_DEBUG_LOG(
"signum:%d", signum);
2885 threadptr_trap_interrupt(vm->ractor.main_thread);
2890timer_thread_check_exceed(rb_hrtime_t abs, rb_hrtime_t now)
2895 else if (abs - now < RB_HRTIME_PER_MSEC) {
2904timer_thread_deq_wakeup(rb_vm_t *vm, rb_hrtime_t now)
2906 rb_thread_t *th = ccan_list_top(&timer_th.waiting, rb_thread_t, sched.waiting_reason.node);
2909 (th->sched.waiting_reason.flags & thread_sched_waiting_timeout) &&
2910 timer_thread_check_exceed(th->sched.waiting_reason.data.timeout, now)) {
2912 RUBY_DEBUG_LOG(
"wakeup th:%u", rb_th_serial(th));
2915 ccan_list_del_init(&th->sched.waiting_reason.node);
2918 th->sched.waiting_reason.flags = thread_sched_waiting_none;
2919 th->sched.waiting_reason.data.result = 0;
2928timer_thread_wakeup_thread(rb_thread_t *th)
2930 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
2933 thread_sched_lock(sched, th);
2935 if (sched->running != th) {
2936 thread_sched_to_ready_common(sched, th,
true,
false);
2942 thread_sched_unlock(sched, th);
2946timer_thread_check_timeout(rb_vm_t *vm)
2948 rb_hrtime_t now = rb_hrtime_now();
2953 while ((th = timer_thread_deq_wakeup(vm, now)) != NULL) {
2954 timer_thread_wakeup_thread(th);
2961timer_thread_check_timeslice(rb_vm_t *vm)
2965 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
2966 RUBY_DEBUG_LOG(
"timeslice th:%u", rb_th_serial(th));
2967 RUBY_VM_SET_TIMER_INTERRUPT(th->ec);
2975 pthread_sigmask(0, NULL, &oldmask);
2976 if (sigismember(&oldmask, SIGVTALRM)) {
2980 RUBY_DEBUG_LOG(
"ok");
2985timer_thread_func(
void *ptr)
2987 rb_vm_t *vm = (rb_vm_t *)ptr;
2988#if defined(RUBY_NT_SERIAL)
2992 RUBY_DEBUG_LOG(
"started%s",
"");
2994 while (system_working) {
2995 timer_thread_check_signal(vm);
2996 timer_thread_check_timeout(vm);
2997 ubf_wakeup_all_threads();
2999 RUBY_DEBUG_LOG(
"system_working:%d", system_working);
3000 timer_thread_polling(vm);
3003 RUBY_DEBUG_LOG(
"terminated");
3009signal_communication_pipe(
int fd)
3012 const uint64_t buff = 1;
3014 const char buff =
'!';
3021 if ((result = write(fd, &buff,
sizeof(buff))) <= 0) {
3024 case EINTR:
goto retry;
3026#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
3031 async_bug_fd(
"rb_thread_wakeup_timer_thread: write", e, fd);
3034 if (TT_DEBUG) WRITE_CONST(2,
"rb_thread_wakeup_timer_thread: write\n");
3042timer_thread_wakeup_force(
void)
3045 signal_communication_pipe(timer_th.comm_fds[1]);
3049timer_thread_wakeup_locked(rb_vm_t *vm)
3052 ASSERT_ractor_sched_locked(vm, NULL);
3054 if (timer_th.created_fork_gen == current_fork_gen) {
3055 if (vm->ractor.sched.timeslice_wait_inf) {
3056 RUBY_DEBUG_LOG(
"wakeup with fd:%d", timer_th.comm_fds[1]);
3057 timer_thread_wakeup_force();
3060 RUBY_DEBUG_LOG(
"will be wakeup...");
3066timer_thread_wakeup(
void)
3068 rb_vm_t *vm = GET_VM();
3070 ractor_sched_lock(vm, NULL);
3072 timer_thread_wakeup_locked(vm);
3074 ractor_sched_unlock(vm, NULL);
3078rb_thread_create_timer_thread(
void)
3080 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
3082 RUBY_DEBUG_LOG(
"fork_gen create:%d current:%d", (
int)created_fork_gen, (
int)current_fork_gen);
3084 timer_th.created_fork_gen = current_fork_gen;
3086 if (created_fork_gen != current_fork_gen) {
3087 if (created_fork_gen != 0) {
3088 RUBY_DEBUG_LOG(
"forked child process");
3090 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
3091#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
3092 close_invalidate(&timer_th.event_fd,
"close event_fd");
3097 ccan_list_head_init(&timer_th.waiting);
3101 setup_communication_pipe_internal(timer_th.comm_fds);
3104 timer_thread_setup_mn();
3107 pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
3111native_stop_timer_thread(
void)
3114 stopped = --system_working <= 0;
3117 RUBY_DEBUG_LOG(
"wakeup send %d", timer_th.comm_fds[1]);
3118 timer_thread_wakeup_force();
3119 RUBY_DEBUG_LOG(
"wakeup sent");
3120 pthread_join(timer_th.pthread_id, NULL);
3123 if (TT_DEBUG) fprintf(stderr,
"stop timer thread\n");
3128native_reset_timer_thread(
void)
3133#ifdef HAVE_SIGALTSTACK
3135ruby_stack_overflowed_p(
const rb_thread_t *th,
const void *addr)
3139 const size_t water_mark = 1024 * 1024;
3140 STACK_GROW_DIR_DETECTION;
3142#ifdef STACKADDR_AVAILABLE
3143 if (get_stack(&base, &size) == 0) {
3145 if (pthread_equal(th->nt->thread_id, native_main_thread.id)) {
3147 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
3148 size = (size_t)rlim.rlim_cur;
3152 base = (
char *)base + STACK_DIR_UPPER(+size, -size);
3157 size = th->ec->machine.stack_maxsize;
3158 base = (
char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
3163 size /= RUBY_STACK_SPACE_RATIO;
3164 if (size > water_mark) size = water_mark;
3165 if (IS_STACK_DIR_UPPER()) {
3166 if (size > ~(
size_t)base+1) size = ~(size_t)base+1;
3167 if (addr > base && addr <= (
void *)((
char *)base + size))
return 1;
3170 if (size > (
size_t)base) size = (size_t)base;
3171 if (addr > (
void *)((
char *)base - size) && addr <= base)
return 1;
3181 if (fd < 0)
return 0;
3183 if (fd == timer_th.comm_fds[0] ||
3184 fd == timer_th.comm_fds[1]
3185#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
3186 || fd == timer_th.event_fd
3189 goto check_fork_gen;
3194 if (timer_th.created_fork_gen == current_fork_gen) {
3206 return pthread_self();
3209#if defined(USE_POLL) && !defined(HAVE_PPOLL)
3212ruby_ppoll(
struct pollfd *fds, nfds_t nfds,
3213 const struct timespec *ts,
const sigset_t *sigmask)
3220 if (ts->tv_sec > INT_MAX/1000)
3221 timeout_ms = INT_MAX;
3223 tmp = (int)(ts->tv_sec * 1000);
3225 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
3226 if (INT_MAX - tmp < tmp2)
3227 timeout_ms = INT_MAX;
3229 timeout_ms = (int)(tmp + tmp2);
3235 return poll(fds, nfds, timeout_ms);
3237# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
3250#define THREAD_BLOCKING_YIELD(th) do { \
3251 const rb_thread_t *next_th; \
3252 struct rb_thread_sched *sched = TH_SCHED(th); \
3253 RB_VM_SAVE_MACHINE_CONTEXT(th); \
3254 thread_sched_to_waiting(sched, (th)); \
3255 next_th = sched->running; \
3256 rb_native_mutex_unlock(&sched->lock_); \
3257 native_thread_yield(); \
3258 if (!next_th && rb_ractor_living_thread_num(th->ractor) > 1) { \
3259 native_thread_yield(); \
3263native_sleep(rb_thread_t *th, rb_hrtime_t *rel)
3267 RUBY_DEBUG_LOG(
"rel:%d", rel ? (
int)*rel : 0);
3269 if (th_has_dedicated_nt(th)) {
3270 native_cond_sleep(th, rel);
3273 thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel);
3277 thread_sched_to_waiting_until_wakeup(sched, th);
3280 RUBY_DEBUG_LOG(
"wakeup");
3285struct rb_internal_thread_event_hook {
3286 rb_internal_thread_event_callback callback;
3290 struct rb_internal_thread_event_hook *next;
3293static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3295rb_internal_thread_event_hook_t *
3298 rb_internal_thread_event_hook_t *hook =
ALLOC_N(rb_internal_thread_event_hook_t, 1);
3299 hook->callback = callback;
3300 hook->user_data = user_data;
3301 hook->event = internal_event;
3304 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3308 hook->next = rb_internal_thread_event_hooks;
3309 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
3311 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3321 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3325 bool success = FALSE;
3327 if (rb_internal_thread_event_hooks == hook) {
3328 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
3332 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3335 if (h->next == hook) {
3336 h->next = hook->next;
3340 }
while ((h = h->next));
3343 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3357 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
3361 if (rb_internal_thread_event_hooks) {
3362 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3364 if (h->event & event) {
3365 rb_internal_thread_event_data_t event_data = {
3368 (*h->callback)(event, &event_data, h->user_data);
3370 }
while((h = h->next));
3372 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
uint32_t rb_event_flag_t
Represents event(s).
#define INT2FIX
Old name of RB_INT2FIX.
#define ZALLOC
Old name of RB_ZALLOC.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define ULL2NUM
Old name of RB_ULL2NUM.
#define NUM2INT
Old name of RB_NUM2INT.
#define Qnil
Old name of RUBY_Qnil.
#define NIL_P
Old name of RB_NIL_P.
void ruby_init_stack(volatile VALUE *addr)
Set stack bottom of Ruby implementation.
VALUE rb_eNotImpError
NotImplementedError exception.
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
void rb_unblock_function_t(void *)
This is the type of UBFs.
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
int len
Length of the buffer.
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]]
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define rb_fd_select
Waits for multiple file descriptors at once.
#define RARRAY_AREF(a, i)
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
#define errno
Ractor-aware version of errno.
The data structure which wraps the fd_set bitmap used by select(2).
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
uintptr_t VALUE
Type that represents a Ruby object.