17#include "internal/compile.h"
18#include "internal/fixnum.h"
19#include "internal/hash.h"
20#include "internal/sanitizers.h"
21#include "internal/gc.h"
22#include "internal/proc.h"
24#include "vm_insnhelper.h"
26#include "probes_helper.h"
29#include "insns_info.inc"
39#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
42align_ptr(uint8_t *ptr, uint32_t multiple)
45 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
52 uint32_t pad = multiple - rem;
61rjit_reserve_addr_space(uint32_t mem_size)
67 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
68 uint32_t
const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
69 uint8_t *
const cfunc_sample_addr = (
void *)&rjit_reserve_addr_space;
70 uint8_t *
const probe_region_end = cfunc_sample_addr + INT32_MAX;
72 uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
81 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
87 if (mem_block != MAP_FAILED) {
92 req_addr += 4 * 1024 * 1024;
93 }
while (req_addr < probe_region_end);
99 (
void *)rjit_reserve_addr_space,
102 MAP_PRIVATE | MAP_ANONYMOUS,
109 if (mem_block == MAP_FAILED) {
115 MAP_PRIVATE | MAP_ANONYMOUS,
122 if (mem_block == MAP_FAILED) {
123 perror(
"ruby: yjit: mmap:");
124 if(
errno == ENOMEM) {
128 rb_bug(
"mmap failed");
139mprotect_write(rb_execution_context_t *ec,
VALUE self,
VALUE rb_mem_block,
VALUE rb_mem_size)
141 void *mem_block = (
void *)
NUM2SIZET(rb_mem_block);
142 uint32_t mem_size =
NUM2UINT(rb_mem_size);
143 return RBOOL(mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0);
147mprotect_exec(rb_execution_context_t *ec,
VALUE self,
VALUE rb_mem_block,
VALUE rb_mem_size)
149 void *mem_block = (
void *)
NUM2SIZET(rb_mem_block);
150 uint32_t mem_size =
NUM2UINT(rb_mem_size);
151 if (mem_size == 0)
return Qfalse;
153 if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
154 rb_bug(
"Couldn't make JIT page (%p, %lu bytes) executable, errno: %s",
155 mem_block, (
unsigned long)mem_size, strerror(
errno));
161rjit_optimized_call(
VALUE *recv, rb_execution_context_t *ec,
int argc,
VALUE *argv,
int kw_splat,
VALUE block_handler)
164 GetProcPtr(recv, proc);
165 return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
177 return rb_str_cat(str1, RSTRING_PTR(str2), RSTRING_LEN(str2));
181rjit_rb_ary_subseq_length(
VALUE ary,
long beg)
184 return rb_ary_subseq(ary, beg,
len);
191 int kw_len = kw_arg->keyword_len;
192 VALUE hash = rb_hash_new_with_size(kw_len);
194 for (
int i = 0; i < kw_len; i++) {
195 VALUE key = kw_arg->keywords[i];
196 VALUE val = *(sp - kw_len + i);
197 rb_hash_aset(hash, key, val);
206rjit_full_cfunc_return(rb_execution_context_t *ec,
VALUE return_value)
208 rb_control_frame_t *cfp = ec->cfp;
210 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
220 EXEC_EVENT_HOOK(ec,
RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
224 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
228 ec->cfp->sp[0] = return_value;
233rjit_get_proc_ptr(
VALUE procv)
236 GetProcPtr(procv, proc);
243extern VALUE rb_rjit_raw_samples;
244extern VALUE rb_rjit_line_samples;
247rjit_record_exit_stack(
const VALUE *exit_pc)
250 if (!rb_rjit_call_p)
return;
253 int insn = rb_vm_insn_addr2opcode((
void *)*exit_pc);
256 VALUE frames_buffer[BUFF_LEN] = { 0 };
257 int lines_buffer[BUFF_LEN] = { 0 };
267 int samples_length = stack_length + 3;
271 int prev_stack_len_index = (int)
RARRAY_LEN(rb_rjit_raw_samples) - samples_length;
272 VALUE prev_stack_len_obj;
273 if (
RARRAY_LEN(rb_rjit_raw_samples) >= samples_length &&
FIXNUM_P(prev_stack_len_obj =
RARRAY_AREF(rb_rjit_raw_samples, prev_stack_len_index))) {
274 int prev_stack_len =
NUM2INT(prev_stack_len_obj);
275 int idx = stack_length - 1;
276 int prev_frame_idx = 0;
277 bool seen_already =
true;
282 if (prev_stack_len == stack_length) {
284 VALUE current_frame = frames_buffer[idx];
285 VALUE prev_frame =
RARRAY_AREF(rb_rjit_raw_samples, prev_stack_len_index + prev_frame_idx + 1);
289 if (current_frame != prev_frame) {
290 seen_already =
false;
300 int prev_idx = (int)
RARRAY_LEN(rb_rjit_raw_samples) - 1;
302 int new_count = prev_count + 1;
304 rb_ary_store(rb_rjit_raw_samples, prev_idx,
INT2NUM(new_count));
305 rb_ary_store(rb_rjit_line_samples, prev_idx,
INT2NUM(new_count));
311 rb_ary_push(rb_rjit_raw_samples,
INT2NUM(stack_length));
312 rb_ary_push(rb_rjit_line_samples,
INT2NUM(stack_length));
314 int idx = stack_length - 1;
317 VALUE frame = frames_buffer[idx];
318 int line = lines_buffer[idx];
320 rb_ary_push(rb_rjit_raw_samples, frame);
321 rb_ary_push(rb_rjit_line_samples,
INT2NUM(line));
327 rb_ary_push(rb_rjit_raw_samples,
INT2NUM(insn));
331 int line = (int)
RARRAY_LEN(rb_rjit_line_samples) - 1;
332 rb_ary_push(rb_rjit_line_samples,
INT2NUM(line));
336 rb_ary_push(rb_rjit_raw_samples,
INT2NUM(1));
337 rb_ary_push(rb_rjit_line_samples,
INT2NUM(1));
347 if (
RTEST(rb_hash_aref(hash, frame_id))) {
351 VALUE frame_info = rb_hash_new();
364 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"name")), name);
365 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"file")), file);
366 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"samples")),
INT2NUM(0));
367 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"total_samples")),
INT2NUM(0));
368 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"edges")), rb_hash_new());
369 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"lines")), rb_hash_new());
372 rb_hash_aset(frame_info,
ID2SYM(rb_intern(
"line")), line);
375 rb_hash_aset(hash, frame_id, frame_info);
380rjit_exit_traces(
void)
382 int samples_len = (int)
RARRAY_LEN(rb_rjit_raw_samples);
385 VALUE result = rb_hash_new();
386 VALUE raw_samples = rb_ary_new_capa(samples_len);
387 VALUE line_samples = rb_ary_new_capa(samples_len);
388 VALUE frames = rb_hash_new();
393 while (idx < samples_len) {
398 rb_ary_push(raw_samples,
SIZET2NUM(num));
399 rb_ary_push(line_samples,
INT2NUM(line_num));
404 for (
int o = 0; o < num; o++) {
405 rjit_add_frame(frames,
RARRAY_AREF(rb_rjit_raw_samples, idx));
407 rb_ary_push(line_samples,
RARRAY_AREF(rb_rjit_line_samples, idx));
412 rb_ary_push(raw_samples,
RARRAY_AREF(rb_rjit_raw_samples, idx));
413 rb_ary_push(line_samples,
RARRAY_AREF(rb_rjit_line_samples, idx));
417 rb_ary_push(raw_samples,
RARRAY_AREF(rb_rjit_raw_samples, idx));
418 rb_ary_push(line_samples,
RARRAY_AREF(rb_rjit_line_samples, idx));
424 rb_hash_aset(result,
ID2SYM(rb_intern(
"raw")), raw_samples);
425 rb_hash_aset(result,
ID2SYM(rb_intern(
"lines")), line_samples);
426 rb_hash_aset(result,
ID2SYM(rb_intern(
"frames")), frames);
433#define OFFSETOF(ptr, member) RB_SIZE2NUM(((char *)&ptr.member - (char*)&ptr) * 8)
435#define SIZEOF(type) RB_SIZE2NUM(sizeof(type))
436#define SIGNED_TYPE_P(type) RBOOL((type)(-1) < (type)(1))
439static size_t rjit_insn_exits[VM_INSTRUCTION_SIZE] = { 0 };
444#ifdef HAVE_LIBCAPSTONE
445#include <capstone/capstone.h>
452 VALUE result = rb_ary_new();
453#ifdef HAVE_LIBCAPSTONE
456 if (cs_open(CS_ARCH_X86, CS_MODE_64, &handle) != CS_ERR_OK) {
464 size_t base_addr =
RTEST(test) ? 0 : from_addr;
465 size_t count = cs_disasm(handle, (
const uint8_t *)from_addr, to_addr - from_addr, base_addr, 0, &insns);
466 for (
size_t i = 0; i < count; i++) {
468 rb_ary_push(result, vals);
472 cs_free(insns, count);
480rjit_enabled_p(rb_execution_context_t *ec,
VALUE self)
482 return RBOOL(rb_rjit_enabled);
486for_each_iseq_i(
void *vstart,
void *vend,
size_t stride,
void *data)
490 for (; v != (
VALUE)vend; v += stride) {
491 void *ptr = asan_poisoned_object_p(v);
492 asan_unpoison_object(v,
false);
494 if (rb_obj_is_iseq(v)) {
495 extern VALUE rb_rjit_iseq_new(rb_iseq_t *iseq);
496 rb_iseq_t *iseq = (rb_iseq_t *)v;
497 rb_funcall(block, rb_intern(
"call"), 1, rb_rjit_iseq_new(iseq));
500 asan_poison_object_if(ptr, v);
506rjit_for_each_iseq(rb_execution_context_t *ec,
VALUE self,
VALUE block)
508 rb_objspace_each_objects(for_each_iseq_i, (
void *)block);
513extern ID rb_get_symbol_id(
VALUE name);
517extern VALUE rb_vm_get_ev_const(rb_execution_context_t *ec,
VALUE orig_klass,
ID id,
VALUE allow_nil);
518extern VALUE rb_vm_getclassvariable(
const rb_iseq_t *iseq,
const rb_control_frame_t *cfp,
ID id, ICVARC ic);
519extern VALUE rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t num,
const VALUE *ptr);
520extern VALUE rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t num,
const VALUE *ptr);
521extern VALUE rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t num,
const VALUE *ptr);
523extern bool rb_simple_iseq_p(
const rb_iseq_t *iseq);
524extern bool rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type,
VALUE obj,
VALUE v);
525extern bool rb_vm_ic_hit_p(IC ic,
const VALUE *reg_ep);
527extern void rb_vm_setinstancevariable(
const rb_iseq_t *iseq,
VALUE obj,
ID id,
VALUE val, IVC ic);
528extern VALUE rb_vm_throw(
const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state,
VALUE throwobj);
529extern VALUE rb_reg_new_ary(
VALUE ary,
int opt);
530extern void rb_vm_setclassvariable(
const rb_iseq_t *iseq,
const rb_control_frame_t *cfp,
ID id,
VALUE val, ICVARC ic);
532extern const rb_callable_method_entry_t *rb_callable_method_entry_or_negative(
VALUE klass,
ID mid);
533extern VALUE rb_vm_yield_with_cfunc(rb_execution_context_t *ec,
const struct rb_captured_block *captured,
int argc,
const VALUE *argv);
536extern void* rb_rjit_entry_stub_hit(
VALUE branch_stub);
537extern void* rb_rjit_branch_stub_hit(
VALUE branch_stub,
int sp_offset,
int target0_p);
538extern uint64_t rb_vm_insns_count;
540#include "rjit_c.rbinc"
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
int rb_profile_frames(int start, int limit, VALUE *buff, int *lines)
Queries mysterious "frame"s of the given range.
VALUE rb_profile_frame_full_label(VALUE frame)
Identical to rb_profile_frame_label(), except it returns a qualified result.
VALUE rb_profile_frame_absolute_path(VALUE frame)
Identical to rb_profile_frame_path(), except it tries to expand the returning path.
VALUE rb_profile_frame_path(VALUE frame)
Queries the path of the passed backtrace.
VALUE rb_profile_frame_first_lineno(VALUE frame)
Queries the first line of the method of the passed frame pointer.
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
uint32_t rb_event_flag_t
Represents event(s).
#define rb_str_new2
Old name of rb_str_new_cstr.
#define INT2FIX
Old name of RB_INT2FIX.
#define ID2SYM
Old name of RB_ID2SYM.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define NUM2UINT
Old name of RB_NUM2UINT.
#define LONG2NUM
Old name of RB_LONG2NUM.
#define Qtrue
Old name of RUBY_Qtrue.
#define NUM2INT
Old name of RB_NUM2INT.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define NIL_P
Old name of RB_NIL_P.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
#define NUM2SIZET
Old name of RB_NUM2SIZE.
VALUE rb_eRuntimeError
RuntimeError exception.
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
int len
Length of the buffer.
#define RARRAY_LEN
Just another name of rb_array_len.
#define RARRAY_AREF(a, i)
#define errno
Ractor-aware version of errno.
#define RTEST
This is an old name of RB_TEST.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.