1#![allow(static_mut_refs)]
5
6use crate::vmcontext::{VMFunctionContext, VMTrampoline};
10use crate::{Trap, VMContext, VMFunctionBody};
11use backtrace::Backtrace;
12use core::ptr::{read, read_unaligned};
13use corosensei::stack::DefaultStack;
14use corosensei::trap::{CoroutineTrapHandler, TrapHandlerRegs};
15use corosensei::{CoroutineResult, ScopedCoroutine, Yielder};
16use scopeguard::defer;
17use std::any::Any;
18use std::cell::Cell;
19use std::error::Error;
20use std::io;
21use std::mem;
22#[cfg(unix)]
23use std::mem::MaybeUninit;
24use std::ptr::{self, NonNull};
25use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering, compiler_fence};
26use std::sync::{LazyLock, Once};
27use wasmer_types::TrapCode;
28
29pub struct VMConfig {
32 pub wasm_stack_size: Option<usize>,
34}
35
36static MAGIC: u8 = 0xc0;
40
41static DEFAULT_STACK_SIZE: AtomicUsize = AtomicUsize::new(1024 * 1024);
42
43#[repr(C)]
46#[cfg(all(target_arch = "aarch64", target_os = "macos"))]
47#[allow(non_camel_case_types)]
48struct ucontext_t {
49 uc_onstack: libc::c_int,
50 uc_sigmask: libc::sigset_t,
51 uc_stack: libc::stack_t,
52 uc_link: *mut libc::ucontext_t,
53 uc_mcsize: usize,
54 uc_mcontext: libc::mcontext_t,
55}
56
57#[repr(C)]
60#[cfg(all(target_arch = "aarch64", target_os = "freebsd"))]
61#[allow(non_camel_case_types)]
62struct ucontext_t {
63 uc_sigmask: libc::sigset_t,
64 uc_mcontext: libc::mcontext_t,
65 uc_link: *mut ucontext_t,
66 uc_stack: libc::stack_t,
67 uc_flags: libc::c_int,
68 spare: [libc::c_int; 4],
69}
70
71#[cfg(all(
72 unix,
73 not(all(target_arch = "aarch64", target_os = "macos")),
74 not(all(target_arch = "aarch64", target_os = "freebsd"))
75))]
76use libc::ucontext_t;
77
78pub fn set_stack_size(size: usize) {
80 DEFAULT_STACK_SIZE.store(size.clamp(8 * 1024, 100 * 1024 * 1024), Ordering::Relaxed);
81}
82
83cfg_if::cfg_if! {
84 if #[cfg(unix)] {
85 pub type TrapHandlerFn<'a> = dyn Fn(libc::c_int, *const libc::siginfo_t, *const libc::c_void) -> bool + Send + Sync + 'a;
87 } else if #[cfg(target_os = "windows")] {
88 pub type TrapHandlerFn<'a> = dyn Fn(*mut windows_sys::Win32::System::Diagnostics::Debug::EXCEPTION_POINTERS) -> bool + Send + Sync + 'a;
90 }
91}
92
93unsafe fn process_illegal_op(addr: usize) -> Option<TrapCode> {
95 unsafe {
96 let mut val: Option<u8> = None;
97 if cfg!(target_arch = "x86_64") {
98 val = if read(addr as *mut u8) & 0xf0 == 0x40
99 && read((addr + 1) as *mut u8) == 0x0f
100 && read((addr + 2) as *mut u8) == 0xb9
101 {
102 Some(read((addr + 3) as *mut u8))
103 } else if read(addr as *mut u8) == 0x0f && read((addr + 1) as *mut u8) == 0xb9 {
104 Some(read((addr + 2) as *mut u8))
105 } else {
106 None
107 }
108 }
109 if cfg!(target_arch = "aarch64") {
110 val = if read_unaligned(addr as *mut u32) & 0xffff0000 == 0 {
111 Some(read(addr as *mut u8))
112 } else {
113 None
114 }
115 }
116 match val.and_then(|val| {
117 if val & MAGIC == MAGIC {
118 Some(val & 0xf)
119 } else {
120 None
121 }
122 }) {
123 None => None,
124 Some(val) => match val {
125 0 => Some(TrapCode::StackOverflow),
126 1 => Some(TrapCode::HeapAccessOutOfBounds),
127 2 => Some(TrapCode::HeapMisaligned),
128 3 => Some(TrapCode::TableAccessOutOfBounds),
129 4 => Some(TrapCode::IndirectCallToNull),
130 5 => Some(TrapCode::BadSignature),
131 6 => Some(TrapCode::IntegerOverflow),
132 7 => Some(TrapCode::IntegerDivisionByZero),
133 8 => Some(TrapCode::BadConversionToInteger),
134 9 => Some(TrapCode::UnreachableCodeReached),
135 10 => Some(TrapCode::UnalignedAtomic),
136 _ => None,
137 },
138 }
139 }
140}
141
142cfg_if::cfg_if! {
143 if #[cfg(unix)] {
144 static mut PREV_SIGSEGV: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
145 static mut PREV_SIGBUS: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
146 static mut PREV_SIGILL: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
147 static mut PREV_SIGFPE: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
148
149 unsafe fn platform_init() { unsafe {
150 let register = |slot: &mut MaybeUninit<libc::sigaction>, signal: i32| {
151 let mut handler: libc::sigaction = mem::zeroed();
152 handler.sa_flags = libc::SA_SIGINFO | libc::SA_NODEFER | libc::SA_ONSTACK;
166 handler.sa_sigaction = trap_handler as usize;
167 libc::sigemptyset(&mut handler.sa_mask);
168 if libc::sigaction(signal, &handler, slot.as_mut_ptr()) != 0 {
169 panic!(
170 "unable to install signal handler: {}",
171 io::Error::last_os_error(),
172 );
173 }
174 };
175
176 register(&mut PREV_SIGSEGV, libc::SIGSEGV);
178
179 register(&mut PREV_SIGILL, libc::SIGILL);
181
182 if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") {
184 register(&mut PREV_SIGFPE, libc::SIGFPE);
185 }
186
187 if cfg!(target_arch = "arm") || cfg!(target_vendor = "apple") {
190 register(&mut PREV_SIGBUS, libc::SIGBUS);
191 }
192
193 #[cfg(target_vendor = "apple")]
196 {
197 use mach2::exception_types::*;
198 use mach2::kern_return::*;
199 use mach2::port::*;
200 use mach2::thread_status::*;
201 use mach2::traps::*;
202 use mach2::mach_types::*;
203
204 unsafe extern "C" {
205 fn task_set_exception_ports(
206 task: task_t,
207 exception_mask: exception_mask_t,
208 new_port: mach_port_t,
209 behavior: exception_behavior_t,
210 new_flavor: thread_state_flavor_t,
211 ) -> kern_return_t;
212 }
213
214 #[allow(non_snake_case)]
215 #[cfg(target_arch = "x86_64")]
216 let MACHINE_THREAD_STATE = x86_THREAD_STATE64;
217 #[allow(non_snake_case)]
218 #[cfg(target_arch = "aarch64")]
219 let MACHINE_THREAD_STATE = 6;
220
221 task_set_exception_ports(
222 mach_task_self(),
223 EXC_MASK_BAD_ACCESS | EXC_MASK_ARITHMETIC | EXC_MASK_BAD_INSTRUCTION,
224 MACH_PORT_NULL,
225 EXCEPTION_STATE_IDENTITY as exception_behavior_t,
226 MACHINE_THREAD_STATE,
227 );
228 }
229 }}
230
231 unsafe extern "C" fn trap_handler(
232 signum: libc::c_int,
233 siginfo: *mut libc::siginfo_t,
234 context: *mut libc::c_void,
235 ) { unsafe {
236 let previous = match signum {
237 libc::SIGSEGV => &PREV_SIGSEGV,
238 libc::SIGBUS => &PREV_SIGBUS,
239 libc::SIGFPE => &PREV_SIGFPE,
240 libc::SIGILL => &PREV_SIGILL,
241 _ => panic!("unknown signal: {signum}"),
242 };
243 let maybe_fault_address = match signum {
245 libc::SIGSEGV | libc::SIGBUS => {
246 Some((*siginfo).si_addr() as usize)
247 }
248 _ => None,
249 };
250 let trap_code = match signum {
251 libc::SIGILL => {
253 let addr = (*siginfo).si_addr() as usize;
254 process_illegal_op(addr)
255 }
256 _ => None,
257 };
258 let ucontext = &mut *(context as *mut ucontext_t);
259 let (pc, sp) = get_pc_sp(ucontext);
260 let handled = TrapHandlerContext::handle_trap(
261 pc,
262 sp,
263 maybe_fault_address,
264 trap_code,
265 |regs| update_context(ucontext, regs),
266 |handler| handler(signum, siginfo, context),
267 );
268
269 if handled {
270 return;
271 }
272
273 let previous = &*previous.as_ptr();
283 if previous.sa_flags & libc::SA_SIGINFO != 0 {
284 mem::transmute::<
285 usize,
286 extern "C" fn(libc::c_int, *mut libc::siginfo_t, *mut libc::c_void),
287 >(previous.sa_sigaction)(signum, siginfo, context)
288 } else if previous.sa_sigaction == libc::SIG_DFL
289 {
290 libc::sigaction(signum, previous, ptr::null_mut());
291 } else if previous.sa_sigaction != libc::SIG_IGN {
292 mem::transmute::<usize, extern "C" fn(libc::c_int)>(
293 previous.sa_sigaction
294 )(signum)
295 }
296 }}
297
298 unsafe fn get_pc_sp(context: &ucontext_t) -> (usize, usize) {
299 let (pc, sp);
300 cfg_if::cfg_if! {
301 if #[cfg(all(
302 any(target_os = "linux", target_os = "android"),
303 target_arch = "x86_64",
304 ))] {
305 pc = context.uc_mcontext.gregs[libc::REG_RIP as usize] as usize;
306 sp = context.uc_mcontext.gregs[libc::REG_RSP as usize] as usize;
307 } else if #[cfg(all(
308 any(target_os = "linux", target_os = "android"),
309 target_arch = "x86",
310 ))] {
311 pc = context.uc_mcontext.gregs[libc::REG_EIP as usize] as usize;
312 sp = context.uc_mcontext.gregs[libc::REG_ESP as usize] as usize;
313 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86"))] {
314 pc = context.uc_mcontext.mc_eip as usize;
315 sp = context.uc_mcontext.mc_esp as usize;
316 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] {
317 pc = context.uc_mcontext.mc_rip as usize;
318 sp = context.uc_mcontext.mc_rsp as usize;
319 } else if #[cfg(all(target_vendor = "apple", target_arch = "x86_64"))] {
320 let mcontext = unsafe { &*context.uc_mcontext };
321 pc = mcontext.__ss.__rip as usize;
322 sp = mcontext.__ss.__rsp as usize;
323 } else if #[cfg(all(
324 any(target_os = "linux", target_os = "android"),
325 target_arch = "aarch64",
326 ))] {
327 pc = context.uc_mcontext.pc as usize;
328 sp = context.uc_mcontext.sp as usize;
329 } else if #[cfg(all(
330 any(target_os = "linux", target_os = "android"),
331 target_arch = "arm",
332 ))] {
333 pc = context.uc_mcontext.arm_pc as usize;
334 sp = context.uc_mcontext.arm_sp as usize;
335 } else if #[cfg(all(
336 any(target_os = "linux", target_os = "android"),
337 any(target_arch = "riscv64", target_arch = "riscv32"),
338 ))] {
339 pc = context.uc_mcontext.__gregs[libc::REG_PC] as usize;
340 sp = context.uc_mcontext.__gregs[libc::REG_SP] as usize;
341 } else if #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] {
342 let mcontext = unsafe { &*context.uc_mcontext };
343 pc = mcontext.__ss.__pc as usize;
344 sp = mcontext.__ss.__sp as usize;
345 } else if #[cfg(all(target_os = "freebsd", target_arch = "aarch64"))] {
346 pc = context.uc_mcontext.mc_gpregs.gp_elr as usize;
347 sp = context.uc_mcontext.mc_gpregs.gp_sp as usize;
348 } else if #[cfg(all(target_os = "linux", target_arch = "loongarch64"))] {
349 pc = context.uc_mcontext.__gregs[1] as usize;
350 sp = context.uc_mcontext.__gregs[3] as usize;
351 } else {
352 compile_error!("Unsupported platform");
353 }
354 };
355 (pc, sp)
356 }
357
358 unsafe fn update_context(context: &mut ucontext_t, regs: TrapHandlerRegs) {
359 cfg_if::cfg_if! {
360 if #[cfg(all(
361 any(target_os = "linux", target_os = "android"),
362 target_arch = "x86_64",
363 ))] {
364 let TrapHandlerRegs { rip, rsp, rbp, rdi, rsi } = regs;
365 context.uc_mcontext.gregs[libc::REG_RIP as usize] = rip as i64;
366 context.uc_mcontext.gregs[libc::REG_RSP as usize] = rsp as i64;
367 context.uc_mcontext.gregs[libc::REG_RBP as usize] = rbp as i64;
368 context.uc_mcontext.gregs[libc::REG_RDI as usize] = rdi as i64;
369 context.uc_mcontext.gregs[libc::REG_RSI as usize] = rsi as i64;
370 } else if #[cfg(all(
371 any(target_os = "linux", target_os = "android"),
372 target_arch = "x86",
373 ))] {
374 let TrapHandlerRegs { eip, esp, ebp, ecx, edx } = regs;
375 context.uc_mcontext.gregs[libc::REG_EIP as usize] = eip as i32;
376 context.uc_mcontext.gregs[libc::REG_ESP as usize] = esp as i32;
377 context.uc_mcontext.gregs[libc::REG_EBP as usize] = ebp as i32;
378 context.uc_mcontext.gregs[libc::REG_ECX as usize] = ecx as i32;
379 context.uc_mcontext.gregs[libc::REG_EDX as usize] = edx as i32;
380 } else if #[cfg(all(target_vendor = "apple", target_arch = "x86_64"))] {
381 let TrapHandlerRegs { rip, rsp, rbp, rdi, rsi } = regs;
382 let mcontext = unsafe { &mut *context.uc_mcontext };
383 mcontext.__ss.__rip = rip;
384 mcontext.__ss.__rsp = rsp;
385 mcontext.__ss.__rbp = rbp;
386 mcontext.__ss.__rdi = rdi;
387 mcontext.__ss.__rsi = rsi;
388 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86"))] {
389 let TrapHandlerRegs { eip, esp, ebp, ecx, edx } = regs;
390 context.uc_mcontext.mc_eip = eip as libc::register_t;
391 context.uc_mcontext.mc_esp = esp as libc::register_t;
392 context.uc_mcontext.mc_ebp = ebp as libc::register_t;
393 context.uc_mcontext.mc_ecx = ecx as libc::register_t;
394 context.uc_mcontext.mc_edx = edx as libc::register_t;
395 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] {
396 let TrapHandlerRegs { rip, rsp, rbp, rdi, rsi } = regs;
397 context.uc_mcontext.mc_rip = rip as libc::register_t;
398 context.uc_mcontext.mc_rsp = rsp as libc::register_t;
399 context.uc_mcontext.mc_rbp = rbp as libc::register_t;
400 context.uc_mcontext.mc_rdi = rdi as libc::register_t;
401 context.uc_mcontext.mc_rsi = rsi as libc::register_t;
402 } else if #[cfg(all(
403 any(target_os = "linux", target_os = "android"),
404 target_arch = "aarch64",
405 ))] {
406 let TrapHandlerRegs { pc, sp, x0, x1, x29, lr } = regs;
407 context.uc_mcontext.pc = pc;
408 context.uc_mcontext.sp = sp;
409 context.uc_mcontext.regs[0] = x0;
410 context.uc_mcontext.regs[1] = x1;
411 context.uc_mcontext.regs[29] = x29;
412 context.uc_mcontext.regs[30] = lr;
413 } else if #[cfg(all(
414 any(target_os = "linux", target_os = "android"),
415 target_arch = "arm",
416 ))] {
417 let TrapHandlerRegs {
418 pc,
419 r0,
420 r1,
421 r7,
422 r11,
423 r13,
424 r14,
425 cpsr_thumb,
426 cpsr_endian,
427 } = regs;
428 context.uc_mcontext.arm_pc = pc;
429 context.uc_mcontext.arm_r0 = r0;
430 context.uc_mcontext.arm_r1 = r1;
431 context.uc_mcontext.arm_r7 = r7;
432 context.uc_mcontext.arm_fp = r11;
433 context.uc_mcontext.arm_sp = r13;
434 context.uc_mcontext.arm_lr = r14;
435 if cpsr_thumb {
436 context.uc_mcontext.arm_cpsr |= 0x20;
437 } else {
438 context.uc_mcontext.arm_cpsr &= !0x20;
439 }
440 if cpsr_endian {
441 context.uc_mcontext.arm_cpsr |= 0x200;
442 } else {
443 context.uc_mcontext.arm_cpsr &= !0x200;
444 }
445 } else if #[cfg(all(
446 any(target_os = "linux", target_os = "android"),
447 any(target_arch = "riscv64", target_arch = "riscv32"),
448 ))] {
449 let TrapHandlerRegs { pc, ra, sp, a0, a1, s0 } = regs;
450 context.uc_mcontext.__gregs[libc::REG_PC] = pc as libc::c_ulong;
451 context.uc_mcontext.__gregs[libc::REG_RA] = ra as libc::c_ulong;
452 context.uc_mcontext.__gregs[libc::REG_SP] = sp as libc::c_ulong;
453 context.uc_mcontext.__gregs[libc::REG_A0] = a0 as libc::c_ulong;
454 context.uc_mcontext.__gregs[libc::REG_A0 + 1] = a1 as libc::c_ulong;
455 context.uc_mcontext.__gregs[libc::REG_S0] = s0 as libc::c_ulong;
456 } else if #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] {
457 let TrapHandlerRegs { pc, sp, x0, x1, x29, lr } = regs;
458 let mcontext = unsafe { &mut *context.uc_mcontext };
459 mcontext.__ss.__pc = pc;
460 mcontext.__ss.__sp = sp;
461 mcontext.__ss.__x[0] = x0;
462 mcontext.__ss.__x[1] = x1;
463 mcontext.__ss.__fp = x29;
464 mcontext.__ss.__lr = lr;
465 } else if #[cfg(all(target_os = "freebsd", target_arch = "aarch64"))] {
466 let TrapHandlerRegs { pc, sp, x0, x1, x29, lr } = regs;
467 context.uc_mcontext.mc_gpregs.gp_elr = pc as libc::register_t;
468 context.uc_mcontext.mc_gpregs.gp_sp = sp as libc::register_t;
469 context.uc_mcontext.mc_gpregs.gp_x[0] = x0 as libc::register_t;
470 context.uc_mcontext.mc_gpregs.gp_x[1] = x1 as libc::register_t;
471 context.uc_mcontext.mc_gpregs.gp_x[29] = x29 as libc::register_t;
472 context.uc_mcontext.mc_gpregs.gp_x[30] = lr as libc::register_t;
473 } else if #[cfg(all(target_os = "linux", target_arch = "loongarch64"))] {
474 let TrapHandlerRegs { pc, sp, a0, a1, fp, ra } = regs;
475 context.uc_mcontext.__pc = pc;
476 context.uc_mcontext.__gregs[1] = ra;
477 context.uc_mcontext.__gregs[3] = sp;
478 context.uc_mcontext.__gregs[4] = a0;
479 context.uc_mcontext.__gregs[5] = a1;
480 context.uc_mcontext.__gregs[22] = fp;
481 } else {
482 compile_error!("Unsupported platform");
483 }
484 };
485 }
486 } else if #[cfg(target_os = "windows")] {
487 use windows_sys::Win32::System::Diagnostics::Debug::{
488 AddVectoredExceptionHandler,
489 CONTEXT,
490 EXCEPTION_CONTINUE_EXECUTION,
491 EXCEPTION_CONTINUE_SEARCH,
492 EXCEPTION_POINTERS,
493 };
494 use windows_sys::Win32::Foundation::{
495 EXCEPTION_ACCESS_VIOLATION,
496 EXCEPTION_ILLEGAL_INSTRUCTION,
497 EXCEPTION_INT_DIVIDE_BY_ZERO,
498 EXCEPTION_INT_OVERFLOW,
499 EXCEPTION_STACK_OVERFLOW,
500 };
501
502 unsafe fn platform_init() {
503 unsafe {
504 let handler = AddVectoredExceptionHandler(1, Some(exception_handler));
508 if handler.is_null() {
509 panic!("failed to add exception handler: {}", io::Error::last_os_error());
510 }
511 }
512 }
513
514 unsafe extern "system" fn exception_handler(
515 exception_info: *mut EXCEPTION_POINTERS
516 ) -> i32 {
517 unsafe {
518 let record = &*(*exception_info).ExceptionRecord;
522 if record.ExceptionCode != EXCEPTION_ACCESS_VIOLATION &&
523 record.ExceptionCode != EXCEPTION_ILLEGAL_INSTRUCTION &&
524 record.ExceptionCode != EXCEPTION_STACK_OVERFLOW &&
525 record.ExceptionCode != EXCEPTION_INT_DIVIDE_BY_ZERO &&
526 record.ExceptionCode != EXCEPTION_INT_OVERFLOW
527 {
528 return EXCEPTION_CONTINUE_SEARCH;
529 }
530
531 let context = &mut *(*exception_info).ContextRecord;
544 let (pc, sp) = get_pc_sp(context);
545
546 let maybe_fault_address = match record.ExceptionCode {
548 EXCEPTION_ACCESS_VIOLATION => Some(record.ExceptionInformation[1]),
549 EXCEPTION_STACK_OVERFLOW => Some(sp),
550 _ => None,
551 };
552 let trap_code = match record.ExceptionCode {
553 EXCEPTION_ILLEGAL_INSTRUCTION => {
555 process_illegal_op(pc)
556 }
557 _ => None,
558 };
559 let handled = TrapHandlerContext::handle_trap(
562 pc,
563 sp,
564 maybe_fault_address,
565 trap_code,
566 |regs| update_context(context, regs),
567 |handler| handler(exception_info),
568 );
569
570 if handled {
571 EXCEPTION_CONTINUE_EXECUTION
572 } else {
573 EXCEPTION_CONTINUE_SEARCH
574 }
575 }
576 }
577
578 unsafe fn get_pc_sp(context: &CONTEXT) -> (usize, usize) {
579 let (pc, sp);
580 cfg_if::cfg_if! {
581 if #[cfg(target_arch = "x86_64")] {
582 pc = context.Rip as usize;
583 sp = context.Rsp as usize;
584 } else if #[cfg(target_arch = "x86")] {
585 pc = context.Rip as usize;
586 sp = context.Rsp as usize;
587 } else {
588 compile_error!("Unsupported platform");
589 }
590 };
591 (pc, sp)
592 }
593
594 unsafe fn update_context(context: &mut CONTEXT, regs: TrapHandlerRegs) {
595 cfg_if::cfg_if! {
596 if #[cfg(target_arch = "x86_64")] {
597 let TrapHandlerRegs { rip, rsp, rbp, rdi, rsi } = regs;
598 context.Rip = rip;
599 context.Rsp = rsp;
600 context.Rbp = rbp;
601 context.Rdi = rdi;
602 context.Rsi = rsi;
603 } else if #[cfg(target_arch = "x86")] {
604 let TrapHandlerRegs { eip, esp, ebp, ecx, edx } = regs;
605 context.Eip = eip;
606 context.Esp = esp;
607 context.Ebp = ebp;
608 context.Ecx = ecx;
609 context.Edx = edx;
610 } else {
611 compile_error!("Unsupported platform");
612 }
613 };
614 }
615 }
616}
617
618pub fn init_traps() {
627 static INIT: Once = Once::new();
628 INIT.call_once(|| unsafe {
629 platform_init();
630 });
631}
632
633pub unsafe fn raise_user_trap(data: Box<dyn Error + Send + Sync>) -> ! {
646 unsafe { unwind_with(UnwindReason::UserTrap(data)) }
647}
648
649pub unsafe fn raise_lib_trap(trap: Trap) -> ! {
661 unsafe { unwind_with(UnwindReason::LibTrap(trap)) }
662}
663
664pub unsafe fn resume_panic(payload: Box<dyn Any + Send>) -> ! {
673 unsafe { unwind_with(UnwindReason::Panic(payload)) }
674}
675
676pub unsafe fn wasmer_call_trampoline(
692 trap_handler: Option<*const TrapHandlerFn<'static>>,
693 config: &VMConfig,
694 vmctx: VMFunctionContext,
695 trampoline: VMTrampoline,
696 callee: *const VMFunctionBody,
697 values_vec: *mut u8,
698) -> Result<(), Trap> {
699 unsafe {
700 catch_traps(trap_handler, config, move || {
701 mem::transmute::<
702 unsafe extern "C" fn(
703 *mut VMContext,
704 *const VMFunctionBody,
705 *mut wasmer_types::RawValue,
706 ),
707 extern "C" fn(VMFunctionContext, *const VMFunctionBody, *mut u8),
708 >(trampoline)(vmctx, callee, values_vec);
709 })
710 }
711}
712
713pub unsafe fn catch_traps<F, R: 'static>(
720 trap_handler: Option<*const TrapHandlerFn<'static>>,
721 config: &VMConfig,
722 closure: F,
723) -> Result<R, Trap>
724where
725 F: FnOnce() -> R + 'static,
726{
727 lazy_per_thread_init()?;
729 let stack_size = config
730 .wasm_stack_size
731 .unwrap_or_else(|| DEFAULT_STACK_SIZE.load(Ordering::Relaxed));
732 on_wasm_stack(stack_size, trap_handler, closure).map_err(UnwindReason::into_trap)
733}
734
735thread_local! {
744 static YIELDER: Cell<Option<NonNull<Yielder<(), UnwindReason>>>> = const { Cell::new(None) };
745 static TRAP_HANDLER: AtomicPtr<TrapHandlerContext> = const { AtomicPtr::new(ptr::null_mut()) };
746}
747
748#[allow(clippy::type_complexity)]
751struct TrapHandlerContext {
752 inner: *const u8,
753 handle_trap: fn(
754 *const u8,
755 usize,
756 usize,
757 Option<usize>,
758 Option<TrapCode>,
759 &mut dyn FnMut(TrapHandlerRegs),
760 ) -> bool,
761 custom_trap: Option<*const TrapHandlerFn<'static>>,
762}
763struct TrapHandlerContextInner<T> {
764 coro_trap_handler: CoroutineTrapHandler<Result<T, UnwindReason>>,
767}
768
769impl TrapHandlerContext {
770 fn install<T, R>(
773 custom_trap: Option<*const TrapHandlerFn<'static>>,
774 coro_trap_handler: CoroutineTrapHandler<Result<T, UnwindReason>>,
775 f: impl FnOnce() -> R,
776 ) -> R {
777 fn func<T>(
779 ptr: *const u8,
780 pc: usize,
781 sp: usize,
782 maybe_fault_address: Option<usize>,
783 trap_code: Option<TrapCode>,
784 update_regs: &mut dyn FnMut(TrapHandlerRegs),
785 ) -> bool {
786 unsafe {
787 (*(ptr as *const TrapHandlerContextInner<T>)).handle_trap(
788 pc,
789 sp,
790 maybe_fault_address,
791 trap_code,
792 update_regs,
793 )
794 }
795 }
796 let inner = TrapHandlerContextInner { coro_trap_handler };
797 let ctx = Self {
798 inner: &inner as *const _ as *const u8,
799 handle_trap: func::<T>,
800 custom_trap,
801 };
802
803 compiler_fence(Ordering::Release);
804 let prev = TRAP_HANDLER.with(|ptr| {
805 let prev = ptr.load(Ordering::Relaxed);
806 ptr.store(&ctx as *const Self as *mut Self, Ordering::Relaxed);
807 prev
808 });
809
810 defer! {
811 TRAP_HANDLER.with(|ptr| ptr.store(prev, Ordering::Relaxed));
812 compiler_fence(Ordering::Acquire);
813 }
814
815 f()
816 }
817
818 unsafe fn handle_trap(
820 pc: usize,
821 sp: usize,
822 maybe_fault_address: Option<usize>,
823 trap_code: Option<TrapCode>,
824 mut update_regs: impl FnMut(TrapHandlerRegs),
825 call_handler: impl Fn(&TrapHandlerFn<'static>) -> bool,
826 ) -> bool {
827 unsafe {
828 let ptr = TRAP_HANDLER.with(|ptr| ptr.load(Ordering::Relaxed));
829 if ptr.is_null() {
830 return false;
831 }
832
833 let ctx = &*ptr;
834
835 if let Some(trap_handler) = ctx.custom_trap {
837 if call_handler(&*trap_handler) {
838 return true;
839 }
840 }
841
842 (ctx.handle_trap)(
843 ctx.inner,
844 pc,
845 sp,
846 maybe_fault_address,
847 trap_code,
848 &mut update_regs,
849 )
850 }
851 }
852}
853
854impl<T> TrapHandlerContextInner<T> {
855 unsafe fn handle_trap(
856 &self,
857 pc: usize,
858 sp: usize,
859 maybe_fault_address: Option<usize>,
860 trap_code: Option<TrapCode>,
861 update_regs: &mut dyn FnMut(TrapHandlerRegs),
862 ) -> bool {
863 unsafe {
864 if !self.coro_trap_handler.stack_ptr_in_bounds(sp) {
867 return false;
868 }
869
870 let signal_trap = trap_code.or_else(|| {
871 maybe_fault_address.map(|addr| {
872 if self.coro_trap_handler.stack_ptr_in_bounds(addr) {
873 TrapCode::StackOverflow
874 } else {
875 TrapCode::HeapAccessOutOfBounds
876 }
877 })
878 });
879
880 let backtrace = if signal_trap == Some(TrapCode::StackOverflow) {
887 Backtrace::from(vec![])
888 } else {
889 Backtrace::new_unresolved()
890 };
891
892 let unwind = UnwindReason::WasmTrap {
895 backtrace,
896 signal_trap,
897 pc,
898 };
899 let regs = self
900 .coro_trap_handler
901 .setup_trap_handler(move || Err(unwind));
902 update_regs(regs);
903 true
904 }
905 }
906}
907
908enum UnwindReason {
909 Panic(Box<dyn Any + Send>),
911 UserTrap(Box<dyn Error + Send + Sync>),
913 LibTrap(Trap),
915 WasmTrap {
917 backtrace: Backtrace,
918 pc: usize,
919 signal_trap: Option<TrapCode>,
920 },
921}
922
923impl UnwindReason {
924 fn into_trap(self) -> Trap {
925 match self {
926 Self::UserTrap(data) => Trap::User(data),
927 Self::LibTrap(trap) => trap,
928 Self::WasmTrap {
929 backtrace,
930 pc,
931 signal_trap,
932 } => Trap::wasm(pc, backtrace, signal_trap),
933 Self::Panic(panic) => std::panic::resume_unwind(panic),
934 }
935 }
936}
937
938unsafe fn unwind_with(reason: UnwindReason) -> ! {
939 unsafe {
940 let yielder = YIELDER
941 .with(|cell| cell.replace(None))
942 .expect("not running on Wasm stack");
943
944 yielder.as_ref().suspend(reason);
945
946 unreachable!();
948 }
949}
950
951fn on_wasm_stack<F: FnOnce() -> T + 'static, T: 'static>(
955 stack_size: usize,
956 trap_handler: Option<*const TrapHandlerFn<'static>>,
957 f: F,
958) -> Result<T, UnwindReason> {
959 static STACK_POOL: LazyLock<crossbeam_queue::SegQueue<DefaultStack>> =
964 LazyLock::new(crossbeam_queue::SegQueue::new);
965
966 let stack = STACK_POOL
967 .pop()
968 .unwrap_or_else(|| DefaultStack::new(stack_size).unwrap());
969 let mut stack = scopeguard::guard(stack, |stack| STACK_POOL.push(stack));
970
971 let coro = ScopedCoroutine::with_stack(&mut *stack, move |yielder, ()| {
973 YIELDER.with(|cell| cell.set(Some(yielder.into())));
975
976 Ok(f())
977 });
978
979 defer! {
981 YIELDER.with(|cell| cell.set(None));
982 }
983
984 coro.scope(|mut coro_ref| {
985 TrapHandlerContext::install(trap_handler, coro_ref.trap_handler(), || {
988 match coro_ref.resume(()) {
989 CoroutineResult::Yield(trap) => {
990 unsafe {
993 coro_ref.force_reset();
994 }
995 Err(trap)
996 }
997 CoroutineResult::Return(result) => result,
998 }
999 })
1000 })
1001}
1002
1003pub fn on_host_stack<F: FnOnce() -> T, T>(f: F) -> T {
1012 let yielder_ptr = YIELDER.with(|cell| cell.replace(None));
1015
1016 let yielder = match yielder_ptr {
1019 Some(ptr) => unsafe { ptr.as_ref() },
1020 None => return f(),
1021 };
1022
1023 defer! {
1025 YIELDER.with(|cell| cell.set(yielder_ptr));
1026 }
1027
1028 struct SendWrapper<T>(T);
1032 unsafe impl<T> Send for SendWrapper<T> {}
1033 let wrapped = SendWrapper(f);
1034 yielder.on_parent_stack(move || {
1035 let wrapped = wrapped;
1036 (wrapped.0)()
1037 })
1038}
1039
1040#[cfg(windows)]
1041pub fn lazy_per_thread_init() -> Result<(), Trap> {
1042 use windows_sys::Win32::System::Threading::SetThreadStackGuarantee;
1046 if unsafe { SetThreadStackGuarantee(&mut 0x10000) } == 0 {
1047 panic!("failed to set thread stack guarantee");
1048 }
1049
1050 Ok(())
1051}
1052
1053#[cfg(unix)]
1060pub fn lazy_per_thread_init() -> Result<(), Trap> {
1061 use std::ptr::null_mut;
1062
1063 thread_local! {
1064 static TLS: Tls = unsafe { init_sigstack() };
1067 }
1068
1069 const MIN_STACK_SIZE: usize = 16 * 4096;
1072
1073 enum Tls {
1074 OutOfMemory,
1075 Allocated {
1076 mmap_ptr: *mut libc::c_void,
1077 mmap_size: usize,
1078 },
1079 BigEnough,
1080 }
1081
1082 unsafe fn init_sigstack() -> Tls {
1083 unsafe {
1084 let mut old_stack = mem::zeroed();
1087 let r = libc::sigaltstack(ptr::null(), &mut old_stack);
1088 assert_eq!(r, 0, "learning about sigaltstack failed");
1089 if old_stack.ss_flags & libc::SS_DISABLE == 0 && old_stack.ss_size >= MIN_STACK_SIZE {
1090 return Tls::BigEnough;
1091 }
1092
1093 let page_size: usize = region::page::size();
1096 let guard_size = page_size;
1097 let alloc_size = guard_size + MIN_STACK_SIZE;
1098
1099 let ptr = libc::mmap(
1100 null_mut(),
1101 alloc_size,
1102 libc::PROT_NONE,
1103 libc::MAP_PRIVATE | libc::MAP_ANON,
1104 -1,
1105 0,
1106 );
1107 if ptr == libc::MAP_FAILED {
1108 return Tls::OutOfMemory;
1109 }
1110
1111 let stack_ptr = (ptr as usize + guard_size) as *mut libc::c_void;
1114 let r = libc::mprotect(
1115 stack_ptr,
1116 MIN_STACK_SIZE,
1117 libc::PROT_READ | libc::PROT_WRITE,
1118 );
1119 assert_eq!(r, 0, "mprotect to configure memory for sigaltstack failed");
1120 let new_stack = libc::stack_t {
1121 ss_sp: stack_ptr,
1122 ss_flags: 0,
1123 ss_size: MIN_STACK_SIZE,
1124 };
1125 let r = libc::sigaltstack(&new_stack, ptr::null_mut());
1126 assert_eq!(r, 0, "registering new sigaltstack failed");
1127
1128 Tls::Allocated {
1129 mmap_ptr: ptr,
1130 mmap_size: alloc_size,
1131 }
1132 }
1133 }
1134
1135 return TLS.with(|tls| {
1138 if let Tls::OutOfMemory = tls {
1139 Err(Trap::oom())
1140 } else {
1141 Ok(())
1142 }
1143 });
1144
1145 impl Drop for Tls {
1146 fn drop(&mut self) {
1147 let (ptr, size) = match self {
1148 Self::Allocated {
1149 mmap_ptr,
1150 mmap_size,
1151 } => (*mmap_ptr, *mmap_size),
1152 _ => return,
1153 };
1154 unsafe {
1155 let r = libc::munmap(ptr, size);
1157 debug_assert_eq!(r, 0, "munmap failed during thread shutdown");
1158 }
1159 }
1160 }
1161}