jonaskruckenberg.de
Next Generation WASM Microkernel Operating System
1// Copyright 2025 Jonas Kruckenberg
2//
3// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5// http://opensource.org/licenses/MIT>, at your option. This file may not be
6// copied, modified, or distributed except according to those terms.
7
8use crate::util::zip_eq::IteratorExt;
9use crate::wasm::cranelift::CraneliftGlobal;
10use crate::wasm::cranelift::env::TranslationEnvironment;
11use crate::wasm::cranelift::state::{ControlStackFrame, ElseData, FuncTranslationState};
12use crate::wasm::cranelift::utils::{
13 block_with_params, blocktype_params_results, f32_translation, f64_translation,
14};
15use crate::wasm::indices::{
16 DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex,
17};
18use crate::wasm::trap::{TRAP_NULL_REFERENCE, TRAP_UNREACHABLE};
19use alloc::vec;
20use alloc::vec::Vec;
21use anyhow::bail;
22use cranelift_codegen::ir;
23use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
24use cranelift_codegen::ir::immediates::Offset32;
25use cranelift_codegen::ir::types::{
26 F32, F32X4, F64, F64X2, I8, I8X16, I16, I16X8, I32, I32X4, I64, I64X2,
27};
28use cranelift_codegen::ir::{
29 AtomicRmwOp, ConstantData, JumpTableData, MemFlags, TrapCode, ValueLabel,
30};
31use cranelift_codegen::ir::{InstBuilder, Type, Value};
32use cranelift_entity::packed_option::ReservedValue;
33use cranelift_frontend::{FunctionBuilder, Variable};
34use hashbrown::{HashMap, hash_map};
35use smallvec::SmallVec;
36use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources};
37
38/// Like `Option<T>` but specifically for passing information about transitions
39/// from reachable to unreachable state and the like from callees to callers.
40///
41/// Marked `must_use` to force callers to update
42/// `FuncTranslationState::reachable` as necessary.
43#[derive(PartialEq, Eq)]
44#[must_use]
45pub enum Reachability<T> {
46 /// The Wasm execution state is reachable, here is a `T`.
47 Reachable(T),
48 /// The Wasm execution state has been determined to be statically
49 /// unreachable. It is the receiver of this value's responsibility to update
50 /// `FuncTranslationState::reachable` as necessary.
51 Unreachable,
52}
53
54/// Given a `Reachability<T>`, unwrap the inner `T` or, when unreachable, set
55/// `state.reachable = false` and return.
56///
57/// Used in combination with calling `prepare_addr` and `prepare_atomic_addr`
58/// when we can statically determine that a Wasm access will unconditionally
59/// trap.
60macro_rules! unwrap_or_return_unreachable_state {
61 ($state:ident, $value:expr) => {
62 match $value {
63 Reachability::Reachable(x) => x,
64 Reachability::Unreachable => {
65 $state.reachable = false;
66 return Ok(());
67 }
68 }
69 };
70}
71
72/// Translates wasm operators into Cranelift IR instructions.
73#[expect(clippy::too_many_lines, reason = "This is the big match statement")]
74pub fn translate_operator(
75 validator: &mut FuncValidator<impl WasmModuleResources>,
76 op: &Operator,
77 builder: &mut FunctionBuilder,
78 state: &mut FuncTranslationState,
79 env: &mut TranslationEnvironment,
80) -> crate::Result<()> {
81 if !state.reachable {
82 translate_unreachable_operator(validator, op, builder, state, env);
83 return Ok(());
84 }
85
86 // Given that we believe the current block is reachable, the FunctionBuilder ought to agree.
87 debug_assert!(!builder.is_unreachable());
88
89 match op {
90 Operator::Unreachable => {
91 builder.ins().trap(TRAP_UNREACHABLE);
92 state.reachable = false;
93 }
94 Operator::Nop => {
95 // We do nothing
96 }
97 Operator::Drop => {
98 state.pop1();
99 }
100 Operator::Select => {
101 let (mut arg1, mut arg2, cond) = state.pop3();
102 if builder.func.dfg.value_type(arg1).is_vector() {
103 arg1 = optionally_bitcast_vector(arg1, I8X16, builder);
104 }
105 if builder.func.dfg.value_type(arg2).is_vector() {
106 arg2 = optionally_bitcast_vector(arg2, I8X16, builder);
107 }
108 state.push1(builder.ins().select(cond, arg1, arg2));
109 }
110 /***************************** Control flow blocks **********************************
111 * When starting a control flow block, we create a new `Block` that will hold the code
112 * after the block, and we push a frame on the control stack. Depending on the type
113 * of block, we create a new `Block` for the body of the block with an associated
114 * jump instruction.
115 *
116 * The `End` instruction pops the last control frame from the control stack, seals
117 * the destination block (since `br` instructions targeting it only appear inside the
118 * block and have already been translated) and modify the value stack to use the
119 * possible `Block`'s arguments values.
120 ***********************************************************************************/
121 Operator::Block { blockty } => {
122 let (params, results) = blocktype_params_results(validator, *blockty);
123 let next = block_with_params(builder, results.clone(), env);
124 state.push_block(next, params.len(), results.len());
125 }
126 Operator::Loop { blockty } => {
127 let (params, results) = blocktype_params_results(validator, *blockty);
128 let loop_body = block_with_params(builder, params.clone(), env);
129 let next = block_with_params(builder, results.clone(), env);
130 canonicalise_then_jump(builder, loop_body, state.peekn(params.len()));
131 state.push_loop(loop_body, next, params.len(), results.len());
132
133 // Pop the initial `Block` actuals and replace them with the `Block`'s
134 // params since control flow joins at the top of the loop.
135 state.popn(params.len());
136 state
137 .stack
138 .extend_from_slice(builder.block_params(loop_body));
139
140 builder.switch_to_block(loop_body);
141 // env.before_loop_header(builder)?;
142 }
143 Operator::If { blockty } => {
144 let val = state.pop1();
145
146 let next_block = builder.create_block();
147 let (params, results) = blocktype_params_results(validator, *blockty);
148 let (destination, else_data) = if params.clone().eq(results.clone()) {
149 // It is possible there is no `else` block, so we will only
150 // allocate a block for it if/when we find the `else`. For now,
151 // we if the condition isn't true, then we jump directly to the
152 // destination block following the whole `if...end`. If we do end
153 // up discovering an `else`, then we will allocate a block for it
154 // and go back and patch the jump.
155 let destination = block_with_params(builder, results.clone(), env);
156 let branch_inst = canonicalise_brif(
157 builder,
158 val,
159 next_block,
160 &[],
161 destination,
162 state.peekn(params.len()),
163 );
164 (
165 destination,
166 ElseData::NoElse {
167 branch_inst,
168 placeholder: destination,
169 },
170 )
171 } else {
172 // The `if` type signature is not valid without an `else` block,
173 // so we eagerly allocate the `else` block here.
174 let destination = block_with_params(builder, results.clone(), env);
175 let else_block = block_with_params(builder, params.clone(), env);
176 canonicalise_brif(
177 builder,
178 val,
179 next_block,
180 &[],
181 else_block,
182 state.peekn(params.len()),
183 );
184 builder.seal_block(else_block);
185 (destination, ElseData::WithElse { else_block })
186 };
187
188 builder.seal_block(next_block); // Only predecessor is the current block.
189 builder.switch_to_block(next_block);
190
191 // Here we append an argument to a Block targeted by an argumentless jump instruction
192 // But in fact there are two cases:
193 // - either the If does not have a Else clause, in that case ty = EmptyBlock
194 // and we add nothing;
195 // - either the If have an Else clause, in that case the destination of this jump
196 // instruction will be changed later when we parse the Else operator.
197 state.push_if(
198 destination,
199 else_data,
200 params.len(),
201 results.len(),
202 *blockty,
203 );
204 }
205 Operator::Else => {
206 let i = state.control_stack.len().checked_sub(1).unwrap();
207 match *state.control_stack.get_mut(i).unwrap() {
208 ControlStackFrame::If {
209 ref else_data,
210 head_is_reachable,
211 ref mut consequent_ends_reachable,
212 num_return_values,
213 blocktype,
214 destination,
215 ..
216 } => {
217 // We finished the consequent, so record its final
218 // reachability state.
219 debug_assert!(consequent_ends_reachable.is_none());
220 *consequent_ends_reachable = Some(state.reachable);
221
222 if head_is_reachable {
223 // We have a branch from the head of the `if` to the `else`.
224 state.reachable = true;
225
226 // Ensure we have a block for the `else` block (it may have
227 // already been pre-allocated, see `ElseData` for details).
228 let else_block = match *else_data {
229 ElseData::NoElse {
230 branch_inst,
231 placeholder,
232 } => {
233 let (params, _results) =
234 blocktype_params_results(validator, blocktype);
235 debug_assert_eq!(params.len(), num_return_values);
236 let else_block = block_with_params(builder, params.clone(), env);
237 canonicalise_then_jump(
238 builder,
239 destination,
240 state.peekn(params.len()),
241 );
242 state.popn(params.len());
243
244 builder.change_jump_destination(
245 branch_inst,
246 placeholder,
247 else_block,
248 );
249 builder.seal_block(else_block);
250 else_block
251 }
252 ElseData::WithElse { else_block } => {
253 canonicalise_then_jump(
254 builder,
255 destination,
256 state.peekn(num_return_values),
257 );
258 state.popn(num_return_values);
259 else_block
260 }
261 };
262
263 // You might be expecting that we push the parameters for this
264 // `else` block here, something like this:
265 //
266 // state.pushn(&control_stack_frame.params);
267 //
268 // We don't do that because they are already on the top of the stack
269 // for us: we pushed the parameters twice when we saw the initial
270 // `if` so that we wouldn't have to save the parameters in the
271 // `ControlStackFrame` as another `Vec` allocation.
272
273 builder.switch_to_block(else_block);
274
275 // We don't bother updating the control frame's `ElseData`
276 // to `WithElse` because nothing else will read it.
277 }
278 }
279 _ => unreachable!(),
280 }
281 }
282 Operator::End => {
283 let frame = state.control_stack.pop().unwrap();
284 let next_block = frame.following_code();
285 let return_count = frame.num_return_values();
286 let return_args = state.peekn_mut(return_count);
287
288 canonicalise_then_jump(builder, next_block, return_args);
289 // You might expect that if we just finished an `if` block that
290 // didn't have a corresponding `else` block, then we would clean
291 // up our duplicate set of parameters that we pushed earlier
292 // right here. However, we don't have to explicitly do that,
293 // since we truncate the stack back to the original height
294 // below.
295
296 builder.switch_to_block(next_block);
297 builder.seal_block(next_block);
298
299 // If it is a loop we also have to seal the body loop block
300 if let ControlStackFrame::Loop { header, .. } = frame {
301 builder.seal_block(header);
302 }
303
304 frame.truncate_value_stack_to_original_size(&mut state.stack);
305 state
306 .stack
307 .extend_from_slice(builder.block_params(next_block));
308 }
309 /**************************** Branch instructions *********************************
310 * The branch instructions all have as arguments a target nesting level, which
311 * corresponds to how many control stack frames do we have to pop to get the
312 * destination `Block`.
313 *
314 * Once the destination `Block` is found, we sometimes have to declare a certain depth
315 * of the stack unreachable, because some branch instructions are terminator.
316 *
317 * The `br_table` case is much more complicated because Cranelift's `br_table` instruction
318 * does not support jump arguments like all the other branch instructions. That is why, in
319 * the case where we would use jump arguments for every other branch instruction, we
320 * need to split the critical edges leaving the `br_tables` by creating one `Block` per
321 * table destination; the `br_table` will point to these newly created `Blocks` and these
322 * `Block`s contain only a jump instruction pointing to the final destination, this time with
323 * jump arguments.
324 *
325 * This system is also implemented in Cranelift's SSA construction algorithm, because
326 * `use_var` located in a destination `Block` of a `br_table` might trigger the addition
327 * of jump arguments in each predecessor branch instruction, one of which might be a
328 * `br_table`.
329 ***********************************************************************************/
330 Operator::Br { relative_depth } => {
331 // FIXME wow this is ugly
332 let i = state
333 .control_stack
334 .len()
335 .checked_sub(1)
336 .unwrap()
337 .checked_sub(usize::try_from(*relative_depth).unwrap())
338 .unwrap();
339
340 let (return_count, br_destination) = {
341 let frame = &mut state.control_stack[i];
342 // We signal that all the code that follows until the next End is unreachable
343 frame.set_branched_to_exit();
344 let return_count = if frame.is_loop() {
345 frame.num_param_values()
346 } else {
347 frame.num_return_values()
348 };
349 (return_count, frame.br_destination())
350 };
351 let destination_args = state.peekn_mut(return_count);
352 canonicalise_then_jump(builder, br_destination, destination_args);
353 state.popn(return_count);
354 state.reachable = false;
355 }
356 Operator::BrIf { relative_depth } => translate_br_if(*relative_depth, builder, state),
357 Operator::BrTable { targets } => {
358 let default = targets.default();
359 let mut min_depth = default;
360 for depth in targets.targets() {
361 let depth = depth?;
362 if depth < min_depth {
363 min_depth = depth;
364 }
365 }
366 let jump_args_count = {
367 // FIXME wow this is ugly
368 let i = state
369 .control_stack
370 .len()
371 .checked_sub(1)
372 .unwrap()
373 .checked_sub(usize::try_from(min_depth).unwrap())
374 .unwrap();
375
376 let min_depth_frame = &state.control_stack[i];
377 if min_depth_frame.is_loop() {
378 min_depth_frame.num_param_values()
379 } else {
380 min_depth_frame.num_return_values()
381 }
382 };
383 let val = state.pop1();
384 let mut data = Vec::with_capacity(targets.len() as usize);
385 if jump_args_count == 0 {
386 // No jump arguments
387 for depth in targets.targets() {
388 let depth = depth?;
389 let block = {
390 // FIXME wow this is ugly
391 let i = state
392 .control_stack
393 .len()
394 .checked_sub(1)
395 .unwrap()
396 .checked_sub(usize::try_from(depth).unwrap())
397 .unwrap();
398
399 let frame = &mut state.control_stack[i];
400 frame.set_branched_to_exit();
401 frame.br_destination()
402 };
403 data.push(builder.func.dfg.block_call(block, &[]));
404 }
405 let block = {
406 // FIXME wow this is ugly
407 let i = state
408 .control_stack
409 .len()
410 .checked_sub(1)
411 .unwrap()
412 .checked_sub(usize::try_from(default).unwrap())
413 .unwrap();
414
415 let frame = &mut state.control_stack[i];
416 frame.set_branched_to_exit();
417 frame.br_destination()
418 };
419 let block = builder.func.dfg.block_call(block, &[]);
420 let jt = builder.create_jump_table(JumpTableData::new(block, &data));
421 builder.ins().br_table(val, jt);
422 } else {
423 // Here we have jump arguments, but Cranelift's br_table doesn't support them
424 // We then proceed to split the edges going out of the br_table
425 let return_count = jump_args_count;
426 let mut dest_block_sequence = vec![];
427 let mut dest_block_map = HashMap::new();
428 for depth in targets.targets() {
429 let depth = depth?;
430 let branch_block = match dest_block_map.entry(depth as usize) {
431 hash_map::Entry::Occupied(entry) => *entry.get(),
432 hash_map::Entry::Vacant(entry) => {
433 let block = builder.create_block();
434 dest_block_sequence.push((depth as usize, block));
435 *entry.insert(block)
436 }
437 };
438 data.push(builder.func.dfg.block_call(branch_block, &[]));
439 }
440 let default_branch_block = match dest_block_map.entry(default as usize) {
441 hash_map::Entry::Occupied(entry) => *entry.get(),
442 hash_map::Entry::Vacant(entry) => {
443 let block = builder.create_block();
444 dest_block_sequence.push((default as usize, block));
445 *entry.insert(block)
446 }
447 };
448 let default_branch_block = builder.func.dfg.block_call(default_branch_block, &[]);
449 let jt = builder.create_jump_table(JumpTableData::new(default_branch_block, &data));
450 builder.ins().br_table(val, jt);
451 for (depth, dest_block) in dest_block_sequence {
452 builder.switch_to_block(dest_block);
453 builder.seal_block(dest_block);
454 let real_dest_block = {
455 // FIXME wow this is ugly
456 let i = state
457 .control_stack
458 .len()
459 .checked_sub(1)
460 .unwrap()
461 .checked_sub(depth)
462 .unwrap();
463
464 let frame = &mut state.control_stack[i];
465 frame.set_branched_to_exit();
466 frame.br_destination()
467 };
468 let destination_args = state.peekn_mut(return_count);
469 canonicalise_then_jump(builder, real_dest_block, destination_args);
470 }
471 state.popn(return_count);
472 }
473 state.reachable = false;
474 }
475 Operator::Return => {
476 let return_count = {
477 let frame = &mut state.control_stack[0];
478 frame.num_return_values()
479 };
480 {
481 let return_args = state.peekn_mut(return_count);
482 // env.handle_before_return(&return_args, builder)?;
483 bitcast_wasm_returns(return_args, builder, env);
484 builder.ins().return_(return_args);
485 }
486 state.popn(return_count);
487 state.reachable = false;
488 }
489 /************************************ Calls ****************************************
490 * The call instructions pop off their arguments from the stack and append their
491 * return values to it. `call_indirect` needs environment support because there is an
492 * argument referring to an index in the external functions table of the module.
493 ************************************************************************************/
494 Operator::Call { function_index } => {
495 let function_index = FuncIndex::from_u32(*function_index);
496 let (fref, num_args) = state.get_direct_func(builder.func, function_index, env);
497
498 // Bitcast any vector arguments to their default type, I8X16, before calling.
499 let args = state.peekn_mut(num_args);
500 bitcast_wasm_params(
501 builder.func.dfg.ext_funcs[fref].signature,
502 args,
503 builder,
504 env,
505 );
506
507 let call = env.translate_call(builder, function_index, fref, args);
508 let inst_results = builder.inst_results(call);
509 debug_assert_eq!(
510 inst_results.len(),
511 builder.func.dfg.signatures[builder.func.dfg.ext_funcs[fref].signature]
512 .returns
513 .len(),
514 "translate_call results should match the call signature"
515 );
516 state.popn(num_args);
517 state.pushn(inst_results);
518 }
519 Operator::CallIndirect {
520 type_index,
521 table_index,
522 } => {
523 let type_index = TypeIndex::from_u32(*type_index);
524 // `type_index` is the index of the function's signature and
525 // `table_index` is the index of the table to search the function
526 // in.
527 let (sigref, num_args) = state.get_indirect_sig(builder.func, type_index, env);
528 let callee = state.pop1();
529
530 // Bitcast any vector arguments to their default type, I8X16, before calling.
531 let args = state.peekn_mut(num_args);
532 bitcast_wasm_params(sigref, args, builder, env);
533
534 let table_index = TableIndex::from_u32(*table_index);
535 let table = state.get_table(builder.func, table_index, env).clone();
536
537 let call = unwrap_or_return_unreachable_state!(state, {
538 env.translate_call_indirect(
539 builder,
540 table_index,
541 &table,
542 type_index,
543 sigref,
544 callee,
545 state.peekn(num_args),
546 )
547 });
548
549 let inst_results = builder.inst_results(call);
550 debug_assert_eq!(
551 inst_results.len(),
552 builder.func.dfg.signatures[sigref].returns.len(),
553 "translate_call_indirect results should match the call signature"
554 );
555 state.popn(num_args);
556 state.pushn(inst_results);
557 }
558 /******************************* Tail Calls ******************************************
559 * The tail call instructions pop their arguments from the stack and
560 * then permanently transfer control to their callee. The indirect
561 * version requires environment support (while the direct version can
562 * optionally be hooked but doesn't require it) it interacts with the
563 * VM's runtime state via tables.
564 ************************************************************************************/
565 Operator::ReturnCall { function_index } => {
566 let function_index = FuncIndex::from_u32(*function_index);
567 let (fref, num_args) = state.get_direct_func(builder.func, function_index, env);
568
569 // Bitcast any vector arguments to their default type, I8X16, before calling.
570 let args = state.peekn_mut(num_args);
571 bitcast_wasm_params(
572 builder.func.dfg.ext_funcs[fref].signature,
573 args,
574 builder,
575 env,
576 );
577
578 env.translate_return_call(builder, function_index, fref, args)?;
579
580 state.popn(num_args);
581 state.reachable = false;
582 }
583 Operator::ReturnCallIndirect {
584 type_index,
585 table_index,
586 } => {
587 let type_index = TypeIndex::from_u32(*type_index);
588 // `type_index` is the index of the function's signature and
589 // `table_index` is the index of the table to search the function
590 // in.
591 let (sigref, num_args) = state.get_indirect_sig(builder.func, type_index, env);
592 let callee = state.pop1();
593
594 // Bitcast any vector arguments to their default type, I8X16, before calling.
595 let args = state.peekn_mut(num_args);
596 bitcast_wasm_params(sigref, args, builder, env);
597
598 env.translate_return_call_indirect(
599 builder,
600 TableIndex::from_u32(*table_index),
601 type_index,
602 sigref,
603 callee,
604 state.peekn(num_args),
605 )?;
606
607 state.popn(num_args);
608 state.reachable = false;
609 }
610 /********************************** Locals ****************************************
611 * `get_local` and `set_local` are treated as non-SSA variables and will completely
612 * disappear in the Cranelift Code
613 ***********************************************************************************/
614 Operator::LocalGet { local_index } => {
615 let val = builder.use_var(Variable::from_u32(*local_index));
616 state.push1(val);
617 let label = ValueLabel::from_u32(*local_index);
618 builder.set_val_label(val, label);
619 }
620 Operator::LocalSet { local_index } => {
621 let mut val = state.pop1();
622
623 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
624 let ty = builder.func.dfg.value_type(val);
625 if ty.is_vector() {
626 val = optionally_bitcast_vector(val, I8X16, builder);
627 }
628
629 builder.def_var(Variable::from_u32(*local_index), val);
630 let label = ValueLabel::from_u32(*local_index);
631 builder.set_val_label(val, label);
632 }
633 Operator::LocalTee { local_index } => {
634 let mut val = state.peek1();
635
636 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
637 let ty = builder.func.dfg.value_type(val);
638 if ty.is_vector() {
639 val = optionally_bitcast_vector(val, I8X16, builder);
640 }
641
642 builder.def_var(Variable::from_u32(*local_index), val);
643 let label = ValueLabel::from_u32(*local_index);
644 builder.set_val_label(val, label);
645 }
646 /********************************** Globals ****************************************
647 * `get_global` and `set_global` are handled by the environment.
648 ***********************************************************************************/
649 Operator::GlobalGet { global_index } => {
650 let global_index = GlobalIndex::from_u32(*global_index);
651 let val = match *state.get_global(builder.func, global_index, env) {
652 CraneliftGlobal::Const(val) => val,
653 CraneliftGlobal::Memory { gv, offset, ty } => {
654 let addr = builder.ins().global_value(env.pointer_type(), gv);
655 let mut flags = MemFlags::trusted();
656 // Put globals in the "table" abstract mem category as well.
657 flags.set_alias_region(Some(ir::AliasRegion::Table));
658 builder.ins().load(ty, flags, addr, offset)
659 }
660 CraneliftGlobal::Custom => {
661 env.translate_custom_global_get(builder, global_index)?
662 }
663 };
664 state.push1(val);
665 }
666 Operator::GlobalSet { global_index } => {
667 let global_index = GlobalIndex::from_u32(*global_index);
668 match *state.get_global(builder.func, global_index, env) {
669 CraneliftGlobal::Const(_) => panic!("global #{global_index:?} is a constant"),
670 CraneliftGlobal::Memory { gv, offset, ty } => {
671 let addr = builder.ins().global_value(env.pointer_type(), gv);
672 let mut flags = MemFlags::trusted();
673 // Put globals in the "table" abstract mem category as well.
674 flags.set_alias_region(Some(ir::AliasRegion::Table));
675 let mut val = state.pop1();
676 // Ensure SIMD values are cast to their default Cranelift type, I8x16.
677 if ty.is_vector() {
678 val = optionally_bitcast_vector(val, I8X16, builder);
679 }
680 debug_assert_eq!(ty, builder.func.dfg.value_type(val));
681 builder.ins().store(flags, val, addr, offset);
682 }
683 CraneliftGlobal::Custom => {
684 let val = state.pop1();
685 env.translate_custom_global_set(builder, global_index, val)?;
686 }
687 }
688 }
689 /******************************* Memory management ***********************************
690 * Memory management is handled by environment. It is usually translated into calls to
691 * special functions.
692 ************************************************************************************/
693 Operator::MemoryGrow { mem } => {
694 // The WebAssembly MVP only supports one linear memory, but we expect the reserved
695 // argument to be a memory index.
696 let mem_index = MemoryIndex::from_u32(*mem);
697 let delta = state.pop1();
698 let ret = env.translate_memory_grow(builder.cursor(), mem_index, delta);
699 state.push1(ret);
700 }
701 Operator::MemorySize { mem } => {
702 let mem_index = MemoryIndex::from_u32(*mem);
703 let ret = env.translate_memory_size(builder.cursor(), mem_index)?;
704 state.push1(ret);
705 }
706
707 Operator::I32Const { value } => {
708 state.push1(builder.ins().iconst(I32, i64::from(*value)));
709 }
710 Operator::I64Const { value } => state.push1(builder.ins().iconst(I64, *value)),
711 Operator::F32Const { value } => {
712 state.push1(builder.ins().f32const(f32_translation(*value)));
713 }
714 Operator::F64Const { value } => {
715 state.push1(builder.ins().f64const(f64_translation(*value)));
716 }
717
718 // integer operators
719 Operator::I32Add | Operator::I64Add => {
720 let (arg1, arg2) = state.pop2();
721 state.push1(builder.ins().iadd(arg1, arg2));
722 }
723 Operator::I32Sub | Operator::I64Sub => {
724 let (arg1, arg2) = state.pop2();
725 state.push1(builder.ins().isub(arg1, arg2));
726 }
727 Operator::I32Mul | Operator::I64Mul => {
728 let (arg1, arg2) = state.pop2();
729 state.push1(builder.ins().imul(arg1, arg2));
730 }
731 Operator::I32DivS | Operator::I64DivS => {
732 let (arg1, arg2) = state.pop2();
733 state.push1(builder.ins().sdiv(arg1, arg2));
734 }
735 Operator::I32DivU | Operator::I64DivU => {
736 let (arg1, arg2) = state.pop2();
737 state.push1(builder.ins().udiv(arg1, arg2));
738 }
739 Operator::I32RemS | Operator::I64RemS => {
740 let (arg1, arg2) = state.pop2();
741 state.push1(builder.ins().srem(arg1, arg2));
742 }
743 Operator::I32RemU | Operator::I64RemU => {
744 let (arg1, arg2) = state.pop2();
745 state.push1(builder.ins().urem(arg1, arg2));
746 }
747 Operator::I32And | Operator::I64And => {
748 let (arg1, arg2) = state.pop2();
749 state.push1(builder.ins().band(arg1, arg2));
750 }
751 Operator::I32Or | Operator::I64Or => {
752 let (arg1, arg2) = state.pop2();
753 state.push1(builder.ins().bor(arg1, arg2));
754 }
755 Operator::I32Xor | Operator::I64Xor => {
756 let (arg1, arg2) = state.pop2();
757 state.push1(builder.ins().bxor(arg1, arg2));
758 }
759 Operator::I32Shl | Operator::I64Shl => {
760 let (arg1, arg2) = state.pop2();
761 state.push1(builder.ins().ishl(arg1, arg2));
762 }
763 Operator::I32ShrS | Operator::I64ShrS => {
764 let (arg1, arg2) = state.pop2();
765 state.push1(builder.ins().sshr(arg1, arg2));
766 }
767 Operator::I32ShrU | Operator::I64ShrU => {
768 let (arg1, arg2) = state.pop2();
769 state.push1(builder.ins().ushr(arg1, arg2));
770 }
771 Operator::I32Rotl | Operator::I64Rotl => {
772 let (arg1, arg2) = state.pop2();
773 state.push1(builder.ins().rotl(arg1, arg2));
774 }
775 Operator::I32Rotr | Operator::I64Rotr => {
776 let (arg1, arg2) = state.pop2();
777 state.push1(builder.ins().rotr(arg1, arg2));
778 }
779 Operator::I32Clz | Operator::I64Clz => {
780 let arg = state.pop1();
781 state.push1(builder.ins().clz(arg));
782 }
783 Operator::I32Ctz | Operator::I64Ctz => {
784 let arg = state.pop1();
785 state.push1(builder.ins().ctz(arg));
786 }
787 Operator::I32Popcnt | Operator::I64Popcnt => {
788 let arg = state.pop1();
789 state.push1(builder.ins().popcnt(arg));
790 }
791 Operator::I32WrapI64 => {
792 let val = state.pop1();
793 state.push1(builder.ins().ireduce(I32, val));
794 }
795 Operator::I64ExtendI32S => {
796 let val = state.pop1();
797 state.push1(builder.ins().sextend(I64, val));
798 }
799 Operator::I64ExtendI32U => {
800 let val = state.pop1();
801 state.push1(builder.ins().uextend(I64, val));
802 }
803
804 // floating-point operators
805 Operator::F32Add | Operator::F64Add => {
806 let (arg1, arg2) = state.pop2();
807 state.push1(builder.ins().fadd(arg1, arg2));
808 }
809 Operator::F32Sub | Operator::F64Sub => {
810 let (arg1, arg2) = state.pop2();
811 state.push1(builder.ins().fsub(arg1, arg2));
812 }
813 Operator::F32Mul | Operator::F64Mul => {
814 let (arg1, arg2) = state.pop2();
815 state.push1(builder.ins().fmul(arg1, arg2));
816 }
817 Operator::F32Div | Operator::F64Div => {
818 let (arg1, arg2) = state.pop2();
819 state.push1(builder.ins().fdiv(arg1, arg2));
820 }
821 Operator::F32Min | Operator::F64Min => {
822 let (arg1, arg2) = state.pop2();
823 state.push1(builder.ins().fmin(arg1, arg2));
824 }
825 Operator::F32Max | Operator::F64Max => {
826 let (arg1, arg2) = state.pop2();
827 state.push1(builder.ins().fmax(arg1, arg2));
828 }
829 Operator::F32Copysign | Operator::F64Copysign => {
830 let (arg1, arg2) = state.pop2();
831 state.push1(builder.ins().fcopysign(arg1, arg2));
832 }
833 Operator::F32Sqrt | Operator::F64Sqrt => {
834 let arg = state.pop1();
835 state.push1(builder.ins().sqrt(arg));
836 }
837 Operator::F32Ceil | Operator::F64Ceil => {
838 let arg = state.pop1();
839 state.push1(builder.ins().ceil(arg));
840 }
841 Operator::F32Floor | Operator::F64Floor => {
842 let arg = state.pop1();
843 state.push1(builder.ins().floor(arg));
844 }
845 Operator::F32Trunc | Operator::F64Trunc => {
846 let arg = state.pop1();
847 state.push1(builder.ins().trunc(arg));
848 }
849 Operator::F32Nearest | Operator::F64Nearest => {
850 let arg = state.pop1();
851 state.push1(builder.ins().nearest(arg));
852 }
853 Operator::F32Abs | Operator::F64Abs => {
854 let val = state.pop1();
855 state.push1(builder.ins().fabs(val));
856 }
857 Operator::F32Neg | Operator::F64Neg => {
858 let arg = state.pop1();
859 state.push1(builder.ins().fneg(arg));
860 }
861 Operator::I32TruncF64S | Operator::I32TruncF32S => {
862 let val = state.pop1();
863 state.push1(builder.ins().fcvt_to_sint(I32, val));
864 }
865 Operator::I32TruncF64U | Operator::I32TruncF32U => {
866 let val = state.pop1();
867 state.push1(builder.ins().fcvt_to_uint(I32, val));
868 }
869 Operator::I64TruncF64U | Operator::I64TruncF32U => {
870 let val = state.pop1();
871 state.push1(builder.ins().fcvt_to_uint(I64, val));
872 }
873 Operator::I64TruncF64S | Operator::I64TruncF32S => {
874 let val = state.pop1();
875 state.push1(builder.ins().fcvt_to_sint(I64, val));
876 }
877 Operator::F32DemoteF64 => {
878 let val = state.pop1();
879 state.push1(builder.ins().fdemote(F32, val));
880 }
881 Operator::F32ConvertI64S | Operator::F32ConvertI32S => {
882 let val = state.pop1();
883 state.push1(builder.ins().fcvt_from_sint(F32, val));
884 }
885 Operator::F32ConvertI64U | Operator::F32ConvertI32U => {
886 let val = state.pop1();
887 state.push1(builder.ins().fcvt_from_uint(F32, val));
888 }
889 Operator::F64ConvertI64S | Operator::F64ConvertI32S => {
890 let val = state.pop1();
891 state.push1(builder.ins().fcvt_from_sint(F64, val));
892 }
893 Operator::F64ConvertI64U | Operator::F64ConvertI32U => {
894 let val = state.pop1();
895 state.push1(builder.ins().fcvt_from_uint(F64, val));
896 }
897 Operator::F64PromoteF32 => {
898 let val = state.pop1();
899 state.push1(builder.ins().fpromote(F64, val));
900 }
901 Operator::F32ReinterpretI32 => {
902 let val = state.pop1();
903 state.push1(builder.ins().bitcast(F32, MemFlags::new(), val));
904 }
905 Operator::F64ReinterpretI64 => {
906 let val = state.pop1();
907 state.push1(builder.ins().bitcast(F64, MemFlags::new(), val));
908 }
909 Operator::I32ReinterpretF32 => {
910 let val = state.pop1();
911 state.push1(builder.ins().bitcast(I32, MemFlags::new(), val));
912 }
913 Operator::I64ReinterpretF64 => {
914 let val = state.pop1();
915 state.push1(builder.ins().bitcast(I64, MemFlags::new(), val));
916 }
917
918 // comparison operators
919 Operator::I32LtS | Operator::I64LtS => {
920 translate_icmp(IntCC::SignedLessThan, builder, state);
921 }
922 Operator::I32LtU | Operator::I64LtU => {
923 translate_icmp(IntCC::UnsignedLessThan, builder, state);
924 }
925 Operator::I32LeS | Operator::I64LeS => {
926 translate_icmp(IntCC::SignedLessThanOrEqual, builder, state);
927 }
928 Operator::I32LeU | Operator::I64LeU => {
929 translate_icmp(IntCC::UnsignedLessThanOrEqual, builder, state);
930 }
931 Operator::I32GtS | Operator::I64GtS => {
932 translate_icmp(IntCC::SignedGreaterThan, builder, state);
933 }
934 Operator::I32GtU | Operator::I64GtU => {
935 translate_icmp(IntCC::UnsignedGreaterThan, builder, state);
936 }
937 Operator::I32GeS | Operator::I64GeS => {
938 translate_icmp(IntCC::SignedGreaterThanOrEqual, builder, state);
939 }
940 Operator::I32GeU | Operator::I64GeU => {
941 translate_icmp(IntCC::UnsignedGreaterThanOrEqual, builder, state);
942 }
943 Operator::I32Eqz | Operator::I64Eqz => {
944 let arg = state.pop1();
945 let val = builder.ins().icmp_imm(IntCC::Equal, arg, 0);
946 state.push1(builder.ins().uextend(I32, val));
947 }
948 Operator::I32Eq | Operator::I64Eq => translate_icmp(IntCC::Equal, builder, state),
949 Operator::F32Eq | Operator::F64Eq => translate_fcmp(FloatCC::Equal, builder, state),
950 Operator::I32Ne | Operator::I64Ne => translate_icmp(IntCC::NotEqual, builder, state),
951 Operator::F32Ne | Operator::F64Ne => translate_fcmp(FloatCC::NotEqual, builder, state),
952 Operator::F32Gt | Operator::F64Gt => translate_fcmp(FloatCC::GreaterThan, builder, state),
953 Operator::F32Ge | Operator::F64Ge => {
954 translate_fcmp(FloatCC::GreaterThanOrEqual, builder, state);
955 }
956 Operator::F32Lt | Operator::F64Lt => translate_fcmp(FloatCC::LessThan, builder, state),
957 Operator::F32Le | Operator::F64Le => {
958 translate_fcmp(FloatCC::LessThanOrEqual, builder, state);
959 }
960
961 /******************************* Load instructions ***********************************
962 * Wasm specifies an integer alignment flag, but we drop it in Cranelift.
963 * The memory base address is provided by the environment.
964 ************************************************************************************/
965 Operator::I32Load8U { memarg } => {
966 unwrap_or_return_unreachable_state!(
967 state,
968 translate_load(memarg, ir::Opcode::Uload8, I32, builder, state, env)
969 );
970 }
971 Operator::I32Load16U { memarg } => {
972 unwrap_or_return_unreachable_state!(
973 state,
974 translate_load(memarg, ir::Opcode::Uload16, I32, builder, state, env)
975 );
976 }
977 Operator::I32Load8S { memarg } => {
978 unwrap_or_return_unreachable_state!(
979 state,
980 translate_load(memarg, ir::Opcode::Sload8, I32, builder, state, env)
981 );
982 }
983 Operator::I32Load16S { memarg } => {
984 unwrap_or_return_unreachable_state!(
985 state,
986 translate_load(memarg, ir::Opcode::Sload16, I32, builder, state, env)
987 );
988 }
989 Operator::I64Load8U { memarg } => {
990 unwrap_or_return_unreachable_state!(
991 state,
992 translate_load(memarg, ir::Opcode::Uload8, I64, builder, state, env)
993 );
994 }
995 Operator::I64Load16U { memarg } => {
996 unwrap_or_return_unreachable_state!(
997 state,
998 translate_load(memarg, ir::Opcode::Uload16, I64, builder, state, env)
999 );
1000 }
1001 Operator::I64Load8S { memarg } => {
1002 unwrap_or_return_unreachable_state!(
1003 state,
1004 translate_load(memarg, ir::Opcode::Sload8, I64, builder, state, env)
1005 );
1006 }
1007 Operator::I64Load16S { memarg } => {
1008 unwrap_or_return_unreachable_state!(
1009 state,
1010 translate_load(memarg, ir::Opcode::Sload16, I64, builder, state, env)
1011 );
1012 }
1013 Operator::I64Load32S { memarg } => {
1014 unwrap_or_return_unreachable_state!(
1015 state,
1016 translate_load(memarg, ir::Opcode::Sload32, I64, builder, state, env)
1017 );
1018 }
1019 Operator::I64Load32U { memarg } => {
1020 unwrap_or_return_unreachable_state!(
1021 state,
1022 translate_load(memarg, ir::Opcode::Uload32, I64, builder, state, env)
1023 );
1024 }
1025 Operator::I32Load { memarg } => {
1026 unwrap_or_return_unreachable_state!(
1027 state,
1028 translate_load(memarg, ir::Opcode::Load, I32, builder, state, env)
1029 );
1030 }
1031 Operator::F32Load { memarg } => {
1032 unwrap_or_return_unreachable_state!(
1033 state,
1034 translate_load(memarg, ir::Opcode::Load, F32, builder, state, env)
1035 );
1036 }
1037 Operator::I64Load { memarg } => {
1038 unwrap_or_return_unreachable_state!(
1039 state,
1040 translate_load(memarg, ir::Opcode::Load, I64, builder, state, env)
1041 );
1042 }
1043 Operator::F64Load { memarg } => {
1044 unwrap_or_return_unreachable_state!(
1045 state,
1046 translate_load(memarg, ir::Opcode::Load, F64, builder, state, env)
1047 );
1048 }
1049
1050 /****************************** Store instructions ***********************************
1051 * Wasm specifies an integer alignment flag but we drop it in Cranelift.
1052 * The memory base address is provided by the environment.
1053 ************************************************************************************/
1054 Operator::I32Store { memarg }
1055 | Operator::I64Store { memarg }
1056 | Operator::F32Store { memarg }
1057 | Operator::F64Store { memarg } => {
1058 translate_store(memarg, ir::Opcode::Store, builder, state, env)?;
1059 }
1060 Operator::I32Store8 { memarg } | Operator::I64Store8 { memarg } => {
1061 translate_store(memarg, ir::Opcode::Istore8, builder, state, env)?;
1062 }
1063 Operator::I32Store16 { memarg } | Operator::I64Store16 { memarg } => {
1064 translate_store(memarg, ir::Opcode::Istore16, builder, state, env)?;
1065 }
1066 Operator::I64Store32 { memarg } => {
1067 translate_store(memarg, ir::Opcode::Istore32, builder, state, env)?;
1068 }
1069
1070 // Sign-extension
1071 // https://github.com/WebAssembly/sign-extension-ops
1072 Operator::I32Extend8S => {
1073 let val = state.pop1();
1074 state.push1(builder.ins().ireduce(I8, val));
1075 let val = state.pop1();
1076 state.push1(builder.ins().sextend(I32, val));
1077 }
1078 Operator::I32Extend16S => {
1079 let val = state.pop1();
1080 state.push1(builder.ins().ireduce(I16, val));
1081 let val = state.pop1();
1082 state.push1(builder.ins().sextend(I32, val));
1083 }
1084 Operator::I64Extend8S => {
1085 let val = state.pop1();
1086 state.push1(builder.ins().ireduce(I8, val));
1087 let val = state.pop1();
1088 state.push1(builder.ins().sextend(I64, val));
1089 }
1090 Operator::I64Extend16S => {
1091 let val = state.pop1();
1092 state.push1(builder.ins().ireduce(I16, val));
1093 let val = state.pop1();
1094 state.push1(builder.ins().sextend(I64, val));
1095 }
1096 Operator::I64Extend32S => {
1097 let val = state.pop1();
1098 state.push1(builder.ins().ireduce(I32, val));
1099 let val = state.pop1();
1100 state.push1(builder.ins().sextend(I64, val));
1101 }
1102
1103 // Non-trapping Float-to-int Conversions
1104 // https://github.com/WebAssembly/nontrapping-float-to-int-conversions
1105 Operator::I32TruncSatF64S | Operator::I32TruncSatF32S => {
1106 let val = state.pop1();
1107 state.push1(builder.ins().fcvt_to_sint_sat(I32, val));
1108 }
1109 Operator::I32TruncSatF64U | Operator::I32TruncSatF32U => {
1110 let val = state.pop1();
1111 state.push1(builder.ins().fcvt_to_uint_sat(I32, val));
1112 }
1113 Operator::I64TruncSatF64S | Operator::I64TruncSatF32S => {
1114 let val = state.pop1();
1115 state.push1(builder.ins().fcvt_to_sint_sat(I64, val));
1116 }
1117 Operator::I64TruncSatF64U | Operator::I64TruncSatF32U => {
1118 let val = state.pop1();
1119 state.push1(builder.ins().fcvt_to_uint_sat(I64, val));
1120 }
1121
1122 // reference-types
1123 // https://github.com/WebAssembly/reference-types
1124 Operator::TableFill { table } => {
1125 let table_index = TableIndex::from_u32(*table);
1126 let len = state.pop1();
1127 let val = state.pop1();
1128 let dest = state.pop1();
1129 env.translate_table_fill(builder.cursor(), table_index, dest, val, len)?;
1130 }
1131 Operator::TableGet { table: index } => {
1132 let table_index = TableIndex::from_u32(*index);
1133 let index = state.pop1();
1134 state.push1(env.translate_table_get(builder.cursor(), table_index, index)?);
1135 }
1136 Operator::TableSet { table: index } => {
1137 let table_index = TableIndex::from_u32(*index);
1138 let value = state.pop1();
1139 let index = state.pop1();
1140 env.translate_table_set(builder.cursor(), table_index, value, index)?;
1141 }
1142 Operator::TableGrow { table: index } => {
1143 let table_index = TableIndex::from_u32(*index);
1144 let delta = state.pop1();
1145 let init_value = state.pop1();
1146 state.push1(env.translate_table_grow(
1147 builder.cursor(),
1148 table_index,
1149 delta,
1150 init_value,
1151 )?);
1152 }
1153 Operator::TableSize { table: index } => {
1154 state.push1(env.translate_table_size(builder.cursor(), TableIndex::from_u32(*index))?);
1155 }
1156 Operator::RefNull { hty } => {
1157 let hty = env.convert_heap_type(*hty);
1158 state.push1(env.translate_ref_null(builder.cursor(), &hty));
1159 }
1160 Operator::RefIsNull => {
1161 let value = state.pop1();
1162 state.push1(env.translate_ref_is_null(builder.cursor(), value));
1163 }
1164 Operator::RefFunc { function_index } => {
1165 let index = FuncIndex::from_u32(*function_index);
1166 state.push1(env.translate_ref_func(builder.cursor(), index)?);
1167 }
1168 Operator::TypedSelect { ty: _ } => {
1169 // We ignore the explicit type parameter as it is only needed for
1170 // validation, which we require to have been performed before
1171 // translation.
1172 let (mut arg1, mut arg2, cond) = state.pop3();
1173 if builder.func.dfg.value_type(arg1).is_vector() {
1174 arg1 = optionally_bitcast_vector(arg1, I8X16, builder);
1175 }
1176 if builder.func.dfg.value_type(arg2).is_vector() {
1177 arg2 = optionally_bitcast_vector(arg2, I8X16, builder);
1178 }
1179 state.push1(builder.ins().select(cond, arg1, arg2));
1180 }
1181 //
1182 // // bulk memory operations
1183 // // https://github.com/WebAssembly/bulk-memory-operations
1184 Operator::MemoryInit { data_index, mem } => {
1185 let mem_index = MemoryIndex::from_u32(*mem);
1186 let len = state.pop1();
1187 let src = state.pop1();
1188 let dest = state.pop1();
1189 env.translate_memory_init(
1190 builder.cursor(),
1191 mem_index,
1192 DataIndex::from_u32(*data_index),
1193 dest,
1194 src,
1195 len,
1196 );
1197 }
1198 Operator::MemoryCopy { src_mem, dst_mem } => {
1199 let src_index = MemoryIndex::from_u32(*src_mem);
1200 let dst_index = MemoryIndex::from_u32(*dst_mem);
1201 let len = state.pop1();
1202 let src_pos = state.pop1();
1203 let dst_pos = state.pop1();
1204 env.translate_memory_copy(
1205 builder.cursor(),
1206 src_index,
1207 dst_index,
1208 src_pos,
1209 dst_pos,
1210 len,
1211 );
1212 }
1213 Operator::MemoryFill { mem } => {
1214 let mem_index = MemoryIndex::from_u32(*mem);
1215 let len = state.pop1();
1216 let val = state.pop1();
1217 let dest = state.pop1();
1218 env.translate_memory_fill(builder.cursor(), mem_index, dest, val, len);
1219 }
1220 Operator::DataDrop { data_index } => {
1221 env.translate_data_drop(builder.cursor(), DataIndex::from_u32(*data_index));
1222 }
1223 Operator::TableInit {
1224 elem_index,
1225 table: table_index,
1226 } => {
1227 let len = state.pop1();
1228 let src = state.pop1();
1229 let dest = state.pop1();
1230 env.translate_table_init(
1231 builder.cursor(),
1232 TableIndex::from_u32(*table_index),
1233 ElemIndex::from_u32(*elem_index),
1234 dest,
1235 src,
1236 len,
1237 )?;
1238 }
1239 Operator::TableCopy {
1240 dst_table: dst_table_index,
1241 src_table: src_table_index,
1242 } => {
1243 let len = state.pop1();
1244 let src = state.pop1();
1245 let dest = state.pop1();
1246 env.translate_table_copy(
1247 builder.cursor(),
1248 TableIndex::from_u32(*dst_table_index),
1249 TableIndex::from_u32(*src_table_index),
1250 dest,
1251 src,
1252 len,
1253 )?;
1254 }
1255 Operator::ElemDrop { elem_index } => {
1256 env.translate_elem_drop(builder.cursor(), ElemIndex::from_u32(*elem_index))?;
1257 }
1258
1259 // threads
1260 // https://github.com/WebAssembly/threads
1261 Operator::MemoryAtomicWait32 { memarg } | Operator::MemoryAtomicWait64 { memarg } => {
1262 // The WebAssembly MVP only supports one linear memory and
1263 // wasmparser will ensure that the memory indices specified are
1264 // zero.
1265 let implied_ty = match op {
1266 Operator::MemoryAtomicWait64 { .. } => I64,
1267 Operator::MemoryAtomicWait32 { .. } => I32,
1268 _ => unreachable!(),
1269 };
1270 let mem_index = MemoryIndex::from_u32(memarg.memory);
1271 let timeout = state.pop1(); // 64 (fixed)
1272 let expected = state.pop1(); // 32 or 64 (per the `Ixx` in `IxxAtomicWait`)
1273 assert_eq!(builder.func.dfg.value_type(expected), implied_ty);
1274 let addr = state.pop1();
1275 let effective_addr = if memarg.offset == 0 {
1276 addr
1277 } else {
1278 // TODO let index_type = environ.mems()[mem].index_type;
1279 let index_type = I32;
1280 let offset = builder
1281 .ins()
1282 .iconst(index_type, i64::try_from(memarg.offset).unwrap());
1283 builder
1284 .ins()
1285 .uadd_overflow_trap(addr, offset, TrapCode::HEAP_OUT_OF_BOUNDS)
1286 };
1287 // `fn translate_atomic_wait` can inspect the type of `expected` to figure out what
1288 // code it needs to generate, if it wants.
1289 let res = env.translate_atomic_wait(
1290 builder.cursor(),
1291 mem_index,
1292 effective_addr,
1293 expected,
1294 timeout,
1295 )?;
1296 state.push1(res);
1297 }
1298 Operator::MemoryAtomicNotify { memarg } => {
1299 let mem_index = MemoryIndex::from_u32(memarg.memory);
1300 let count = state.pop1(); // 32 (fixed)
1301 let addr = state.pop1();
1302 let effective_addr = if memarg.offset == 0 {
1303 addr
1304 } else {
1305 // TODO let index_type = environ.mems()[mem].index_type;
1306 let index_type = I32;
1307 let offset = builder
1308 .ins()
1309 .iconst(index_type, i64::try_from(memarg.offset).unwrap());
1310 builder
1311 .ins()
1312 .uadd_overflow_trap(addr, offset, TrapCode::HEAP_OUT_OF_BOUNDS)
1313 };
1314 let res =
1315 env.translate_atomic_notify(builder.cursor(), mem_index, effective_addr, count)?;
1316 state.push1(res);
1317 }
1318 Operator::I32AtomicLoad { memarg } => {
1319 translate_atomic_load(I32, I32, memarg, builder, state, env)?;
1320 }
1321 Operator::I64AtomicLoad { memarg } => {
1322 translate_atomic_load(I64, I64, memarg, builder, state, env)?;
1323 }
1324 Operator::I32AtomicLoad8U { memarg } => {
1325 translate_atomic_load(I32, I8, memarg, builder, state, env)?;
1326 }
1327 Operator::I32AtomicLoad16U { memarg } => {
1328 translate_atomic_load(I32, I16, memarg, builder, state, env)?;
1329 }
1330 Operator::I64AtomicLoad8U { memarg } => {
1331 translate_atomic_load(I64, I8, memarg, builder, state, env)?;
1332 }
1333 Operator::I64AtomicLoad16U { memarg } => {
1334 translate_atomic_load(I64, I16, memarg, builder, state, env)?;
1335 }
1336 Operator::I64AtomicLoad32U { memarg } => {
1337 translate_atomic_load(I64, I32, memarg, builder, state, env)?;
1338 }
1339
1340 Operator::I32AtomicStore { memarg } => {
1341 translate_atomic_store(I32, memarg, builder, state, env)?;
1342 }
1343 Operator::I64AtomicStore { memarg } => {
1344 translate_atomic_store(I64, memarg, builder, state, env)?;
1345 }
1346 Operator::I32AtomicStore8 { memarg } => {
1347 translate_atomic_store(I8, memarg, builder, state, env)?;
1348 }
1349 Operator::I32AtomicStore16 { memarg } => {
1350 translate_atomic_store(I16, memarg, builder, state, env)?;
1351 }
1352 Operator::I64AtomicStore8 { memarg } => {
1353 translate_atomic_store(I8, memarg, builder, state, env)?;
1354 }
1355 Operator::I64AtomicStore16 { memarg } => {
1356 translate_atomic_store(I16, memarg, builder, state, env)?;
1357 }
1358 Operator::I64AtomicStore32 { memarg } => {
1359 translate_atomic_store(I32, memarg, builder, state, env)?;
1360 }
1361
1362 Operator::I32AtomicRmwAdd { memarg } => {
1363 translate_atomic_rmw(I32, I32, AtomicRmwOp::Add, memarg, builder, state, env)?;
1364 }
1365 Operator::I64AtomicRmwAdd { memarg } => {
1366 translate_atomic_rmw(I64, I64, AtomicRmwOp::Add, memarg, builder, state, env)?;
1367 }
1368 Operator::I32AtomicRmw8AddU { memarg } => {
1369 translate_atomic_rmw(I32, I8, AtomicRmwOp::Add, memarg, builder, state, env)?;
1370 }
1371 Operator::I32AtomicRmw16AddU { memarg } => {
1372 translate_atomic_rmw(I32, I16, AtomicRmwOp::Add, memarg, builder, state, env)?;
1373 }
1374 Operator::I64AtomicRmw8AddU { memarg } => {
1375 translate_atomic_rmw(I64, I8, AtomicRmwOp::Add, memarg, builder, state, env)?;
1376 }
1377 Operator::I64AtomicRmw16AddU { memarg } => {
1378 translate_atomic_rmw(I64, I16, AtomicRmwOp::Add, memarg, builder, state, env)?;
1379 }
1380 Operator::I64AtomicRmw32AddU { memarg } => {
1381 translate_atomic_rmw(I64, I32, AtomicRmwOp::Add, memarg, builder, state, env)?;
1382 }
1383
1384 Operator::I32AtomicRmwSub { memarg } => {
1385 translate_atomic_rmw(I32, I32, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1386 }
1387 Operator::I64AtomicRmwSub { memarg } => {
1388 translate_atomic_rmw(I64, I64, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1389 }
1390 Operator::I32AtomicRmw8SubU { memarg } => {
1391 translate_atomic_rmw(I32, I8, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1392 }
1393 Operator::I32AtomicRmw16SubU { memarg } => {
1394 translate_atomic_rmw(I32, I16, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1395 }
1396 Operator::I64AtomicRmw8SubU { memarg } => {
1397 translate_atomic_rmw(I64, I8, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1398 }
1399 Operator::I64AtomicRmw16SubU { memarg } => {
1400 translate_atomic_rmw(I64, I16, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1401 }
1402 Operator::I64AtomicRmw32SubU { memarg } => {
1403 translate_atomic_rmw(I64, I32, AtomicRmwOp::Sub, memarg, builder, state, env)?;
1404 }
1405
1406 Operator::I32AtomicRmwAnd { memarg } => {
1407 translate_atomic_rmw(I32, I32, AtomicRmwOp::And, memarg, builder, state, env)?;
1408 }
1409 Operator::I64AtomicRmwAnd { memarg } => {
1410 translate_atomic_rmw(I64, I64, AtomicRmwOp::And, memarg, builder, state, env)?;
1411 }
1412 Operator::I32AtomicRmw8AndU { memarg } => {
1413 translate_atomic_rmw(I32, I8, AtomicRmwOp::And, memarg, builder, state, env)?;
1414 }
1415 Operator::I32AtomicRmw16AndU { memarg } => {
1416 translate_atomic_rmw(I32, I16, AtomicRmwOp::And, memarg, builder, state, env)?;
1417 }
1418 Operator::I64AtomicRmw8AndU { memarg } => {
1419 translate_atomic_rmw(I64, I8, AtomicRmwOp::And, memarg, builder, state, env)?;
1420 }
1421 Operator::I64AtomicRmw16AndU { memarg } => {
1422 translate_atomic_rmw(I64, I16, AtomicRmwOp::And, memarg, builder, state, env)?;
1423 }
1424 Operator::I64AtomicRmw32AndU { memarg } => {
1425 translate_atomic_rmw(I64, I32, AtomicRmwOp::And, memarg, builder, state, env)?;
1426 }
1427
1428 Operator::I32AtomicRmwOr { memarg } => {
1429 translate_atomic_rmw(I32, I32, AtomicRmwOp::Or, memarg, builder, state, env)?;
1430 }
1431 Operator::I64AtomicRmwOr { memarg } => {
1432 translate_atomic_rmw(I64, I64, AtomicRmwOp::Or, memarg, builder, state, env)?;
1433 }
1434 Operator::I32AtomicRmw8OrU { memarg } => {
1435 translate_atomic_rmw(I32, I8, AtomicRmwOp::Or, memarg, builder, state, env)?;
1436 }
1437 Operator::I32AtomicRmw16OrU { memarg } => {
1438 translate_atomic_rmw(I32, I16, AtomicRmwOp::Or, memarg, builder, state, env)?;
1439 }
1440 Operator::I64AtomicRmw8OrU { memarg } => {
1441 translate_atomic_rmw(I64, I8, AtomicRmwOp::Or, memarg, builder, state, env)?;
1442 }
1443 Operator::I64AtomicRmw16OrU { memarg } => {
1444 translate_atomic_rmw(I64, I16, AtomicRmwOp::Or, memarg, builder, state, env)?;
1445 }
1446 Operator::I64AtomicRmw32OrU { memarg } => {
1447 translate_atomic_rmw(I64, I32, AtomicRmwOp::Or, memarg, builder, state, env)?;
1448 }
1449
1450 Operator::I32AtomicRmwXor { memarg } => {
1451 translate_atomic_rmw(I32, I32, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1452 }
1453 Operator::I64AtomicRmwXor { memarg } => {
1454 translate_atomic_rmw(I64, I64, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1455 }
1456 Operator::I32AtomicRmw8XorU { memarg } => {
1457 translate_atomic_rmw(I32, I8, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1458 }
1459 Operator::I32AtomicRmw16XorU { memarg } => {
1460 translate_atomic_rmw(I32, I16, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1461 }
1462 Operator::I64AtomicRmw8XorU { memarg } => {
1463 translate_atomic_rmw(I64, I8, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1464 }
1465 Operator::I64AtomicRmw16XorU { memarg } => {
1466 translate_atomic_rmw(I64, I16, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1467 }
1468 Operator::I64AtomicRmw32XorU { memarg } => {
1469 translate_atomic_rmw(I64, I32, AtomicRmwOp::Xor, memarg, builder, state, env)?;
1470 }
1471
1472 Operator::I32AtomicRmwXchg { memarg } => {
1473 translate_atomic_rmw(I32, I32, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1474 }
1475 Operator::I64AtomicRmwXchg { memarg } => {
1476 translate_atomic_rmw(I64, I64, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1477 }
1478 Operator::I32AtomicRmw8XchgU { memarg } => {
1479 translate_atomic_rmw(I32, I8, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1480 }
1481 Operator::I32AtomicRmw16XchgU { memarg } => {
1482 translate_atomic_rmw(I32, I16, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1483 }
1484 Operator::I64AtomicRmw8XchgU { memarg } => {
1485 translate_atomic_rmw(I64, I8, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1486 }
1487 Operator::I64AtomicRmw16XchgU { memarg } => {
1488 translate_atomic_rmw(I64, I16, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1489 }
1490 Operator::I64AtomicRmw32XchgU { memarg } => {
1491 translate_atomic_rmw(I64, I32, AtomicRmwOp::Xchg, memarg, builder, state, env)?;
1492 }
1493
1494 Operator::I32AtomicRmwCmpxchg { memarg } => {
1495 translate_atomic_cas(I32, I32, memarg, builder, state, env)?;
1496 }
1497 Operator::I64AtomicRmwCmpxchg { memarg } => {
1498 translate_atomic_cas(I64, I64, memarg, builder, state, env)?;
1499 }
1500 Operator::I32AtomicRmw8CmpxchgU { memarg } => {
1501 translate_atomic_cas(I32, I8, memarg, builder, state, env)?;
1502 }
1503 Operator::I32AtomicRmw16CmpxchgU { memarg } => {
1504 translate_atomic_cas(I32, I16, memarg, builder, state, env)?;
1505 }
1506 Operator::I64AtomicRmw8CmpxchgU { memarg } => {
1507 translate_atomic_cas(I64, I8, memarg, builder, state, env)?;
1508 }
1509 Operator::I64AtomicRmw16CmpxchgU { memarg } => {
1510 translate_atomic_cas(I64, I16, memarg, builder, state, env)?;
1511 }
1512 Operator::I64AtomicRmw32CmpxchgU { memarg } => {
1513 translate_atomic_cas(I64, I32, memarg, builder, state, env)?;
1514 }
1515 Operator::AtomicFence { .. } => {
1516 builder.ins().fence();
1517 }
1518
1519 // 128-bit SIMD
1520 // - https://github.com/webassembly/simd
1521 // - https://webassembly.github.io/simd/core/binary/instructions.html
1522 Operator::V128Const { value } => {
1523 let data = value.bytes().to_vec().into();
1524 let handle = builder.func.dfg.constants.insert(data);
1525 let value = builder.ins().vconst(I8X16, handle);
1526 // the v128.const is typed in CLIF as a I8x16 but bitcast to a different type
1527 // before use
1528 state.push1(value);
1529 }
1530 Operator::V128Load { memarg } => {
1531 unwrap_or_return_unreachable_state!(
1532 state,
1533 translate_load(memarg, ir::Opcode::Load, I8X16, builder, state, env)
1534 );
1535 }
1536 Operator::V128Load8x8S { memarg } => {
1537 let index = state.pop1();
1538 let mem = state.get_memory(builder.func, MemoryIndex::from_u32(memarg.memory), env);
1539 let (flags, _, base) = unwrap_or_return_unreachable_state!(
1540 state,
1541 mem.prepare_addr(builder, index, 8, memarg, env)
1542 );
1543 let loaded = builder.ins().sload8x8(flags, base, 0i32);
1544 state.push1(loaded);
1545 }
1546 Operator::V128Load8x8U { memarg } => {
1547 let index = state.pop1();
1548 let mem = state.get_memory(builder.func, MemoryIndex::from_u32(memarg.memory), env);
1549 let (flags, _, base) = unwrap_or_return_unreachable_state!(
1550 state,
1551 mem.prepare_addr(builder, index, 8, memarg, env)
1552 );
1553 let loaded = builder.ins().uload8x8(flags, base, 0i32);
1554 state.push1(loaded);
1555 }
1556 Operator::V128Load16x4S { memarg } => {
1557 let index = state.pop1();
1558 let mem = state.get_memory(builder.func, MemoryIndex::from_u32(memarg.memory), env);
1559 let (flags, _, base) = unwrap_or_return_unreachable_state!(
1560 state,
1561 mem.prepare_addr(builder, index, 8, memarg, env)
1562 );
1563 let loaded = builder.ins().sload16x4(flags, base, 0i32);
1564 state.push1(loaded);
1565 }
1566 Operator::V128Load16x4U { memarg } => {
1567 let index = state.pop1();
1568 let mem = state.get_memory(builder.func, MemoryIndex::from_u32(memarg.memory), env);
1569 let (flags, _, base) = unwrap_or_return_unreachable_state!(
1570 state,
1571 mem.prepare_addr(builder, index, 8, memarg, env)
1572 );
1573 let loaded = builder.ins().uload16x4(flags, base, 0i32);
1574 state.push1(loaded);
1575 }
1576 Operator::V128Load32x2S { memarg } => {
1577 let index = state.pop1();
1578 let mem = state.get_memory(builder.func, MemoryIndex::from_u32(memarg.memory), env);
1579 let (flags, _, base) = unwrap_or_return_unreachable_state!(
1580 state,
1581 mem.prepare_addr(builder, index, 8, memarg, env)
1582 );
1583 let loaded = builder.ins().sload32x2(flags, base, 0i32);
1584 state.push1(loaded);
1585 }
1586 Operator::V128Load32x2U { memarg } => {
1587 let index = state.pop1();
1588 let mem = state.get_memory(builder.func, MemoryIndex::from_u32(memarg.memory), env);
1589 let (flags, _, base) = unwrap_or_return_unreachable_state!(
1590 state,
1591 mem.prepare_addr(builder, index, 8, memarg, env)
1592 );
1593 let loaded = builder.ins().uload32x2(flags, base, 0i32);
1594 state.push1(loaded);
1595 }
1596 Operator::V128Store { .. } => todo!(),
1597 Operator::I8x16Splat | Operator::I16x8Splat => {
1598 let reduced = builder.ins().ireduce(type_of(op).lane_type(), state.pop1());
1599 let splatted = builder.ins().splat(type_of(op), reduced);
1600 state.push1(splatted);
1601 }
1602 Operator::I32x4Splat
1603 | Operator::I64x2Splat
1604 | Operator::F32x4Splat
1605 | Operator::F64x2Splat => {
1606 let splatted = builder.ins().splat(type_of(op), state.pop1());
1607 state.push1(splatted);
1608 }
1609 Operator::V128Load8Splat { memarg }
1610 | Operator::V128Load16Splat { memarg }
1611 | Operator::V128Load32Splat { memarg }
1612 | Operator::V128Load64Splat { memarg } => {
1613 unwrap_or_return_unreachable_state!(
1614 state,
1615 translate_load(
1616 memarg,
1617 ir::Opcode::Load,
1618 type_of(op).lane_type(),
1619 builder,
1620 state,
1621 env,
1622 )
1623 );
1624 let splatted = builder.ins().splat(type_of(op), state.pop1());
1625 state.push1(splatted);
1626 }
1627 Operator::V128Load32Zero { memarg } | Operator::V128Load64Zero { memarg } => {
1628 unwrap_or_return_unreachable_state!(
1629 state,
1630 translate_load(
1631 memarg,
1632 ir::Opcode::Load,
1633 type_of(op).lane_type(),
1634 builder,
1635 state,
1636 env,
1637 )
1638 );
1639 let as_vector = builder.ins().scalar_to_vector(type_of(op), state.pop1());
1640 state.push1(as_vector);
1641 }
1642 Operator::V128Load8Lane { memarg, lane }
1643 | Operator::V128Load16Lane { memarg, lane }
1644 | Operator::V128Load32Lane { memarg, lane }
1645 | Operator::V128Load64Lane { memarg, lane } => {
1646 let vector = pop1_with_bitcast(state, type_of(op), builder);
1647 unwrap_or_return_unreachable_state!(
1648 state,
1649 translate_load(
1650 memarg,
1651 ir::Opcode::Load,
1652 type_of(op).lane_type(),
1653 builder,
1654 state,
1655 env,
1656 )
1657 );
1658 let replacement = state.pop1();
1659 state.push1(builder.ins().insertlane(vector, replacement, *lane));
1660 }
1661 Operator::V128Store8Lane { memarg, lane }
1662 | Operator::V128Store16Lane { memarg, lane }
1663 | Operator::V128Store32Lane { memarg, lane }
1664 | Operator::V128Store64Lane { memarg, lane } => {
1665 let vector = pop1_with_bitcast(state, type_of(op), builder);
1666 state.push1(builder.ins().extractlane(vector, *lane));
1667 translate_store(memarg, ir::Opcode::Store, builder, state, env)?;
1668 }
1669 Operator::I8x16ExtractLaneS { lane } | Operator::I16x8ExtractLaneS { lane } => {
1670 let vector = pop1_with_bitcast(state, type_of(op), builder);
1671 let extracted = builder.ins().extractlane(vector, *lane);
1672 state.push1(builder.ins().sextend(I32, extracted));
1673 }
1674 Operator::I8x16ExtractLaneU { lane } | Operator::I16x8ExtractLaneU { lane } => {
1675 let vector = pop1_with_bitcast(state, type_of(op), builder);
1676 let extracted = builder.ins().extractlane(vector, *lane);
1677 state.push1(builder.ins().uextend(I32, extracted));
1678 // On x86, PEXTRB zeroes the upper bits of the destination register of extractlane so
1679 // uextend could be elided; for now, uextend is needed for Cranelift's type checks to
1680 // work.
1681 }
1682 Operator::I32x4ExtractLane { lane }
1683 | Operator::I64x2ExtractLane { lane }
1684 | Operator::F32x4ExtractLane { lane }
1685 | Operator::F64x2ExtractLane { lane } => {
1686 let vector = pop1_with_bitcast(state, type_of(op), builder);
1687 state.push1(builder.ins().extractlane(vector, *lane));
1688 }
1689 Operator::I8x16ReplaceLane { lane } | Operator::I16x8ReplaceLane { lane } => {
1690 let (vector, replacement) = state.pop2();
1691 let ty = type_of(op);
1692 let reduced = builder.ins().ireduce(ty.lane_type(), replacement);
1693 let vector = optionally_bitcast_vector(vector, ty, builder);
1694 state.push1(builder.ins().insertlane(vector, reduced, *lane));
1695 }
1696 Operator::I32x4ReplaceLane { lane }
1697 | Operator::I64x2ReplaceLane { lane }
1698 | Operator::F32x4ReplaceLane { lane }
1699 | Operator::F64x2ReplaceLane { lane } => {
1700 let (vector, replacement) = state.pop2();
1701 let vector = optionally_bitcast_vector(vector, type_of(op), builder);
1702 state.push1(builder.ins().insertlane(vector, replacement, *lane));
1703 }
1704 Operator::I8x16Shuffle { lanes, .. } => {
1705 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
1706 let lanes = ConstantData::from(lanes.as_ref());
1707 let mask = builder.func.dfg.immediates.push(lanes);
1708 let shuffled = builder.ins().shuffle(a, b, mask);
1709 state.push1(shuffled);
1710 // At this point the original types of a and b are lost; users of this value (i.e. this
1711 // WASM-to-CLIF translator) may need to bitcast for type-correctness. This is due
1712 // to WASM using the less specific v128 type for certain operations and more specific
1713 // types (e.g. i8x16) for others.
1714 }
1715 Operator::I8x16Swizzle => {
1716 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
1717 state.push1(builder.ins().swizzle(a, b));
1718 }
1719 Operator::I8x16Add | Operator::I16x8Add | Operator::I32x4Add | Operator::I64x2Add => {
1720 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1721 state.push1(builder.ins().iadd(a, b));
1722 }
1723 Operator::I8x16AddSatS | Operator::I16x8AddSatS => {
1724 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1725 state.push1(builder.ins().sadd_sat(a, b));
1726 }
1727 Operator::I8x16AddSatU | Operator::I16x8AddSatU => {
1728 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1729 state.push1(builder.ins().uadd_sat(a, b));
1730 }
1731 Operator::I8x16Sub | Operator::I16x8Sub | Operator::I32x4Sub | Operator::I64x2Sub => {
1732 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1733 state.push1(builder.ins().isub(a, b));
1734 }
1735 Operator::I8x16SubSatS | Operator::I16x8SubSatS => {
1736 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1737 state.push1(builder.ins().ssub_sat(a, b));
1738 }
1739 Operator::I8x16SubSatU | Operator::I16x8SubSatU => {
1740 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1741 state.push1(builder.ins().usub_sat(a, b));
1742 }
1743 Operator::I8x16MinS | Operator::I16x8MinS | Operator::I32x4MinS => {
1744 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1745 state.push1(builder.ins().smin(a, b));
1746 }
1747 Operator::I8x16MinU | Operator::I16x8MinU | Operator::I32x4MinU => {
1748 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1749 state.push1(builder.ins().umin(a, b));
1750 }
1751 Operator::I8x16MaxS | Operator::I16x8MaxS | Operator::I32x4MaxS => {
1752 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1753 state.push1(builder.ins().smax(a, b));
1754 }
1755 Operator::I8x16MaxU | Operator::I16x8MaxU | Operator::I32x4MaxU => {
1756 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1757 state.push1(builder.ins().umax(a, b));
1758 }
1759 Operator::I8x16AvgrU | Operator::I16x8AvgrU => {
1760 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1761 state.push1(builder.ins().avg_round(a, b));
1762 }
1763 Operator::I8x16Neg | Operator::I16x8Neg | Operator::I32x4Neg | Operator::I64x2Neg => {
1764 let a = pop1_with_bitcast(state, type_of(op), builder);
1765 state.push1(builder.ins().ineg(a));
1766 }
1767 Operator::I8x16Abs | Operator::I16x8Abs | Operator::I32x4Abs | Operator::I64x2Abs => {
1768 let a = pop1_with_bitcast(state, type_of(op), builder);
1769 state.push1(builder.ins().iabs(a));
1770 }
1771 Operator::I16x8Mul | Operator::I32x4Mul | Operator::I64x2Mul => {
1772 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1773 state.push1(builder.ins().imul(a, b));
1774 }
1775 Operator::V128Or => {
1776 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1777 state.push1(builder.ins().bor(a, b));
1778 }
1779 Operator::V128Xor => {
1780 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1781 state.push1(builder.ins().bxor(a, b));
1782 }
1783 Operator::V128And => {
1784 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1785 state.push1(builder.ins().band(a, b));
1786 }
1787 Operator::V128AndNot => {
1788 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1789 state.push1(builder.ins().band_not(a, b));
1790 }
1791 Operator::V128Not => {
1792 let a = state.pop1();
1793 state.push1(builder.ins().bnot(a));
1794 }
1795 Operator::I8x16Shl | Operator::I16x8Shl | Operator::I32x4Shl | Operator::I64x2Shl => {
1796 let (a, b) = state.pop2();
1797 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1798 // The spec expects to shift with `b mod lanewidth`; This is directly compatible
1799 // with cranelift's instruction.
1800 state.push1(builder.ins().ishl(bitcast_a, b));
1801 }
1802 Operator::I8x16ShrU | Operator::I16x8ShrU | Operator::I32x4ShrU | Operator::I64x2ShrU => {
1803 let (a, b) = state.pop2();
1804 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1805 // The spec expects to shift with `b mod lanewidth`; This is directly compatible
1806 // with cranelift's instruction.
1807 state.push1(builder.ins().ushr(bitcast_a, b));
1808 }
1809 Operator::I8x16ShrS | Operator::I16x8ShrS | Operator::I32x4ShrS | Operator::I64x2ShrS => {
1810 let (a, b) = state.pop2();
1811 let bitcast_a = optionally_bitcast_vector(a, type_of(op), builder);
1812 // The spec expects to shift with `b mod lanewidth`; This is directly compatible
1813 // with cranelift's instruction.
1814 state.push1(builder.ins().sshr(bitcast_a, b));
1815 }
1816 Operator::V128Bitselect => {
1817 let (a, b, c) = pop3_with_bitcast(state, I8X16, builder);
1818 // The CLIF operand ordering is slightly different and the types of all three
1819 // operands must match (hence the bitcast).
1820 state.push1(builder.ins().bitselect(c, a, b));
1821 }
1822 Operator::V128AnyTrue => {
1823 let a = pop1_with_bitcast(state, type_of(op), builder);
1824 let bool_result = builder.ins().vany_true(a);
1825 state.push1(builder.ins().uextend(I32, bool_result));
1826 }
1827 Operator::I8x16AllTrue
1828 | Operator::I16x8AllTrue
1829 | Operator::I32x4AllTrue
1830 | Operator::I64x2AllTrue => {
1831 let a = pop1_with_bitcast(state, type_of(op), builder);
1832 let bool_result = builder.ins().vall_true(a);
1833 state.push1(builder.ins().uextend(I32, bool_result));
1834 }
1835 Operator::I8x16Bitmask
1836 | Operator::I16x8Bitmask
1837 | Operator::I32x4Bitmask
1838 | Operator::I64x2Bitmask => {
1839 let a = pop1_with_bitcast(state, type_of(op), builder);
1840 state.push1(builder.ins().vhigh_bits(I32, a));
1841 }
1842 Operator::I8x16Eq | Operator::I16x8Eq | Operator::I32x4Eq | Operator::I64x2Eq => {
1843 translate_vector_icmp(IntCC::Equal, type_of(op), builder, state);
1844 }
1845 Operator::I8x16Ne | Operator::I16x8Ne | Operator::I32x4Ne | Operator::I64x2Ne => {
1846 translate_vector_icmp(IntCC::NotEqual, type_of(op), builder, state);
1847 }
1848 Operator::I8x16GtS | Operator::I16x8GtS | Operator::I32x4GtS | Operator::I64x2GtS => {
1849 translate_vector_icmp(IntCC::SignedGreaterThan, type_of(op), builder, state);
1850 }
1851 Operator::I8x16LtS | Operator::I16x8LtS | Operator::I32x4LtS | Operator::I64x2LtS => {
1852 translate_vector_icmp(IntCC::SignedLessThan, type_of(op), builder, state);
1853 }
1854 Operator::I8x16GtU | Operator::I16x8GtU | Operator::I32x4GtU => {
1855 translate_vector_icmp(IntCC::UnsignedGreaterThan, type_of(op), builder, state);
1856 }
1857 Operator::I8x16LtU | Operator::I16x8LtU | Operator::I32x4LtU => {
1858 translate_vector_icmp(IntCC::UnsignedLessThan, type_of(op), builder, state);
1859 }
1860 Operator::I8x16GeS | Operator::I16x8GeS | Operator::I32x4GeS | Operator::I64x2GeS => {
1861 translate_vector_icmp(IntCC::SignedGreaterThanOrEqual, type_of(op), builder, state);
1862 }
1863 Operator::I8x16LeS | Operator::I16x8LeS | Operator::I32x4LeS | Operator::I64x2LeS => {
1864 translate_vector_icmp(IntCC::SignedLessThanOrEqual, type_of(op), builder, state);
1865 }
1866 Operator::I8x16GeU | Operator::I16x8GeU | Operator::I32x4GeU => translate_vector_icmp(
1867 IntCC::UnsignedGreaterThanOrEqual,
1868 type_of(op),
1869 builder,
1870 state,
1871 ),
1872 Operator::I8x16LeU | Operator::I16x8LeU | Operator::I32x4LeU => {
1873 translate_vector_icmp(IntCC::UnsignedLessThanOrEqual, type_of(op), builder, state);
1874 }
1875 Operator::F32x4Eq | Operator::F64x2Eq => {
1876 translate_vector_fcmp(FloatCC::Equal, type_of(op), builder, state);
1877 }
1878 Operator::F32x4Ne | Operator::F64x2Ne => {
1879 translate_vector_fcmp(FloatCC::NotEqual, type_of(op), builder, state);
1880 }
1881 Operator::F32x4Lt | Operator::F64x2Lt => {
1882 translate_vector_fcmp(FloatCC::LessThan, type_of(op), builder, state);
1883 }
1884 Operator::F32x4Gt | Operator::F64x2Gt => {
1885 translate_vector_fcmp(FloatCC::GreaterThan, type_of(op), builder, state);
1886 }
1887 Operator::F32x4Le | Operator::F64x2Le => {
1888 translate_vector_fcmp(FloatCC::LessThanOrEqual, type_of(op), builder, state);
1889 }
1890 Operator::F32x4Ge | Operator::F64x2Ge => {
1891 translate_vector_fcmp(FloatCC::GreaterThanOrEqual, type_of(op), builder, state);
1892 }
1893 Operator::F32x4Add | Operator::F64x2Add => {
1894 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1895 state.push1(builder.ins().fadd(a, b));
1896 }
1897 Operator::F32x4Sub | Operator::F64x2Sub => {
1898 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1899 state.push1(builder.ins().fsub(a, b));
1900 }
1901 Operator::F32x4Mul | Operator::F64x2Mul => {
1902 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1903 state.push1(builder.ins().fmul(a, b));
1904 }
1905 Operator::F32x4Div | Operator::F64x2Div => {
1906 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1907 state.push1(builder.ins().fdiv(a, b));
1908 }
1909 Operator::F32x4Max | Operator::F64x2Max => {
1910 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1911 state.push1(builder.ins().fmax(a, b));
1912 }
1913 Operator::F32x4Min | Operator::F64x2Min => {
1914 let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
1915 state.push1(builder.ins().fmin(a, b));
1916 }
1917 Operator::F32x4PMax | Operator::F64x2PMax => {
1918 // Note the careful ordering here with respect to `fcmp` and
1919 // `bitselect`. This matches the spec definition of:
1920 //
1921 // fpmax(z1, z2) =
1922 // * If z1 is less than z2 then return z2.
1923 // * Else return z1.
1924 let ty = type_of(op);
1925 let (a, b) = pop2_with_bitcast(state, ty, builder);
1926 let cmp = builder.ins().fcmp(FloatCC::LessThan, a, b);
1927 let cmp = optionally_bitcast_vector(cmp, ty, builder);
1928 state.push1(builder.ins().bitselect(cmp, b, a));
1929 }
1930 Operator::F32x4PMin | Operator::F64x2PMin => {
1931 // Note the careful ordering here which is similar to `pmax` above:
1932 //
1933 // fpmin(z1, z2) =
1934 // * If z2 is less than z1 then return z2.
1935 // * Else return z1.
1936 let ty = type_of(op);
1937 let (a, b) = pop2_with_bitcast(state, ty, builder);
1938 let cmp = builder.ins().fcmp(FloatCC::LessThan, b, a);
1939 let cmp = optionally_bitcast_vector(cmp, ty, builder);
1940 state.push1(builder.ins().bitselect(cmp, b, a));
1941 }
1942 Operator::F32x4Sqrt | Operator::F64x2Sqrt => {
1943 let a = pop1_with_bitcast(state, type_of(op), builder);
1944 state.push1(builder.ins().sqrt(a));
1945 }
1946 Operator::F32x4Neg | Operator::F64x2Neg => {
1947 let a = pop1_with_bitcast(state, type_of(op), builder);
1948 state.push1(builder.ins().fneg(a));
1949 }
1950 Operator::F32x4Abs | Operator::F64x2Abs => {
1951 let a = pop1_with_bitcast(state, type_of(op), builder);
1952 state.push1(builder.ins().fabs(a));
1953 }
1954 Operator::F32x4ConvertI32x4S => {
1955 let a = pop1_with_bitcast(state, I32X4, builder);
1956 state.push1(builder.ins().fcvt_from_sint(F32X4, a));
1957 }
1958 Operator::F32x4ConvertI32x4U => {
1959 let a = pop1_with_bitcast(state, I32X4, builder);
1960 state.push1(builder.ins().fcvt_from_uint(F32X4, a));
1961 }
1962 Operator::F64x2ConvertLowI32x4S => {
1963 let a = pop1_with_bitcast(state, I32X4, builder);
1964 let widened_a = builder.ins().swiden_low(a);
1965 state.push1(builder.ins().fcvt_from_sint(F64X2, widened_a));
1966 }
1967 Operator::F64x2ConvertLowI32x4U => {
1968 let a = pop1_with_bitcast(state, I32X4, builder);
1969 let widened_a = builder.ins().uwiden_low(a);
1970 state.push1(builder.ins().fcvt_from_uint(F64X2, widened_a));
1971 }
1972 Operator::F64x2PromoteLowF32x4 => {
1973 let a = pop1_with_bitcast(state, F32X4, builder);
1974 state.push1(builder.ins().fvpromote_low(a));
1975 }
1976 Operator::F32x4DemoteF64x2Zero => {
1977 let a = pop1_with_bitcast(state, F64X2, builder);
1978 state.push1(builder.ins().fvdemote(a));
1979 }
1980 Operator::I32x4TruncSatF32x4S => {
1981 let a = pop1_with_bitcast(state, F32X4, builder);
1982 state.push1(builder.ins().fcvt_to_sint_sat(I32X4, a));
1983 }
1984 Operator::I32x4TruncSatF64x2SZero => {
1985 let a = pop1_with_bitcast(state, F64X2, builder);
1986 let converted_a = builder.ins().fcvt_to_sint_sat(I64X2, a);
1987 let handle = builder.func.dfg.constants.insert(vec![0u8; 16].into());
1988 let zero = builder.ins().vconst(I64X2, handle);
1989
1990 state.push1(builder.ins().snarrow(converted_a, zero));
1991 }
1992 // FIXME(#5913): the relaxed instructions here are translated the same
1993 // as the saturating instructions, even when the code generator
1994 // configuration allow for different semantics across hosts. On x86,
1995 // however, it's theoretically possible to have a slightly more optimal
1996 // lowering which accounts for NaN differently, although the lowering is
1997 // still not trivial (e.g. one instruction). At this time the
1998 // more-optimal-but-still-large lowering for x86 is not implemented so
1999 // the relaxed instructions are listed here instead of down below with
2000 // the other relaxed instructions. An x86-specific implementation (or
2001 // perhaps for other backends too) should be added and the codegen for
2002 // the relaxed instruction should conditionally be different.
2003 Operator::I32x4RelaxedTruncF32x4U | Operator::I32x4TruncSatF32x4U => {
2004 let a = pop1_with_bitcast(state, F32X4, builder);
2005 state.push1(builder.ins().fcvt_to_uint_sat(I32X4, a));
2006 }
2007 Operator::I32x4RelaxedTruncF64x2UZero | Operator::I32x4TruncSatF64x2UZero => {
2008 let a = pop1_with_bitcast(state, F64X2, builder);
2009 let converted_a = builder.ins().fcvt_to_uint_sat(I64X2, a);
2010 let handle = builder.func.dfg.constants.insert(vec![0u8; 16].into());
2011 let zero = builder.ins().vconst(I64X2, handle);
2012
2013 state.push1(builder.ins().uunarrow(converted_a, zero));
2014 }
2015 Operator::I8x16NarrowI16x8S => {
2016 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2017 state.push1(builder.ins().snarrow(a, b));
2018 }
2019 Operator::I16x8NarrowI32x4S => {
2020 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
2021 state.push1(builder.ins().snarrow(a, b));
2022 }
2023 Operator::I8x16NarrowI16x8U => {
2024 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2025 state.push1(builder.ins().unarrow(a, b));
2026 }
2027 Operator::I16x8NarrowI32x4U => {
2028 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
2029 state.push1(builder.ins().unarrow(a, b));
2030 }
2031 Operator::I16x8ExtendLowI8x16S => {
2032 let a = pop1_with_bitcast(state, I8X16, builder);
2033 state.push1(builder.ins().swiden_low(a));
2034 }
2035 Operator::I16x8ExtendHighI8x16S => {
2036 let a = pop1_with_bitcast(state, I8X16, builder);
2037 state.push1(builder.ins().swiden_high(a));
2038 }
2039 Operator::I16x8ExtendLowI8x16U => {
2040 let a = pop1_with_bitcast(state, I8X16, builder);
2041 state.push1(builder.ins().uwiden_low(a));
2042 }
2043 Operator::I16x8ExtendHighI8x16U => {
2044 let a = pop1_with_bitcast(state, I8X16, builder);
2045 state.push1(builder.ins().uwiden_high(a));
2046 }
2047 Operator::I32x4ExtendLowI16x8S => {
2048 let a = pop1_with_bitcast(state, I16X8, builder);
2049 state.push1(builder.ins().swiden_low(a));
2050 }
2051 Operator::I32x4ExtendHighI16x8S => {
2052 let a = pop1_with_bitcast(state, I16X8, builder);
2053 state.push1(builder.ins().swiden_high(a));
2054 }
2055 Operator::I32x4ExtendLowI16x8U => {
2056 let a = pop1_with_bitcast(state, I16X8, builder);
2057 state.push1(builder.ins().uwiden_low(a));
2058 }
2059 Operator::I32x4ExtendHighI16x8U => {
2060 let a = pop1_with_bitcast(state, I16X8, builder);
2061 state.push1(builder.ins().uwiden_high(a));
2062 }
2063 Operator::I64x2ExtendLowI32x4S => {
2064 let a = pop1_with_bitcast(state, I32X4, builder);
2065 state.push1(builder.ins().swiden_low(a));
2066 }
2067 Operator::I64x2ExtendHighI32x4S => {
2068 let a = pop1_with_bitcast(state, I32X4, builder);
2069 state.push1(builder.ins().swiden_high(a));
2070 }
2071 Operator::I64x2ExtendLowI32x4U => {
2072 let a = pop1_with_bitcast(state, I32X4, builder);
2073 state.push1(builder.ins().uwiden_low(a));
2074 }
2075 Operator::I64x2ExtendHighI32x4U => {
2076 let a = pop1_with_bitcast(state, I32X4, builder);
2077 state.push1(builder.ins().uwiden_high(a));
2078 }
2079 Operator::I16x8ExtAddPairwiseI8x16S => {
2080 let a = pop1_with_bitcast(state, I8X16, builder);
2081 let widen_low = builder.ins().swiden_low(a);
2082 let widen_high = builder.ins().swiden_high(a);
2083 state.push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2084 }
2085 Operator::I32x4ExtAddPairwiseI16x8S => {
2086 let a = pop1_with_bitcast(state, I16X8, builder);
2087 let widen_low = builder.ins().swiden_low(a);
2088 let widen_high = builder.ins().swiden_high(a);
2089 state.push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2090 }
2091 Operator::I16x8ExtAddPairwiseI8x16U => {
2092 let a = pop1_with_bitcast(state, I8X16, builder);
2093 let widen_low = builder.ins().uwiden_low(a);
2094 let widen_high = builder.ins().uwiden_high(a);
2095 state.push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2096 }
2097 Operator::I32x4ExtAddPairwiseI16x8U => {
2098 let a = pop1_with_bitcast(state, I16X8, builder);
2099 let widen_low = builder.ins().uwiden_low(a);
2100 let widen_high = builder.ins().uwiden_high(a);
2101 state.push1(builder.ins().iadd_pairwise(widen_low, widen_high));
2102 }
2103 Operator::F32x4Ceil | Operator::F64x2Ceil => {
2104 // This is something of a misuse of `type_of`, because that produces the return type
2105 // of `op`. In this case we want the arg type, but we know it's the same as the
2106 // return type. Same for the 3 cases below.
2107 let arg = pop1_with_bitcast(state, type_of(op), builder);
2108 state.push1(builder.ins().ceil(arg));
2109 }
2110 Operator::F32x4Floor | Operator::F64x2Floor => {
2111 let arg = pop1_with_bitcast(state, type_of(op), builder);
2112 state.push1(builder.ins().floor(arg));
2113 }
2114 Operator::F32x4Trunc | Operator::F64x2Trunc => {
2115 let arg = pop1_with_bitcast(state, type_of(op), builder);
2116 state.push1(builder.ins().trunc(arg));
2117 }
2118 Operator::F32x4Nearest | Operator::F64x2Nearest => {
2119 let arg = pop1_with_bitcast(state, type_of(op), builder);
2120 state.push1(builder.ins().nearest(arg));
2121 }
2122 Operator::I32x4DotI16x8S => {
2123 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2124 let alo = builder.ins().swiden_low(a);
2125 let blo = builder.ins().swiden_low(b);
2126 let lo = builder.ins().imul(alo, blo);
2127 let ahi = builder.ins().swiden_high(a);
2128 let bhi = builder.ins().swiden_high(b);
2129 let hi = builder.ins().imul(ahi, bhi);
2130 state.push1(builder.ins().iadd_pairwise(lo, hi));
2131 }
2132 Operator::I8x16Popcnt => {
2133 let arg = pop1_with_bitcast(state, type_of(op), builder);
2134 state.push1(builder.ins().popcnt(arg));
2135 }
2136 Operator::I16x8Q15MulrSatS => {
2137 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2138 state.push1(builder.ins().sqmul_round_sat(a, b));
2139 }
2140 Operator::I16x8ExtMulLowI8x16S => {
2141 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2142 let a_low = builder.ins().swiden_low(a);
2143 let b_low = builder.ins().swiden_low(b);
2144 state.push1(builder.ins().imul(a_low, b_low));
2145 }
2146 Operator::I16x8ExtMulHighI8x16S => {
2147 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2148 let a_high = builder.ins().swiden_high(a);
2149 let b_high = builder.ins().swiden_high(b);
2150 state.push1(builder.ins().imul(a_high, b_high));
2151 }
2152 Operator::I16x8ExtMulLowI8x16U => {
2153 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2154 let a_low = builder.ins().uwiden_low(a);
2155 let b_low = builder.ins().uwiden_low(b);
2156 state.push1(builder.ins().imul(a_low, b_low));
2157 }
2158 Operator::I16x8ExtMulHighI8x16U => {
2159 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2160 let a_high = builder.ins().uwiden_high(a);
2161 let b_high = builder.ins().uwiden_high(b);
2162 state.push1(builder.ins().imul(a_high, b_high));
2163 }
2164 Operator::I32x4ExtMulLowI16x8S => {
2165 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2166 let a_low = builder.ins().swiden_low(a);
2167 let b_low = builder.ins().swiden_low(b);
2168 state.push1(builder.ins().imul(a_low, b_low));
2169 }
2170 Operator::I32x4ExtMulHighI16x8S => {
2171 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2172 let a_high = builder.ins().swiden_high(a);
2173 let b_high = builder.ins().swiden_high(b);
2174 state.push1(builder.ins().imul(a_high, b_high));
2175 }
2176 Operator::I32x4ExtMulLowI16x8U => {
2177 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2178 let a_low = builder.ins().uwiden_low(a);
2179 let b_low = builder.ins().uwiden_low(b);
2180 state.push1(builder.ins().imul(a_low, b_low));
2181 }
2182 Operator::I32x4ExtMulHighI16x8U => {
2183 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2184 let a_high = builder.ins().uwiden_high(a);
2185 let b_high = builder.ins().uwiden_high(b);
2186 state.push1(builder.ins().imul(a_high, b_high));
2187 }
2188 Operator::I64x2ExtMulLowI32x4S => {
2189 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
2190 let a_low = builder.ins().swiden_low(a);
2191 let b_low = builder.ins().swiden_low(b);
2192 state.push1(builder.ins().imul(a_low, b_low));
2193 }
2194 Operator::I64x2ExtMulHighI32x4S => {
2195 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
2196 let a_high = builder.ins().swiden_high(a);
2197 let b_high = builder.ins().swiden_high(b);
2198 state.push1(builder.ins().imul(a_high, b_high));
2199 }
2200 Operator::I64x2ExtMulLowI32x4U => {
2201 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
2202 let a_low = builder.ins().uwiden_low(a);
2203 let b_low = builder.ins().uwiden_low(b);
2204 state.push1(builder.ins().imul(a_low, b_low));
2205 }
2206 Operator::I64x2ExtMulHighI32x4U => {
2207 let (a, b) = pop2_with_bitcast(state, I32X4, builder);
2208 let a_high = builder.ins().uwiden_high(a);
2209 let b_high = builder.ins().uwiden_high(b);
2210 state.push1(builder.ins().imul(a_high, b_high));
2211 }
2212
2213 // Relaxed SIMD operators
2214 // https://github.com/WebAssembly/relaxed-simd
2215 Operator::F32x4RelaxedMax | Operator::F64x2RelaxedMax => {
2216 let ty = type_of(op);
2217 let (a, b) = pop2_with_bitcast(state, ty, builder);
2218 state.push1(if env.relaxed_simd_deterministic() || !env.is_x86() {
2219 // Deterministic semantics match the `fmax` instruction, or
2220 // the `fAAxBB.max` wasm instruction.
2221 builder.ins().fmax(a, b)
2222 } else {
2223 // Note that this matches the `pmax` translation which has
2224 // careful ordering of its operands to trigger
2225 // pattern-matches in the x86 backend.
2226 let cmp = builder.ins().fcmp(FloatCC::LessThan, a, b);
2227 let cmp = optionally_bitcast_vector(cmp, ty, builder);
2228 builder.ins().bitselect(cmp, b, a)
2229 });
2230 }
2231
2232 Operator::F32x4RelaxedMin | Operator::F64x2RelaxedMin => {
2233 let ty = type_of(op);
2234 let (a, b) = pop2_with_bitcast(state, ty, builder);
2235 state.push1(if env.relaxed_simd_deterministic() || !env.is_x86() {
2236 // Deterministic semantics match the `fmin` instruction, or
2237 // the `fAAxBB.min` wasm instruction.
2238 builder.ins().fmin(a, b)
2239 } else {
2240 // Note that this matches the `pmin` translation which has
2241 // careful ordering of its operands to trigger
2242 // pattern-matches in the x86 backend.
2243 let cmp = builder.ins().fcmp(FloatCC::LessThan, b, a);
2244 let cmp = optionally_bitcast_vector(cmp, ty, builder);
2245 builder.ins().bitselect(cmp, b, a)
2246 });
2247 }
2248
2249 Operator::I8x16RelaxedSwizzle => {
2250 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2251 state.push1(
2252 if env.relaxed_simd_deterministic() || !env.use_x86_pshufb_for_relaxed_swizzle() {
2253 // Deterministic semantics match the `i8x16.swizzle`
2254 // instruction which is the CLIF `swizzle`.
2255 builder.ins().swizzle(a, b)
2256 } else {
2257 builder.ins().x86_pshufb(a, b)
2258 },
2259 );
2260 }
2261
2262 Operator::F32x4RelaxedMadd | Operator::F64x2RelaxedMadd => {
2263 let (a, b, c) = pop3_with_bitcast(state, type_of(op), builder);
2264 state.push1(
2265 if env.relaxed_simd_deterministic() || env.has_native_fma() {
2266 // Deterministic semantics are "fused multiply and add"
2267 // which the CLIF `fma` guarantees.
2268 builder.ins().fma(a, b, c)
2269 } else {
2270 let mul = builder.ins().fmul(a, b);
2271 builder.ins().fadd(mul, c)
2272 },
2273 );
2274 }
2275 Operator::F32x4RelaxedNmadd | Operator::F64x2RelaxedNmadd => {
2276 let (a, b, c) = pop3_with_bitcast(state, type_of(op), builder);
2277 let a = builder.ins().fneg(a);
2278 state.push1(
2279 if env.relaxed_simd_deterministic() || env.has_native_fma() {
2280 // Deterministic semantics are "fused multiply and add"
2281 // which the CLIF `fma` guarantees.
2282 builder.ins().fma(a, b, c)
2283 } else {
2284 let mul = builder.ins().fmul(a, b);
2285 builder.ins().fadd(mul, c)
2286 },
2287 );
2288 }
2289
2290 Operator::I8x16RelaxedLaneselect
2291 | Operator::I16x8RelaxedLaneselect
2292 | Operator::I32x4RelaxedLaneselect
2293 | Operator::I64x2RelaxedLaneselect => {
2294 let ty = type_of(op);
2295 let (a, b, c) = pop3_with_bitcast(state, ty, builder);
2296 // Note that the variable swaps here are intentional due to
2297 // the difference of the order of the wasm op and the clif
2298 // op.
2299 state.push1(
2300 if env.relaxed_simd_deterministic()
2301 || !env.use_x86_blendv_for_relaxed_laneselect(ty)
2302 {
2303 // Deterministic semantics are a `bitselect` along the lines
2304 // of the wasm `v128.bitselect` instruction.
2305 builder.ins().bitselect(c, a, b)
2306 } else {
2307 builder.ins().x86_blendv(c, a, b)
2308 },
2309 );
2310 }
2311
2312 Operator::I32x4RelaxedTruncF32x4S => {
2313 let a = pop1_with_bitcast(state, F32X4, builder);
2314 state.push1(if env.relaxed_simd_deterministic() || !env.is_x86() {
2315 // Deterministic semantics are to match the
2316 // `i32x4.trunc_sat_f32x4_s` instruction.
2317 builder.ins().fcvt_to_sint_sat(I32X4, a)
2318 } else {
2319 builder.ins().x86_cvtt2dq(I32X4, a)
2320 });
2321 }
2322 Operator::I32x4RelaxedTruncF64x2SZero => {
2323 let a = pop1_with_bitcast(state, F64X2, builder);
2324 let converted_a = if env.relaxed_simd_deterministic() || !env.is_x86() {
2325 // Deterministic semantics are to match the
2326 // `i32x4.trunc_sat_f64x2_s_zero` instruction.
2327 builder.ins().fcvt_to_sint_sat(I64X2, a)
2328 } else {
2329 builder.ins().x86_cvtt2dq(I64X2, a)
2330 };
2331 let handle = builder.func.dfg.constants.insert(vec![0u8; 16].into());
2332 let zero = builder.ins().vconst(I64X2, handle);
2333
2334 state.push1(builder.ins().snarrow(converted_a, zero));
2335 }
2336 Operator::I16x8RelaxedQ15mulrS => {
2337 let (a, b) = pop2_with_bitcast(state, I16X8, builder);
2338 state.push1(
2339 if env.relaxed_simd_deterministic() || !env.use_x86_pmulhrsw_for_relaxed_q15mul() {
2340 // Deterministic semantics are to match the
2341 // `i16x8.q15mulr_sat_s` instruction.
2342 builder.ins().sqmul_round_sat(a, b)
2343 } else {
2344 builder.ins().x86_pmulhrsw(a, b)
2345 },
2346 );
2347 }
2348 Operator::I16x8RelaxedDotI8x16I7x16S => {
2349 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2350 state.push1(
2351 if env.relaxed_simd_deterministic() || !env.use_x86_pmaddubsw_for_dot() {
2352 // Deterministic semantics are to treat both operands as
2353 // signed integers and perform the dot product.
2354 let alo = builder.ins().swiden_low(a);
2355 let blo = builder.ins().swiden_low(b);
2356 let lo = builder.ins().imul(alo, blo);
2357 let ahi = builder.ins().swiden_high(a);
2358 let bhi = builder.ins().swiden_high(b);
2359 let hi = builder.ins().imul(ahi, bhi);
2360 builder.ins().iadd_pairwise(lo, hi)
2361 } else {
2362 builder.ins().x86_pmaddubsw(a, b)
2363 },
2364 );
2365 }
2366
2367 Operator::I32x4RelaxedDotI8x16I7x16AddS => {
2368 let c = pop1_with_bitcast(state, I32X4, builder);
2369 let (a, b) = pop2_with_bitcast(state, I8X16, builder);
2370 let dot = if env.relaxed_simd_deterministic() || !env.use_x86_pmaddubsw_for_dot() {
2371 // Deterministic semantics are to treat both operands as
2372 // signed integers and perform the dot product.
2373 let alo = builder.ins().swiden_low(a);
2374 let blo = builder.ins().swiden_low(b);
2375 let lo = builder.ins().imul(alo, blo);
2376 let ahi = builder.ins().swiden_high(a);
2377 let bhi = builder.ins().swiden_high(b);
2378 let hi = builder.ins().imul(ahi, bhi);
2379 builder.ins().iadd_pairwise(lo, hi)
2380 } else {
2381 builder.ins().x86_pmaddubsw(a, b)
2382 };
2383 let dotlo = builder.ins().swiden_low(dot);
2384 let dothi = builder.ins().swiden_high(dot);
2385 let dot32 = builder.ins().iadd_pairwise(dotlo, dothi);
2386 state.push1(builder.ins().iadd(dot32, c));
2387 }
2388
2389 // Typed Function references
2390 // https://github.com/WebAssembly/function-references
2391 Operator::CallRef { type_index } => {
2392 // Get function signature
2393 // `index` is the index of the function's signature and `table_index` is the index of
2394 // the table to search the function in.
2395 let (sigref, num_args) =
2396 state.get_indirect_sig(builder.func, TypeIndex::from_u32(*type_index), env);
2397 let callee = state.pop1();
2398
2399 // Get the `callee` operand type and check whether it's `Some(Some(<reference type>))` and
2400 // that the reference type is non-nullable in which case we can omit the null check.
2401 // If `get_operand_type` returns `Some(None)` that means it doesn't know in which case we
2402 // default to explicit null checks too.
2403 let ty = validator.get_operand_type(0);
2404 let needs_null_check = ty.expect("expected operand on stack").is_none_or(|ty| {
2405 let ty = ty.as_reference_type().expect("expected reference type");
2406
2407 ty.is_nullable()
2408 });
2409
2410 // Bitcast any vector arguments to their default type, I8X16, before calling.
2411 let args = state.peekn_mut(num_args);
2412 bitcast_wasm_params(sigref, args, builder, env);
2413
2414 let call = env.translate_call_ref(
2415 builder,
2416 sigref,
2417 callee,
2418 state.peekn(num_args),
2419 needs_null_check,
2420 )?;
2421
2422 let inst_results = builder.inst_results(call);
2423 debug_assert_eq!(
2424 inst_results.len(),
2425 builder.func.dfg.signatures[sigref].returns.len(),
2426 "translate_call_ref results should match the call signature"
2427 );
2428 state.popn(num_args);
2429 state.pushn(inst_results);
2430 }
2431 Operator::RefAsNonNull => {
2432 let r = state.pop1();
2433 let is_null = env.translate_ref_is_null(builder.cursor(), r);
2434 builder.ins().trapnz(is_null, TRAP_NULL_REFERENCE);
2435 state.push1(r);
2436 }
2437 Operator::BrOnNull { relative_depth } => {
2438 let r = state.pop1();
2439 let (br_destination, inputs) = translate_br_if_args(*relative_depth, state);
2440 let is_null = env.translate_ref_is_null(builder.cursor(), r);
2441 let else_block = builder.create_block();
2442 canonicalise_brif(builder, is_null, br_destination, inputs, else_block, &[]);
2443
2444 builder.seal_block(else_block); // The only predecessor is the current block.
2445 builder.switch_to_block(else_block);
2446 state.push1(r);
2447 }
2448 Operator::BrOnNonNull { relative_depth } => {
2449 // We write this a bit differently from the spec to avoid an extra
2450 // block/branch and the typed accounting thereof. Instead of the
2451 // spec's approach, it's described as such:
2452 // Peek the value val from the stack.
2453 // If val is ref.null ht, then: pop the value val from the stack.
2454 // Else: Execute the instruction (br relative_depth).
2455 let is_null = env.translate_ref_is_null(builder.cursor(), state.peek1());
2456 let (br_destination, inputs) = translate_br_if_args(*relative_depth, state);
2457 let else_block = builder.create_block();
2458 canonicalise_brif(builder, is_null, else_block, &[], br_destination, inputs);
2459
2460 // In the null case, pop the ref
2461 state.pop1();
2462
2463 builder.seal_block(else_block); // The only predecessor is the current block.
2464
2465 // The rest of the translation operates on our is null case, which is
2466 // currently an empty block
2467 builder.switch_to_block(else_block);
2468 }
2469 Operator::ReturnCallRef { type_index } => {
2470 // Get function signature
2471 // `index` is the index of the function's signature and `table_index` is the index of
2472 // the table to search the function in.
2473 let (sigref, num_args) =
2474 state.get_indirect_sig(builder.func, TypeIndex::from_u32(*type_index), env);
2475 let callee = state.pop1();
2476
2477 // Bitcast any vector arguments to their default type, I8X16, before calling.
2478 let args = state.peekn_mut(num_args);
2479 bitcast_wasm_params(sigref, args, builder, env);
2480
2481 env.translate_return_call_ref(builder, sigref, callee, state.peekn(num_args))?;
2482
2483 state.popn(num_args);
2484 state.reachable = false;
2485 }
2486
2487 // Garbage Collection
2488 // http://github.com/WebAssembly/gc
2489 Operator::RefI31 => {
2490 let val = state.pop1();
2491 let i31ref = env.translate_ref_i31(builder.cursor(), val)?;
2492 state.push1(i31ref);
2493 }
2494 Operator::I31GetS => {
2495 let i31ref = state.pop1();
2496 let val = env.translate_i31_get_s(builder.cursor(), i31ref)?;
2497 state.push1(val);
2498 }
2499 Operator::I31GetU => {
2500 let i31ref = state.pop1();
2501 let val = env.translate_i31_get_u(builder.cursor(), i31ref)?;
2502 state.push1(val);
2503 }
2504 Operator::StructNew { .. }
2505 | Operator::StructNewDefault { .. }
2506 | Operator::StructGet { .. }
2507 | Operator::StructGetS { .. }
2508 | Operator::StructGetU { .. }
2509 | Operator::StructSet { .. }
2510 | Operator::ArrayNew { .. }
2511 | Operator::ArrayNewDefault { .. }
2512 | Operator::ArrayNewFixed { .. }
2513 | Operator::ArrayNewData { .. }
2514 | Operator::ArrayNewElem { .. }
2515 | Operator::ArrayGet { .. }
2516 | Operator::ArrayGetS { .. }
2517 | Operator::ArrayGetU { .. }
2518 | Operator::ArraySet { .. }
2519 | Operator::ArrayLen
2520 | Operator::ArrayFill { .. }
2521 | Operator::ArrayCopy { .. }
2522 | Operator::ArrayInitData { .. }
2523 | Operator::ArrayInitElem { .. }
2524 | Operator::RefTestNonNull { .. }
2525 | Operator::RefTestNullable { .. }
2526 | Operator::RefCastNonNull { .. }
2527 | Operator::RefCastNullable { .. }
2528 | Operator::BrOnCast { .. }
2529 | Operator::BrOnCastFail { .. }
2530 | Operator::AnyConvertExtern
2531 | Operator::ExternConvertAny
2532 | Operator::RefEq => {
2533 bail!(
2534 "Feature used by the WebAssembly code is not yet supported: Garbage Collection Proposal"
2535 );
2536 }
2537
2538 /******************************* Active Proposals *****************************************/
2539 // memory control (experimental)
2540 // https://github.com/WebAssembly/memory-control
2541 Operator::MemoryDiscard { .. } => {
2542 bail!(
2543 "Feature used by the WebAssembly code is not yet supported: Memory Control Proposal"
2544 );
2545 }
2546
2547 // shared-everything threads
2548 // https://github.com/WebAssembly/shared-everything-threads
2549 Operator::GlobalAtomicGet { .. }
2550 | Operator::GlobalAtomicSet { .. }
2551 | Operator::GlobalAtomicRmwAdd { .. }
2552 | Operator::GlobalAtomicRmwSub { .. }
2553 | Operator::GlobalAtomicRmwAnd { .. }
2554 | Operator::GlobalAtomicRmwOr { .. }
2555 | Operator::GlobalAtomicRmwXor { .. }
2556 | Operator::GlobalAtomicRmwXchg { .. }
2557 | Operator::GlobalAtomicRmwCmpxchg { .. }
2558 | Operator::TableAtomicGet { .. }
2559 | Operator::TableAtomicSet { .. }
2560 | Operator::TableAtomicRmwXchg { .. }
2561 | Operator::TableAtomicRmwCmpxchg { .. }
2562 | Operator::StructAtomicGet { .. }
2563 | Operator::StructAtomicGetS { .. }
2564 | Operator::StructAtomicGetU { .. }
2565 | Operator::StructAtomicSet { .. }
2566 | Operator::StructAtomicRmwAdd { .. }
2567 | Operator::StructAtomicRmwSub { .. }
2568 | Operator::StructAtomicRmwAnd { .. }
2569 | Operator::StructAtomicRmwOr { .. }
2570 | Operator::StructAtomicRmwXor { .. }
2571 | Operator::StructAtomicRmwXchg { .. }
2572 | Operator::StructAtomicRmwCmpxchg { .. }
2573 | Operator::ArrayAtomicGet { .. }
2574 | Operator::ArrayAtomicGetS { .. }
2575 | Operator::ArrayAtomicGetU { .. }
2576 | Operator::ArrayAtomicSet { .. }
2577 | Operator::ArrayAtomicRmwAdd { .. }
2578 | Operator::ArrayAtomicRmwSub { .. }
2579 | Operator::ArrayAtomicRmwAnd { .. }
2580 | Operator::ArrayAtomicRmwOr { .. }
2581 | Operator::ArrayAtomicRmwXor { .. }
2582 | Operator::ArrayAtomicRmwXchg { .. }
2583 | Operator::ArrayAtomicRmwCmpxchg { .. }
2584 | Operator::RefI31Shared => {
2585 bail!(
2586 "Feature used by the WebAssembly code is not yet supported: Shared-Everything Threads Proposal"
2587 );
2588 }
2589
2590 // Exception handling
2591 // https://github.com/WebAssembly/exception-handling
2592 Operator::TryTable { .. } | Operator::Throw { .. } | Operator::ThrowRef => {
2593 bail!(
2594 "Feature used by the WebAssembly code is not yet supported: Exception Handling Proposal"
2595 );
2596 }
2597 // Deprecated old instructions from the exceptions proposal
2598 Operator::Try { .. }
2599 | Operator::Catch { .. }
2600 | Operator::Rethrow { .. }
2601 | Operator::Delegate { .. }
2602 | Operator::CatchAll => {
2603 bail!(
2604 "Feature used by the WebAssembly code is not yet supported: Legacy Exception Handling Proposal"
2605 );
2606 }
2607
2608 // Stack switching
2609 // https://github.com/WebAssembly/stack-switching
2610 Operator::ContNew { .. }
2611 | Operator::ContBind { .. }
2612 | Operator::Suspend { .. }
2613 | Operator::Resume { .. }
2614 | Operator::ResumeThrow { .. }
2615 | Operator::Switch { .. } => {
2616 bail!(
2617 "Feature used by the WebAssembly code is not yet supported: Stack Switching Proposal"
2618 );
2619 }
2620
2621 // wide arithmetic
2622 // https://github.com/WebAssembly/wide-arithmetic
2623 Operator::I64Add128
2624 | Operator::I64Sub128
2625 | Operator::I64MulWideS
2626 | Operator::I64MulWideU => {
2627 bail!(
2628 "Feature used by the WebAssembly code is not yet supported: Wide Arithmetic Proposal"
2629 );
2630 }
2631
2632 _ => unimplemented!(),
2633 }
2634
2635 Ok(())
2636}
2637
2638/// Deals with a Wasm instruction located in an unreachable portion of the code. Most of them
2639/// are dropped but special ones like `End` or `Else` signal the potential end of the unreachable
2640/// portion so the translation state must be updated accordingly.
2641fn translate_unreachable_operator(
2642 validator: &FuncValidator<impl WasmModuleResources>,
2643 op: &Operator,
2644 builder: &mut FunctionBuilder,
2645 state: &mut FuncTranslationState,
2646 env: &mut TranslationEnvironment,
2647) {
2648 debug_assert!(!state.reachable);
2649 match *op {
2650 Operator::If { blockty } => {
2651 // Push a placeholder control stack entry. The if isn't reachable,
2652 // so we don't have any branches anywhere.
2653 state.push_if(
2654 ir::Block::reserved_value(),
2655 ElseData::NoElse {
2656 branch_inst: ir::Inst::reserved_value(),
2657 placeholder: ir::Block::reserved_value(),
2658 },
2659 0,
2660 0,
2661 blockty,
2662 );
2663 }
2664 Operator::Loop { blockty: _ } | Operator::Block { blockty: _ } => {
2665 state.push_block(ir::Block::reserved_value(), 0, 0);
2666 }
2667 Operator::Else => {
2668 let i = state.control_stack.len().checked_sub(1).unwrap();
2669 match state.control_stack[i] {
2670 ControlStackFrame::If {
2671 ref else_data,
2672 head_is_reachable,
2673 ref mut consequent_ends_reachable,
2674 blocktype,
2675 ..
2676 } => {
2677 debug_assert!(consequent_ends_reachable.is_none());
2678 *consequent_ends_reachable = Some(state.reachable);
2679
2680 if head_is_reachable {
2681 // We have a branch from the head of the `if` to the `else`.
2682 state.reachable = true;
2683
2684 let else_block = match *else_data {
2685 ElseData::NoElse {
2686 branch_inst,
2687 placeholder,
2688 } => {
2689 let (params, _results) =
2690 blocktype_params_results(validator, blocktype);
2691 let else_block = block_with_params(builder, params, env);
2692 let frame = state.control_stack.last().unwrap();
2693 frame.truncate_value_stack_to_else_params(&mut state.stack);
2694
2695 // We change the target of the branch instruction.
2696 builder.change_jump_destination(
2697 branch_inst,
2698 placeholder,
2699 else_block,
2700 );
2701 builder.seal_block(else_block);
2702 else_block
2703 }
2704 ElseData::WithElse { else_block } => {
2705 let frame = state.control_stack.last().unwrap();
2706 frame.truncate_value_stack_to_else_params(&mut state.stack);
2707 else_block
2708 }
2709 };
2710
2711 builder.switch_to_block(else_block);
2712
2713 // Again, no need to push the parameters for the `else`,
2714 // since we already did when we saw the original `if`. See
2715 // the comment for translating `Operator::Else` in
2716 // `translate_operator` for details.
2717 }
2718 }
2719 _ => unreachable!(),
2720 }
2721 }
2722 Operator::End => {
2723 let stack = &mut state.stack;
2724 let control_stack = &mut state.control_stack;
2725 let frame = control_stack.pop().unwrap();
2726
2727 // Pop unused parameters from stack.
2728 frame.truncate_value_stack_to_original_size(stack);
2729
2730 let reachable_anyway = match frame {
2731 // If it is a loop we also have to seal the body loop block
2732 ControlStackFrame::Loop { header, .. } => {
2733 builder.seal_block(header);
2734 // And loops can't have branches to the end.
2735 false
2736 }
2737 // Since we are only in this function when in unreachable code,
2738 // we know that the alternative just ended unreachable. Whether
2739 // the following block is reachable depends on if the consequent
2740 // ended reachable or not.
2741 ControlStackFrame::If {
2742 head_is_reachable,
2743 consequent_ends_reachable: Some(consequent_ends_reachable),
2744 ..
2745 } => head_is_reachable && consequent_ends_reachable,
2746 // If we never set `consequent_ends_reachable` then that means
2747 // we are finishing the consequent now, and there was no
2748 // `else`. Whether the following block is reachable depends only
2749 // on if the head was reachable.
2750 ControlStackFrame::If {
2751 head_is_reachable,
2752 consequent_ends_reachable: None,
2753 ..
2754 } => head_is_reachable,
2755 // All other control constructs are already handled.
2756 _ => false,
2757 };
2758
2759 if frame.exit_is_branched_to() || reachable_anyway {
2760 builder.switch_to_block(frame.following_code());
2761 builder.seal_block(frame.following_code());
2762
2763 // And add the return values of the block but only if the next block is reachable
2764 // (which corresponds to testing if the stack depth is 1)
2765 stack.extend_from_slice(builder.block_params(frame.following_code()));
2766 state.reachable = true;
2767 }
2768 }
2769 _ => {
2770 // We don't parse because this is unreachable code
2771 }
2772 }
2773}
2774
2775/// Translate a load instruction.
2776///
2777/// Returns the execution state's reachability after the load is translated.
2778fn translate_load(
2779 memarg: &MemArg,
2780 opcode: ir::Opcode,
2781 result_ty: Type,
2782 builder: &mut FunctionBuilder,
2783 state: &mut FuncTranslationState,
2784 env: &mut TranslationEnvironment,
2785) -> Reachability<()> {
2786 let memory_index = MemoryIndex::from_u32(memarg.memory);
2787 let index = state.pop1();
2788 let mem_op_size = mem_op_size(opcode, result_ty);
2789
2790 let mem = state.get_memory(builder.func, memory_index, env);
2791 let (flags, _wasm_index, base) =
2792 match mem.prepare_addr(builder, index, mem_op_size, memarg, env) {
2793 Reachability::Unreachable => return Reachability::Unreachable,
2794 Reachability::Reachable((f, i, b)) => (f, i, b),
2795 };
2796
2797 let (load, dfg) = builder
2798 .ins()
2799 .Load(opcode, result_ty, flags, Offset32::new(0), base);
2800 state.push1(dfg.first_result(load));
2801
2802 Reachability::Reachable(())
2803}
2804
2805/// Translate a store instruction.
2806fn translate_store(
2807 memarg: &MemArg,
2808 opcode: ir::Opcode,
2809 builder: &mut FunctionBuilder,
2810 state: &mut FuncTranslationState,
2811 env: &mut TranslationEnvironment,
2812) -> crate::Result<()> {
2813 let memory_index = MemoryIndex::from_u32(memarg.memory);
2814 let val = state.pop1();
2815 let index = state.pop1();
2816 let val_ty = builder.func.dfg.value_type(val);
2817 let mem_op_size = mem_op_size(opcode, val_ty);
2818
2819 let mem = state.get_memory(builder.func, memory_index, env);
2820 let (flags, _wasm_index, base) = unwrap_or_return_unreachable_state!(
2821 state,
2822 mem.prepare_addr(builder, index, mem_op_size, memarg, env)
2823 );
2824
2825 builder
2826 .ins()
2827 .Store(opcode, val_ty, flags, Offset32::new(0), val, base);
2828 Ok(())
2829}
2830
2831fn translate_atomic_rmw(
2832 _widened_ty: Type,
2833 _access_ty: Type,
2834 _op: AtomicRmwOp,
2835 _memarg: &MemArg,
2836 _builder: &mut FunctionBuilder,
2837 _state: &mut FuncTranslationState,
2838 _env: &mut TranslationEnvironment,
2839) -> crate::Result<()> {
2840 todo!()
2841}
2842
2843fn translate_atomic_cas(
2844 _widened_ty: Type,
2845 _access_ty: Type,
2846 _memarg: &MemArg,
2847 _builder: &mut FunctionBuilder,
2848 _state: &mut FuncTranslationState,
2849 _env: &mut TranslationEnvironment,
2850) -> crate::Result<()> {
2851 todo!()
2852}
2853
2854fn translate_atomic_load(
2855 _widened_ty: Type,
2856 _access_ty: Type,
2857 _memarg: &MemArg,
2858 _builder: &mut FunctionBuilder,
2859 _state: &mut FuncTranslationState,
2860 _env: &mut TranslationEnvironment,
2861) -> crate::Result<()> {
2862 todo!()
2863}
2864
2865fn translate_atomic_store(
2866 _access_ty: Type,
2867 _memarg: &MemArg,
2868 _builder: &mut FunctionBuilder,
2869 _state: &mut FuncTranslationState,
2870 _env: &mut TranslationEnvironment,
2871) -> crate::Result<()> {
2872 todo!()
2873}
2874
2875fn mem_op_size(opcode: ir::Opcode, ty: Type) -> u8 {
2876 match opcode {
2877 ir::Opcode::Istore8 | ir::Opcode::Sload8 | ir::Opcode::Uload8 => 1,
2878 ir::Opcode::Istore16 | ir::Opcode::Sload16 | ir::Opcode::Uload16 => 2,
2879 ir::Opcode::Istore32 | ir::Opcode::Sload32 | ir::Opcode::Uload32 => 4,
2880 ir::Opcode::Store | ir::Opcode::Load => u8::try_from(ty.bytes()).unwrap(),
2881 _ => panic!("unknown size of mem op for {opcode:?}"),
2882 }
2883}
2884
2885fn translate_icmp(cc: IntCC, builder: &mut FunctionBuilder, state: &mut FuncTranslationState) {
2886 let (arg0, arg1) = state.pop2();
2887 let val = builder.ins().icmp(cc, arg0, arg1);
2888 state.push1(builder.ins().uextend(I32, val));
2889}
2890
2891fn translate_vector_icmp(
2892 cc: IntCC,
2893 needed_type: Type,
2894 builder: &mut FunctionBuilder,
2895 state: &mut FuncTranslationState,
2896) {
2897 let (a, b) = state.pop2();
2898 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
2899 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
2900 state.push1(builder.ins().icmp(cc, bitcast_a, bitcast_b));
2901}
2902
2903fn translate_fcmp(cc: FloatCC, builder: &mut FunctionBuilder, state: &mut FuncTranslationState) {
2904 let (arg0, arg1) = state.pop2();
2905 let val = builder.ins().fcmp(cc, arg0, arg1);
2906 state.push1(builder.ins().uextend(I32, val));
2907}
2908
2909fn translate_vector_fcmp(
2910 cc: FloatCC,
2911 needed_type: Type,
2912 builder: &mut FunctionBuilder,
2913 state: &mut FuncTranslationState,
2914) {
2915 let (a, b) = state.pop2();
2916 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
2917 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
2918 state.push1(builder.ins().fcmp(cc, bitcast_a, bitcast_b));
2919}
2920
2921fn translate_br_if(
2922 relative_depth: u32,
2923 builder: &mut FunctionBuilder,
2924 state: &mut FuncTranslationState,
2925) {
2926 let val = state.pop1();
2927 let (br_destination, inputs) = translate_br_if_args(relative_depth, state);
2928 let next_block = builder.create_block();
2929 canonicalise_brif(builder, val, br_destination, inputs, next_block, &[]);
2930
2931 builder.seal_block(next_block); // The only predecessor is the current block.
2932 builder.switch_to_block(next_block);
2933}
2934
2935fn translate_br_if_args(
2936 relative_depth: u32,
2937 state: &mut FuncTranslationState,
2938) -> (ir::Block, &mut [Value]) {
2939 // FIXME fix this ugly mess
2940 let i = state
2941 .control_stack
2942 .len()
2943 .checked_sub(1)
2944 .unwrap()
2945 .checked_sub(usize::try_from(relative_depth).unwrap())
2946 .unwrap();
2947
2948 let (return_count, br_destination) = {
2949 let frame = &mut state.control_stack[i];
2950 // The values returned by the branch are still available for the reachable
2951 // code that comes after it
2952 frame.set_branched_to_exit();
2953 let return_count = if frame.is_loop() {
2954 frame.num_param_values()
2955 } else {
2956 frame.num_return_values()
2957 };
2958 (return_count, frame.br_destination())
2959 };
2960 let inputs = state.peekn_mut(return_count);
2961 (br_destination, inputs)
2962}
2963
2964/// Determine the returned value type of a WebAssembly operator
2965fn type_of(operator: &Operator) -> Type {
2966 match operator {
2967 Operator::V128Load { .. }
2968 | Operator::V128Store { .. }
2969 | Operator::V128Const { .. }
2970 | Operator::V128Not
2971 | Operator::V128And
2972 | Operator::V128AndNot
2973 | Operator::V128Or
2974 | Operator::V128Xor
2975 | Operator::V128AnyTrue
2976 | Operator::V128Bitselect => I8X16, // default type representing V128
2977
2978 Operator::I8x16Shuffle { .. }
2979 | Operator::I8x16Splat
2980 | Operator::V128Load8Splat { .. }
2981 | Operator::V128Load8Lane { .. }
2982 | Operator::V128Store8Lane { .. }
2983 | Operator::I8x16ExtractLaneS { .. }
2984 | Operator::I8x16ExtractLaneU { .. }
2985 | Operator::I8x16ReplaceLane { .. }
2986 | Operator::I8x16Eq
2987 | Operator::I8x16Ne
2988 | Operator::I8x16LtS
2989 | Operator::I8x16LtU
2990 | Operator::I8x16GtS
2991 | Operator::I8x16GtU
2992 | Operator::I8x16LeS
2993 | Operator::I8x16LeU
2994 | Operator::I8x16GeS
2995 | Operator::I8x16GeU
2996 | Operator::I8x16Neg
2997 | Operator::I8x16Abs
2998 | Operator::I8x16AllTrue
2999 | Operator::I8x16Shl
3000 | Operator::I8x16ShrS
3001 | Operator::I8x16ShrU
3002 | Operator::I8x16Add
3003 | Operator::I8x16AddSatS
3004 | Operator::I8x16AddSatU
3005 | Operator::I8x16Sub
3006 | Operator::I8x16SubSatS
3007 | Operator::I8x16SubSatU
3008 | Operator::I8x16MinS
3009 | Operator::I8x16MinU
3010 | Operator::I8x16MaxS
3011 | Operator::I8x16MaxU
3012 | Operator::I8x16AvgrU
3013 | Operator::I8x16Bitmask
3014 | Operator::I8x16Popcnt
3015 | Operator::I8x16RelaxedLaneselect => I8X16,
3016
3017 Operator::I16x8Splat
3018 | Operator::V128Load16Splat { .. }
3019 | Operator::V128Load16Lane { .. }
3020 | Operator::V128Store16Lane { .. }
3021 | Operator::I16x8ExtractLaneS { .. }
3022 | Operator::I16x8ExtractLaneU { .. }
3023 | Operator::I16x8ReplaceLane { .. }
3024 | Operator::I16x8Eq
3025 | Operator::I16x8Ne
3026 | Operator::I16x8LtS
3027 | Operator::I16x8LtU
3028 | Operator::I16x8GtS
3029 | Operator::I16x8GtU
3030 | Operator::I16x8LeS
3031 | Operator::I16x8LeU
3032 | Operator::I16x8GeS
3033 | Operator::I16x8GeU
3034 | Operator::I16x8Neg
3035 | Operator::I16x8Abs
3036 | Operator::I16x8AllTrue
3037 | Operator::I16x8Shl
3038 | Operator::I16x8ShrS
3039 | Operator::I16x8ShrU
3040 | Operator::I16x8Add
3041 | Operator::I16x8AddSatS
3042 | Operator::I16x8AddSatU
3043 | Operator::I16x8Sub
3044 | Operator::I16x8SubSatS
3045 | Operator::I16x8SubSatU
3046 | Operator::I16x8MinS
3047 | Operator::I16x8MinU
3048 | Operator::I16x8MaxS
3049 | Operator::I16x8MaxU
3050 | Operator::I16x8AvgrU
3051 | Operator::I16x8Mul
3052 | Operator::I16x8Bitmask
3053 | Operator::I16x8RelaxedLaneselect => I16X8,
3054
3055 Operator::I32x4Splat
3056 | Operator::V128Load32Splat { .. }
3057 | Operator::V128Load32Lane { .. }
3058 | Operator::V128Store32Lane { .. }
3059 | Operator::I32x4ExtractLane { .. }
3060 | Operator::I32x4ReplaceLane { .. }
3061 | Operator::I32x4Eq
3062 | Operator::I32x4Ne
3063 | Operator::I32x4LtS
3064 | Operator::I32x4LtU
3065 | Operator::I32x4GtS
3066 | Operator::I32x4GtU
3067 | Operator::I32x4LeS
3068 | Operator::I32x4LeU
3069 | Operator::I32x4GeS
3070 | Operator::I32x4GeU
3071 | Operator::I32x4Neg
3072 | Operator::I32x4Abs
3073 | Operator::I32x4AllTrue
3074 | Operator::I32x4Shl
3075 | Operator::I32x4ShrS
3076 | Operator::I32x4ShrU
3077 | Operator::I32x4Add
3078 | Operator::I32x4Sub
3079 | Operator::I32x4Mul
3080 | Operator::I32x4MinS
3081 | Operator::I32x4MinU
3082 | Operator::I32x4MaxS
3083 | Operator::I32x4MaxU
3084 | Operator::I32x4Bitmask
3085 | Operator::I32x4TruncSatF32x4S
3086 | Operator::I32x4TruncSatF32x4U
3087 | Operator::I32x4RelaxedLaneselect
3088 | Operator::V128Load32Zero { .. } => I32X4,
3089
3090 Operator::I64x2Splat
3091 | Operator::V128Load64Splat { .. }
3092 | Operator::V128Load64Lane { .. }
3093 | Operator::V128Store64Lane { .. }
3094 | Operator::I64x2ExtractLane { .. }
3095 | Operator::I64x2ReplaceLane { .. }
3096 | Operator::I64x2Eq
3097 | Operator::I64x2Ne
3098 | Operator::I64x2LtS
3099 | Operator::I64x2GtS
3100 | Operator::I64x2LeS
3101 | Operator::I64x2GeS
3102 | Operator::I64x2Neg
3103 | Operator::I64x2Abs
3104 | Operator::I64x2AllTrue
3105 | Operator::I64x2Shl
3106 | Operator::I64x2ShrS
3107 | Operator::I64x2ShrU
3108 | Operator::I64x2Add
3109 | Operator::I64x2Sub
3110 | Operator::I64x2Mul
3111 | Operator::I64x2Bitmask
3112 | Operator::I64x2RelaxedLaneselect
3113 | Operator::V128Load64Zero { .. } => I64X2,
3114
3115 Operator::F32x4Splat
3116 | Operator::F32x4ExtractLane { .. }
3117 | Operator::F32x4ReplaceLane { .. }
3118 | Operator::F32x4Eq
3119 | Operator::F32x4Ne
3120 | Operator::F32x4Lt
3121 | Operator::F32x4Gt
3122 | Operator::F32x4Le
3123 | Operator::F32x4Ge
3124 | Operator::F32x4Abs
3125 | Operator::F32x4Neg
3126 | Operator::F32x4Sqrt
3127 | Operator::F32x4Add
3128 | Operator::F32x4Sub
3129 | Operator::F32x4Mul
3130 | Operator::F32x4Div
3131 | Operator::F32x4Min
3132 | Operator::F32x4Max
3133 | Operator::F32x4PMin
3134 | Operator::F32x4PMax
3135 | Operator::F32x4ConvertI32x4S
3136 | Operator::F32x4ConvertI32x4U
3137 | Operator::F32x4Ceil
3138 | Operator::F32x4Floor
3139 | Operator::F32x4Trunc
3140 | Operator::F32x4Nearest
3141 | Operator::F32x4RelaxedMax
3142 | Operator::F32x4RelaxedMin
3143 | Operator::F32x4RelaxedMadd
3144 | Operator::F32x4RelaxedNmadd => F32X4,
3145
3146 Operator::F64x2Splat
3147 | Operator::F64x2ExtractLane { .. }
3148 | Operator::F64x2ReplaceLane { .. }
3149 | Operator::F64x2Eq
3150 | Operator::F64x2Ne
3151 | Operator::F64x2Lt
3152 | Operator::F64x2Gt
3153 | Operator::F64x2Le
3154 | Operator::F64x2Ge
3155 | Operator::F64x2Abs
3156 | Operator::F64x2Neg
3157 | Operator::F64x2Sqrt
3158 | Operator::F64x2Add
3159 | Operator::F64x2Sub
3160 | Operator::F64x2Mul
3161 | Operator::F64x2Div
3162 | Operator::F64x2Min
3163 | Operator::F64x2Max
3164 | Operator::F64x2PMin
3165 | Operator::F64x2PMax
3166 | Operator::F64x2Ceil
3167 | Operator::F64x2Floor
3168 | Operator::F64x2Trunc
3169 | Operator::F64x2Nearest
3170 | Operator::F64x2RelaxedMax
3171 | Operator::F64x2RelaxedMin
3172 | Operator::F64x2RelaxedMadd
3173 | Operator::F64x2RelaxedNmadd => F64X2,
3174
3175 _ => unimplemented!(
3176 "Currently only SIMD instructions are mapped to their return type; the \
3177 following instruction is not mapped: {:?}",
3178 operator
3179 ),
3180 }
3181}
3182
3183/// Some SIMD operations only operate on I8X16 in CLIF; this will convert them to that type by
3184/// adding a bitcast if necessary.
3185fn optionally_bitcast_vector(
3186 value: Value,
3187 needed_type: Type,
3188 builder: &mut FunctionBuilder,
3189) -> Value {
3190 if builder.func.dfg.value_type(value) != needed_type {
3191 let mut flags = MemFlags::new();
3192 flags.set_endianness(ir::Endianness::Little);
3193 builder.ins().bitcast(needed_type, flags, value)
3194 } else {
3195 value
3196 }
3197}
3198
3199#[inline(always)]
3200fn is_non_canonical_v128(ty: Type) -> bool {
3201 matches!(ty, I64X2 | I32X4 | I16X8 | F32X4 | F64X2)
3202}
3203
3204/// Cast to I8X16, any vector values in `values` that are of "non-canonical" type (meaning, not
3205/// I8X16), and return them in a slice. A pre-scan is made to determine whether any casts are
3206/// actually necessary, and if not, the original slice is returned. Otherwise the cast values
3207/// are returned in a slice that belongs to the caller-supplied `SmallVec`.
3208fn canonicalise_v128_values<'a>(
3209 tmp_canonicalised: &'a mut SmallVec<[Value; 16]>,
3210 builder: &mut FunctionBuilder,
3211 values: &'a [Value],
3212) -> &'a [Value] {
3213 debug_assert!(tmp_canonicalised.is_empty());
3214 // First figure out if any of the parameters need to be cast. Mostly they don't need to be.
3215 let any_non_canonical = values
3216 .iter()
3217 .any(|v| is_non_canonical_v128(builder.func.dfg.value_type(*v)));
3218 // Hopefully we take this exit most of the time, hence doing no mem allocation.
3219 if !any_non_canonical {
3220 return values;
3221 }
3222 // Otherwise we'll have to cast, and push the resulting `Value`s into `canonicalised`.
3223 for v in values {
3224 tmp_canonicalised.push(if is_non_canonical_v128(builder.func.dfg.value_type(*v)) {
3225 let mut flags = MemFlags::new();
3226 flags.set_endianness(ir::Endianness::Little);
3227 builder.ins().bitcast(I8X16, flags, *v)
3228 } else {
3229 *v
3230 });
3231 }
3232 tmp_canonicalised.as_slice()
3233}
3234
3235/// Generate a `jump` instruction, but first cast all 128-bit vector values to I8X16 if they
3236/// don't have that type. This is done in somewhat roundabout way so as to ensure that we
3237/// almost never have to do any mem allocation.
3238fn canonicalise_then_jump(
3239 builder: &mut FunctionBuilder,
3240 destination: ir::Block,
3241 params: &[Value],
3242) -> ir::Inst {
3243 let mut tmp_canonicalised = SmallVec::<[Value; 16]>::new();
3244 let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params);
3245 builder.ins().jump(destination, canonicalised)
3246}
3247
3248/// The same but for a `brif` instruction.
3249fn canonicalise_brif(
3250 builder: &mut FunctionBuilder,
3251 cond: Value,
3252 block_then: ir::Block,
3253 params_then: &[Value],
3254 block_else: ir::Block,
3255 params_else: &[Value],
3256) -> ir::Inst {
3257 let mut tmp_canonicalised_then = SmallVec::<[Value; 16]>::new();
3258 let canonicalised_then =
3259 canonicalise_v128_values(&mut tmp_canonicalised_then, builder, params_then);
3260 let mut tmp_canonicalised_else = SmallVec::<[Value; 16]>::new();
3261 let canonicalised_else =
3262 canonicalise_v128_values(&mut tmp_canonicalised_else, builder, params_else);
3263 builder.ins().brif(
3264 cond,
3265 block_then,
3266 canonicalised_then,
3267 block_else,
3268 canonicalised_else,
3269 )
3270}
3271
3272/// A helper for popping and bitcasting a single value; since SIMD values can lose their type by
3273/// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF
3274/// typing issues.
3275fn pop1_with_bitcast(
3276 state: &mut FuncTranslationState,
3277 needed_type: Type,
3278 builder: &mut FunctionBuilder,
3279) -> Value {
3280 optionally_bitcast_vector(state.pop1(), needed_type, builder)
3281}
3282
3283/// A helper for popping and bitcasting two values; since SIMD values can lose their type by
3284/// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF
3285/// typing issues.
3286fn pop2_with_bitcast(
3287 state: &mut FuncTranslationState,
3288 needed_type: Type,
3289 builder: &mut FunctionBuilder,
3290) -> (Value, Value) {
3291 let (a, b) = state.pop2();
3292 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
3293 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
3294 (bitcast_a, bitcast_b)
3295}
3296
3297fn pop3_with_bitcast(
3298 state: &mut FuncTranslationState,
3299 needed_type: Type,
3300 builder: &mut FunctionBuilder,
3301) -> (Value, Value, Value) {
3302 let (a, b, c) = state.pop3();
3303 let bitcast_a = optionally_bitcast_vector(a, needed_type, builder);
3304 let bitcast_b = optionally_bitcast_vector(b, needed_type, builder);
3305 let bitcast_c = optionally_bitcast_vector(c, needed_type, builder);
3306 (bitcast_a, bitcast_b, bitcast_c)
3307}
3308
3309fn bitcast_arguments<'a>(
3310 builder: &FunctionBuilder,
3311 arguments: &'a mut [Value],
3312 params: &[ir::AbiParam],
3313 param_predicate: impl Fn(usize) -> bool,
3314) -> Vec<(Type, &'a mut Value)> {
3315 let filtered_param_types = params
3316 .iter()
3317 .enumerate()
3318 .filter(|(i, _)| param_predicate(*i))
3319 .map(|(_, param)| param.value_type);
3320
3321 let pairs = filtered_param_types.zip_eq(arguments);
3322
3323 // The arguments which need to be bitcasted are those which have some vector type but the type
3324 // expected by the parameter is not the same vector type as that of the provided argument.
3325 pairs
3326 .filter(|(param_type, _)| param_type.is_vector())
3327 .filter(|(param_type, arg)| {
3328 let arg_type = builder.func.dfg.value_type(**arg);
3329 assert!(
3330 arg_type.is_vector(),
3331 "unexpected type mismatch: expected {}, argument {} was actually of type {}",
3332 param_type,
3333 *arg,
3334 arg_type
3335 );
3336
3337 // This is the same check that would be done by `optionally_bitcast_vector`, except we
3338 // can't take a mutable borrow of the FunctionBuilder here, so we defer inserting the
3339 // bitcast instruction to the caller.
3340 arg_type != *param_type
3341 })
3342 .collect()
3343}
3344
3345/// A helper for bitcasting a sequence of return values for the function currently being built. If
3346/// a value is a vector type that does not match its expected type, this will modify the value in
3347/// place to point to the result of a `bitcast`. This conversion is necessary to parse Wasm
3348/// code that uses `V128` as function parameters (or implicitly in block parameters) and still use
3349/// specific CLIF types (e.g. `I32X4`) in the function body.
3350pub fn bitcast_wasm_returns(
3351 arguments: &mut [Value],
3352 builder: &mut FunctionBuilder,
3353 env: &mut TranslationEnvironment,
3354) {
3355 let changes = bitcast_arguments(builder, arguments, &builder.func.signature.returns, |i| {
3356 env.is_wasm_return(&builder.func.signature, i)
3357 });
3358 for (t, arg) in changes {
3359 let mut flags = MemFlags::new();
3360 flags.set_endianness(ir::Endianness::Little);
3361 *arg = builder.ins().bitcast(t, flags, *arg);
3362 }
3363}
3364
3365/// Like `bitcast_wasm_returns`, but for the parameters being passed to a specified callee.
3366fn bitcast_wasm_params(
3367 callee_signature: ir::SigRef,
3368 arguments: &mut [Value],
3369 builder: &mut FunctionBuilder,
3370 env: &mut TranslationEnvironment,
3371) {
3372 let callee_signature = &builder.func.dfg.signatures[callee_signature];
3373 let changes = bitcast_arguments(builder, arguments, &callee_signature.params, |i| {
3374 env.is_wasm_parameter(i)
3375 });
3376 for (t, arg) in changes {
3377 let mut flags = MemFlags::new();
3378 flags.set_endianness(ir::Endianness::Little);
3379 *arg = builder.ins().bitcast(t, flags, *arg);
3380 }
3381}