start implementing `in` queries, but found bugs

TODO.md

···
+# TODO
+
+* Array/Set/Record iteration (`for value in iterable`)
+* Runtime new atom (beware Ruby symbol problem)
+* `defer` statement
+* `test` definitions
+* Rethink `const` (`slot` and `slot mut`?)
+* Upgrade `for` to be a folding construct
+* Fix `break`/`continue` in various areas
+* Finish FFI
+* Build real standard library
+* Fix JIT mode, for "normal" use

testsuite/query-in/main.tri

···
+import "trilogy:debug" use dbg
+
+proc main!() {
+  for x in [1, 2, 3] {
+    dbg!(x)
+  }
+
+  for x in [|4, 4, 4, 5, 5|] {
+    dbg!(x)
+  }
+
+  for x in {| 'a => 1, 'b => 2 |} {
+    dbg!(x)
+  }
+}

testsuite/query-in/spec.toml

···
+output = "1\n2\n3\n4\n5\na:1\nb:2\n"

testsuite/rule-recursive/main.tri

···
+import "trilogy:debug" use dbg
+
+rule array_iterate(element, [element, .._])
+rule array_iterate(element, [_, ..rest]) <- array_iterate(element, rest)
+rule array_iterate(_, []) <- end
+
+proc main!() {
+  for array_iterate(x, [1, 2, 3]) {
+    dbg!(x)
+  }
+}

testsuite/rule-recursive/spec.toml

···
+output = "1\n2\n3\n"

trilogy-llvm/core/core.c

···
 #include "trilogy_character.h"
 #include "trilogy_number.h"
 #include "trilogy_record.h"
+#include "trilogy_set.h"
 #include "trilogy_string.h"
 #include "trilogy_struct.h"
 #include "trilogy_tuple.h"
···
     trilogy_string_value* rhs_str = trilogy_string_assume(rhs);
     return trilogy_string_unglue_end(rv, lhs_str, rhs_str);
 }
+
+void set_to_array(trilogy_value* rv, trilogy_value* set_val) {
+    trilogy_set_value* set = trilogy_set_untag(set_val);
+    trilogy_set_to_array(rv, set);
+}
+
+void record_to_array(trilogy_value* rv, trilogy_value* record_val) {
+    trilogy_record_value* record = trilogy_record_untag(record_val);
+    trilogy_record_to_array(rv, record);
+}

trilogy-llvm/core/core.h

···
 
 void lookup_atom(trilogy_value* rv, trilogy_value* atom);
 void to_string(trilogy_value* rv, trilogy_value* val);
+
+void set_to_array(trilogy_value* rv, trilogy_value* val);
+void record_to_array(trilogy_value* rv, trilogy_value* val);

trilogy-llvm/core/trilogy_record.c

···
 #include "trilogy_record.h"
 #include "internal.h"
+#include "trilogy_array.h"
 #include "trilogy_tuple.h"
 #include "trilogy_value.h"
 #include "types.h"
···
     }
     return true;
 }
+
+trilogy_array_value*
+trilogy_record_to_array(trilogy_value* tv, trilogy_record_value* record) {
+    trilogy_array_value* arr = trilogy_array_init_cap(tv, record->len);
+    for (uint64_t i = 0; i < record->cap; ++i) {
+        trilogy_tuple_value* entry = &record->contents[i];
+        if (entry->fst.tag == TAG_UNDEFINED) continue;
+        trilogy_value val = trilogy_undefined;
+        trilogy_tuple_clone_into(&val, entry);
+        trilogy_array_push(arr, &val);
+    }
+    assert(arr->len == record->len);
+    return arr;
+}

trilogy-llvm/core/trilogy_record.h

···
 );
 
 void trilogy_record_destroy(trilogy_record_value* rec);
+
+trilogy_array_value*
+trilogy_record_to_array(trilogy_value* tv, trilogy_record_value* rec);

trilogy-llvm/core/trilogy_set.c

···
 #include "trilogy_set.h"
 #include "internal.h"
+#include "trilogy_array.h"
 #include "trilogy_value.h"
 #include "types.h"
 #include <assert.h>
···
     }
     return true;
 }
+
+trilogy_array_value*
+trilogy_set_to_array(trilogy_value* tv, trilogy_set_value* set) {
+    trilogy_array_value* arr = trilogy_array_init_cap(tv, set->len);
+    for (uint64_t i = 0; i < set->cap; ++i) {
+        trilogy_tuple_value* entry = &set->contents[i];
+        if (entry->fst.tag == TAG_UNDEFINED) continue;
+        trilogy_value val = trilogy_undefined;
+        trilogy_value_clone_into(&val, &entry->fst);
+        trilogy_array_push(arr, &val);
+    }
+    assert(arr->len == set->len);
+    return arr;
+}

trilogy-llvm/core/trilogy_set.h

···
 trilogy_set_value* trilogy_set_untag(trilogy_value* val);
 trilogy_set_value* trilogy_set_assume(trilogy_value* val);
 
-void trilogy_set_destroy(trilogy_set_value* arr);
+void trilogy_set_destroy(trilogy_set_value* set);
+
+trilogy_array_value*
+trilogy_set_to_array(trilogy_value* tv, trilogy_set_value* set);

trilogy-llvm/core/types.h

···
     trilogy_array_value* closure;
 } trilogy_module;
 
+/**
+ * The foreign object works/appears as a module in Trilogy, but internally is
+ * backed by some native stuff instead. The contents takes the place of the
+ * closure, and is received as the last parameter when calling methods.
+ *
+ * RAII is not a thing, the contents are possibly unclosed resources,
+ * the Trilogy code should correctly open/close them if needed.
+ */
+typedef struct trilogy_foreign_object {
+    /**
+     * The reference count for this foreign object instance.
+     */
+    uint32_t rc;
+    /**
+     * The number of members in this foreign object.
+     */
+    size_t len;
+    /**
+     * The sorted list of IDs for members in this foreign object.
+     */
+    uint64_t* member_ids;
+    /**
+     * The actual pointers to contained member accessors.
+     */
+    void** members;
+    /**
+     * Pointer to the contained foreign object.
+     */
+    void* contents;
+} trilogy_foreign_object;
+
 char* type_name(trilogy_value_tag tag);

trilogy-llvm/src/call.rs

···
             break_to,
             continue_to,
             next_to,
-            done_to,
+            done_to, // NOTE: done_to may be a variable that gets closed and becomes invalid above
         ]);
         args.extend_from_slice(arguments);
 

trilogy-llvm/src/core.rs

···
         let f = self.declare_core("bitwise_invert", 1);
         self.call_core(target, f, &[value.into()]);
     }
+
+    pub(crate) fn elem(&self) -> PointerValue<'ctx> {
+        self.get_core("elem")
+    }
 }

trilogy-llvm/src/expression/mod.rs

···
         let done_function = self.add_continuation("done");
         let (done_continuation, done_continuation_point) =
             self.capture_current_continuation_as_break(done_function, "for_break");
-        let next_iteration = self.compile_iterator(&expr.query, done_continuation)?;
+        let next_iteration = self.compile_query_iteration(&expr.query, done_continuation)?;
         self.bind_temporary(next_iteration);
         if let Some(value) = self.compile_expression(&expr.value, name) {
             let next_iteration = self.use_temporary(next_iteration).unwrap();
···
         let done_function = self.add_continuation("done");
         let (done_continuation, done_continuation_point) =
             self.capture_current_continuation_as_break(done_function, "for_break");
-        let next_iteration = self.compile_iterator(&expr.query, done_continuation)?;
+        let next_iteration = self.compile_query_iteration(&expr.query, done_continuation)?;
         self.bind_temporary(next_iteration);
         if let Some(value) = self.compile_expression(&expr.value, "element") {
             let arr_val = self.use_temporary(output).unwrap();
···
         let done_function = self.add_continuation("done");
         let (done_continuation, done_continuation_point) =
             self.capture_current_continuation_as_break(done_function, "for_break");
-        let next_iteration = self.compile_iterator(&expr.query, done_continuation)?;
+        let next_iteration = self.compile_query_iteration(&expr.query, done_continuation)?;
         self.bind_temporary(next_iteration);
         if let Some(value) = self.compile_expression(&expr.value, "element") {
             let set_val = self.use_temporary(output).unwrap();
···
         let done_function = self.add_continuation("done");
         let (done_continuation, done_continuation_point) =
             self.capture_current_continuation_as_break(done_function, "for_break");
-        let next_iteration = self.compile_iterator(&expr.query, done_continuation)?;
+        let next_iteration = self.compile_query_iteration(&expr.query, done_continuation)?;
         self.bind_temporary(next_iteration);
         let Value::Mapping(mapping) = &expr.value.value else {
             unreachable!()
···
                 // case. Appears to be some cleanup that is being missed.
                 let end = self.get_end("");
                 self.bind_temporary(end);
-                let next_to = self.compile_iterator(&decl.query, end)?;
+                let next_to = self.compile_query_iteration(&decl.query, end)?;
                 self.push_execution(next_to);
                 self.compile_expression(&decl.body, name)
-            },
+            }
         }
     }
 
···
                 let end_false_fn = self.add_continuation("is_false");
                 let (end_false_cont, end_false_cp) =
                     self.capture_current_continuation(end_false_fn, "is_false");
-                let next = self.compile_iterator(query, end_false_cont)?;
+                let next = self.compile_query_iteration(query, end_false_cont)?;
                 self.trilogy_value_destroy(next);
                 let result = self.allocate_const(self.bool_const(true), "");
                 self.call_known_continuation(self.use_temporary(merge_to_cont).unwrap(), result);

trilogy-llvm/src/query.rs

···
 use trilogy_ir::{Id, ir};
 
 impl<'ctx> Codegen<'ctx> {
-    pub(crate) fn compile_iterator(
+    pub(crate) fn compile_query_iteration(
         &self,
         query: &ir::Query,
         done_to: PointerValue<'ctx>,
···
                 // NOTE: at this time, the lookup expression is not an arbitrary expression, only a
                 // reference/module path, so branching is not possible here.
                 let rule = self.compile_expression(&lookup.path, "rule")?;
-                // NOTE: similarly, the arguments of a query must be syntactically both patterns and
-                // expressions, so we can again guarantee that branching is not possible.
-                let arguments = lookup
-                    .patterns
-                    .iter()
-                    .map(|pattern| {
-                        if !pattern.can_evaluate() {
-                            // This pattern is not possibly an expression, e.g. due to containing a wildcard
-                            // or other pattern-only syntax element. It can only be used as an output parameter.
-                            let arg = self.allocate_undefined("out_arg");
-                            self.bind_temporary(arg);
-                            return Some(arg);
-                        }
-                        let variables = pattern
-                            .bindings()
-                            .iter()
-                            .map(|var| self.get_variable(var).unwrap())
-                            .collect::<Vec<_>>();
-                        if variables.is_empty() {
-                            let arg = self.compile_expression(pattern, "in_arg")?;
-                            self.bind_temporary(arg);
-                            return Some(arg);
-                        }
-                        // This pattern is (potentially) fully defined. We must confirm by checking all
-                        // the variables at runtime, and if they are all set, then we can construct
-                        // this pattern.
-                        let out_block = self.context.append_basic_block(self.get_function(), "out");
-                        let original_function = self.get_function();
-                        let snapshot = self.snapshot_function_context();
-                        for variable in variables {
-                            let next_arg =
-                                self.context.append_basic_block(self.get_function(), "next");
-                            self.branch_undefined(variable.ptr(), out_block, next_arg);
-                            self.builder.position_at_end(next_arg);
-                        }
-                        let in_arg = self.compile_expression(pattern, "in_arg")?;
-                        let final_function = self.get_function();
-                        if original_function == final_function {
-                            let merge_block =
-                                self.context.append_basic_block(final_function, "merge_arg");
-
-                            let in_block = self.builder.get_insert_block().unwrap();
-                            self.builder
-                                .build_unconditional_branch(merge_block)
-                                .unwrap();
-
-                            self.builder.position_at_end(out_block);
-                            let out_arg = self.allocate_undefined("out_arg");
-                            self.builder
-                                .build_unconditional_branch(merge_block)
-                                .unwrap();
-
-                            self.builder.position_at_end(merge_block);
-                            let phi = self
-                                .builder
-                                .build_phi(self.context.ptr_type(AddressSpace::default()), "arg")
-                                .unwrap();
-                            phi.add_incoming(&[(&in_arg, in_block), (&out_arg, out_block)]);
-                            let arg = phi.as_basic_value().into_pointer_value();
-                            self.bind_temporary(arg);
-                            Some(arg)
-                        } else {
-                            let arg_continuation = self.add_continuation("arg");
-                            let end = self.continue_in_scope(arg_continuation, in_arg);
-                            self.end_continuation_point_as_close(end);
-
-                            self.restore_function_context(snapshot);
-                            self.builder.position_at_end(out_block);
-                            let end = self.void_continue_in_scope(arg_continuation);
-                            self.end_continuation_point_as_close(end);
-
-                            self.begin_next_function(arg_continuation);
-                            let arg = self.get_continuation("arg");
-                            self.bind_temporary(arg);
-                            Some(arg)
-                        }
-                    })
-                    .collect::<Option<Vec<_>>>()?;
-
-                let (next_iteration_inner, output_arguments) = self.call_rule(
-                    rule,
-                    &arguments,
-                    self.use_temporary(done_to).unwrap(),
-                    "lookup_next",
-                );
-                self.bind_temporary(next_iteration_inner);
-
-                // Wrap the next iteration with our own, as a lookup requires some cleanup
-                // before starting its next internal iteration.
-                let next_iteration_with_cleanup = self.add_continuation("lookup.next");
-                let (next_iteration_with_cleanup_continuation, next_iteration_with_cleanup_cp) =
-                    self.capture_current_continuation(next_iteration_with_cleanup, "lookup.next");
-
-                let bound_before_lookup = bound_ids.len();
-                for (pattern, out_value) in lookup.patterns.iter().zip(output_arguments) {
-                    let out_value = self.use_temporary(out_value).unwrap();
-                    self.compile_pattern_match_with_bindings(
-                        pattern,
-                        out_value,
-                        next_iteration_with_cleanup_continuation,
-                        bound_ids,
-                    )?;
-                }
-
-                self.call_known_continuation(
-                    self.use_temporary(next_to).unwrap(),
-                    next_iteration_with_cleanup_continuation,
-                );
-
-                self.become_continuation_point(next_iteration_with_cleanup_cp);
-                self.begin_next_function(next_iteration_with_cleanup);
-                self.cleanup_go_next(next_iteration_inner, bound_ids, bound_before_lookup);
+                self.perform_lookup(rule, &lookup.patterns, next_to, done_to, bound_ids)?;
             }
             ir::QueryValue::Disjunction(disj) => {
                 let disj_second_fn = self.add_continuation("disj.second");
···
                 )?;
                 self.next_cleanup(done_to, next_to, bound_ids, pre_len, "assign_next");
             }
-            ir::QueryValue::Element(..) => todo!(),
+            ir::QueryValue::Element(unification) => {
+                let elem = self.elem();
+                self.perform_lookup(
+                    elem,
+                    [&unification.pattern, &unification.expression],
+                    next_to,
+                    done_to,
+                    bound_ids,
+                )?;
+            }
             ir::QueryValue::Not(query) => {
                 let go_next_fn = self.add_continuation("not.next");
                 let (go_next, go_next_cp) =
···
         keep_ids: usize,
         name: &str,
     ) {
-        // Wrap the next iteration with our own, as a lookup requires some cleanup
+        // Wrap the next iteration with our own, as a lookup or unification requires some cleanup
         // before starting its next internal iteration.
         let next_iteration_with_cleanup = self.add_continuation(name);
         let (next_iteration_with_cleanup_continuation, next_iteration_with_cleanup_cp) =
···
         }
         let next_iteration = self.use_temporary(next_iteration).unwrap();
         self.void_call_continuation(next_iteration);
+    }
+
+    fn perform_lookup<'a>(
+        &self,
+        rule: PointerValue<'ctx>,
+        patterns: impl IntoIterator<Item = &'a ir::Expression> + Copy,
+        next_to: PointerValue<'ctx>,
+        done_to: PointerValue<'ctx>,
+        bound_ids: &mut Vec<Id>,
+    ) -> Option<()> {
+        // NOTE: similarly, the arguments of a query must be syntactically both patterns and
+        // expressions, so we can again guarantee that branching is not possible.
+        let arguments = patterns
+            .into_iter()
+            .map(|pattern| {
+                if !pattern.can_evaluate() {
+                    // This pattern is not possibly an expression, e.g. due to containing a wildcard
+                    // or other pattern-only syntax element. It can only be used as an output parameter.
+                    let arg = self.allocate_undefined("out_arg");
+                    self.bind_temporary(arg);
+                    return Some(arg);
+                }
+                let variables = pattern
+                    .bindings()
+                    .iter()
+                    .map(|var| self.get_variable(var).unwrap())
+                    .collect::<Vec<_>>();
+                if variables.is_empty() {
+                    let arg = self.compile_expression(pattern, "in_arg")?;
+                    self.bind_temporary(arg);
+                    return Some(arg);
+                }
+                // This pattern is (potentially) fully defined. We must confirm by checking all
+                // the variables at runtime, and if they are all set, then we can construct
+                // this pattern.
+                let out_block = self.context.append_basic_block(self.get_function(), "out");
+                let original_function = self.get_function();
+                let snapshot = self.snapshot_function_context();
+                for variable in variables {
+                    let next_arg = self.context.append_basic_block(self.get_function(), "next");
+                    self.branch_undefined(variable.ptr(), out_block, next_arg);
+                    self.builder.position_at_end(next_arg);
+                }
+                let in_arg = self.compile_expression(pattern, "in_arg")?;
+                let final_function = self.get_function();
+                if original_function == final_function {
+                    let merge_block = self.context.append_basic_block(final_function, "merge_arg");
+
+                    let in_block = self.builder.get_insert_block().unwrap();
+                    self.builder
+                        .build_unconditional_branch(merge_block)
+                        .unwrap();
+
+                    self.builder.position_at_end(out_block);
+                    let out_arg = self.allocate_undefined("out_arg");
+                    self.builder
+                        .build_unconditional_branch(merge_block)
+                        .unwrap();
+
+                    self.builder.position_at_end(merge_block);
+                    let phi = self
+                        .builder
+                        .build_phi(self.context.ptr_type(AddressSpace::default()), "arg")
+                        .unwrap();
+                    phi.add_incoming(&[(&in_arg, in_block), (&out_arg, out_block)]);
+                    let arg = phi.as_basic_value().into_pointer_value();
+                    self.bind_temporary(arg);
+                    Some(arg)
+                } else {
+                    let arg_continuation = self.add_continuation("arg");
+                    let end = self.continue_in_scope(arg_continuation, in_arg);
+                    self.end_continuation_point_as_close(end);
+
+                    self.restore_function_context(snapshot);
+                    self.builder.position_at_end(out_block);
+                    let end = self.void_continue_in_scope(arg_continuation);
+                    self.end_continuation_point_as_close(end);
+
+                    self.begin_next_function(arg_continuation);
+                    let arg = self.get_continuation("arg");
+                    self.bind_temporary(arg);
+                    Some(arg)
+                }
+            })
+            .collect::<Option<Vec<_>>>()?;
+
+        let (next_iteration_inner, output_arguments) = self.call_rule(
+            rule,
+            &arguments,
+            self.use_temporary(done_to).unwrap(),
+            "lookup_next",
+        );
+        self.bind_temporary(next_iteration_inner);
+
+        // Wrap the next iteration with our own, as a lookup requires some cleanup
+        // before starting its next internal iteration.
+        let next_iteration_with_cleanup = self.add_continuation("lookup.next");
+        let (next_iteration_with_cleanup_continuation, next_iteration_with_cleanup_cp) =
+            self.capture_current_continuation(next_iteration_with_cleanup, "lookup.next");
+
+        let bound_before_lookup = bound_ids.len();
+        for (pattern, out_value) in patterns.into_iter().zip(output_arguments) {
+            let out_value = self.use_temporary(out_value).unwrap();
+            self.compile_pattern_match_with_bindings(
+                pattern,
+                out_value,
+                next_iteration_with_cleanup_continuation,
+                bound_ids,
+            )?;
+        }
+
+        self.call_known_continuation(
+            self.use_temporary(next_to).unwrap(),
+            next_iteration_with_cleanup_continuation,
+        );
+
+        self.become_continuation_point(next_iteration_with_cleanup_cp);
+        self.begin_next_function(next_iteration_with_cleanup);
+        self.cleanup_go_next(next_iteration_inner, bound_ids, bound_before_lookup);
+        Some(())
     }
 }

trilogy-llvm/src/rule.rs

···
             let (go_to_next_overload, next_overload_cp) =
                 self.capture_current_continuation_as_break(next_overload_function, "next_overload");
 
+            let next_overload_clone = self.allocate_value("next_overload_clone");
+            self.trilogy_value_clone_into(next_overload_clone, go_to_next_overload);
+            self.bind_temporary(next_overload_clone);
+
             for (param, &value) in overload.parameters.iter().zip(&input_args) {
                 let original_insert_function = self.get_function();
                 let original_snapshot = self.snapshot_function_context();
···
                 }
             }
             // The rule runs the iterator, resulting in new bindings being set.
-            let Some(next_iteration) = self.compile_iterator(&overload.body, go_to_next_overload)
+            let Some(next_iteration) =
+                self.compile_query_iteration(&overload.body, go_to_next_overload)
             else {
                 break 'outer;
             };
···
                     // If the input and output are both going to be undefined, then go right to the
                     // next overload.
                     self.builder.position_at_end(fully_unbound);
-                    let next_overload = self.use_temporary(go_to_next_overload).unwrap();
+                    let next_overload = self.use_temporary(next_overload_clone).unwrap();
                     self.void_call_continuation(next_overload);
 
                     // There's actually no action for the rebind-input case, only to reuse the argument later

trilogy/src/stdlib/core.tri

···
   export shift_right_contract
 
   extern proc primitive_to_string!(val)
+  export primitive_to_string
+
+  extern proc set_to_array!(val)
+  export set_to_array
+
+  extern proc record_to_array!(val)
+  export record_to_array
 }
 
 const print = c::print
···
 export rem
 func cons lhs rhs = c::cons!(lhs, rhs)
 export cons
+
+func to_array (arr and typeof 'array) = arr
+func to_array (set and typeof 'set) = c::set_to_array!(set)
+func to_array (record and typeof 'record) = c::record_to_array!(record)
+export to_array
+
+rule elem(element, [element, .._])
+rule elem(element, [_, ..rest]) <- elem(element, rest)
+rule elem(_, []) <- end
+rule elem(element, set and typeof 'set) <-
+  array = c::set_to_array!(set)
+  and elem(element, array)
+rule elem(element, record and typeof 'record) <- array = c::record_to_array!(record)
+  and elem(element, array)
+export elem