jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2020-2022, Andreas Kling <kling@serenityos.org>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7#include <AK/Badge.h>
8#include <AK/Debug.h>
9#include <AK/HashTable.h>
10#include <AK/StackInfo.h>
11#include <AK/TemporaryChange.h>
12#include <LibCore/ElapsedTimer.h>
13#include <LibJS/Heap/CellAllocator.h>
14#include <LibJS/Heap/Handle.h>
15#include <LibJS/Heap/Heap.h>
16#include <LibJS/Heap/HeapBlock.h>
17#include <LibJS/Interpreter.h>
18#include <LibJS/Runtime/Object.h>
19#include <LibJS/Runtime/WeakContainer.h>
20#include <LibJS/SafeFunction.h>
21#include <setjmp.h>
22
23#ifdef AK_OS_SERENITY
24# include <serenity.h>
25#endif
26
27namespace JS {
28
29#ifdef AK_OS_SERENITY
30static int gc_perf_string_id;
31#endif
32
33// NOTE: We keep a per-thread list of custom ranges. This hinges on the assumption that there is one JS VM per thread.
34static __thread HashMap<FlatPtr*, size_t>* s_custom_ranges_for_conservative_scan = nullptr;
35
36Heap::Heap(VM& vm)
37 : m_vm(vm)
38{
39#ifdef AK_OS_SERENITY
40 auto gc_signpost_string = "Garbage collection"sv;
41 gc_perf_string_id = perf_register_string(gc_signpost_string.characters_without_null_termination(), gc_signpost_string.length());
42#endif
43
44 if constexpr (HeapBlock::min_possible_cell_size <= 16) {
45 m_allocators.append(make<CellAllocator>(16));
46 }
47 static_assert(HeapBlock::min_possible_cell_size <= 24, "Heap Cell tracking uses too much data!");
48 m_allocators.append(make<CellAllocator>(32));
49 m_allocators.append(make<CellAllocator>(64));
50 m_allocators.append(make<CellAllocator>(96));
51 m_allocators.append(make<CellAllocator>(128));
52 m_allocators.append(make<CellAllocator>(256));
53 m_allocators.append(make<CellAllocator>(512));
54 m_allocators.append(make<CellAllocator>(1024));
55 m_allocators.append(make<CellAllocator>(3072));
56}
57
58Heap::~Heap()
59{
60 vm().string_cache().clear();
61 vm().deprecated_string_cache().clear();
62 collect_garbage(CollectionType::CollectEverything);
63}
64
65ALWAYS_INLINE CellAllocator& Heap::allocator_for_size(size_t cell_size)
66{
67 for (auto& allocator : m_allocators) {
68 if (allocator->cell_size() >= cell_size)
69 return *allocator;
70 }
71 dbgln("Cannot get CellAllocator for cell size {}, largest available is {}!", cell_size, m_allocators.last()->cell_size());
72 VERIFY_NOT_REACHED();
73}
74
75Cell* Heap::allocate_cell(size_t size)
76{
77 if (should_collect_on_every_allocation()) {
78 collect_garbage();
79 } else if (m_allocations_since_last_gc > m_max_allocations_between_gc) {
80 m_allocations_since_last_gc = 0;
81 collect_garbage();
82 } else {
83 ++m_allocations_since_last_gc;
84 }
85
86 auto& allocator = allocator_for_size(size);
87 return allocator.allocate_cell(*this);
88}
89
90void Heap::collect_garbage(CollectionType collection_type, bool print_report)
91{
92 VERIFY(!m_collecting_garbage);
93 TemporaryChange change(m_collecting_garbage, true);
94
95#ifdef AK_OS_SERENITY
96 static size_t global_gc_counter = 0;
97 perf_event(PERF_EVENT_SIGNPOST, gc_perf_string_id, global_gc_counter++);
98#endif
99
100 Core::ElapsedTimer collection_measurement_timer;
101 if (print_report)
102 collection_measurement_timer.start();
103
104 if (collection_type == CollectionType::CollectGarbage) {
105 if (m_gc_deferrals) {
106 m_should_gc_when_deferral_ends = true;
107 return;
108 }
109 HashTable<Cell*> roots;
110 gather_roots(roots);
111 mark_live_cells(roots);
112 }
113 finalize_unmarked_cells();
114 sweep_dead_cells(print_report, collection_measurement_timer);
115}
116
117void Heap::gather_roots(HashTable<Cell*>& roots)
118{
119 vm().gather_roots(roots);
120 gather_conservative_roots(roots);
121
122 for (auto& handle : m_handles)
123 roots.set(handle.cell());
124
125 for (auto& vector : m_marked_vectors)
126 vector.gather_roots(roots);
127
128 if constexpr (HEAP_DEBUG) {
129 dbgln("gather_roots:");
130 for (auto* root : roots)
131 dbgln(" + {}", root);
132 }
133}
134
135__attribute__((no_sanitize("address"))) void Heap::gather_conservative_roots(HashTable<Cell*>& roots)
136{
137 FlatPtr dummy;
138
139 dbgln_if(HEAP_DEBUG, "gather_conservative_roots:");
140
141 jmp_buf buf;
142 setjmp(buf);
143
144 HashTable<FlatPtr> possible_pointers;
145
146 auto* raw_jmp_buf = reinterpret_cast<FlatPtr const*>(buf);
147
148 auto add_possible_value = [&](FlatPtr data) {
149 if constexpr (sizeof(FlatPtr*) == sizeof(Value)) {
150 // Because Value stores pointers in non-canonical form we have to check if the top bytes
151 // match any pointer-backed tag, in that case we have to extract the pointer to its
152 // canonical form and add that as a possible pointer.
153 if ((data & SHIFTED_IS_CELL_PATTERN) == SHIFTED_IS_CELL_PATTERN)
154 possible_pointers.set(Value::extract_pointer_bits(data));
155 else
156 possible_pointers.set(data);
157 } else {
158 static_assert((sizeof(Value) % sizeof(FlatPtr*)) == 0);
159 // In the 32-bit case we will look at the top and bottom part of Value separately we just
160 // add both the upper and lower bytes as possible pointers.
161 possible_pointers.set(data);
162 }
163 };
164
165 for (size_t i = 0; i < ((size_t)sizeof(buf)) / sizeof(FlatPtr); i += sizeof(FlatPtr))
166 add_possible_value(raw_jmp_buf[i]);
167
168 auto stack_reference = bit_cast<FlatPtr>(&dummy);
169 auto& stack_info = m_vm.stack_info();
170
171 for (FlatPtr stack_address = stack_reference; stack_address < stack_info.top(); stack_address += sizeof(FlatPtr)) {
172 auto data = *reinterpret_cast<FlatPtr*>(stack_address);
173 add_possible_value(data);
174 }
175
176 // NOTE: If we have any custom ranges registered, scan those as well.
177 // This is where JS::SafeFunction closures get marked.
178 if (s_custom_ranges_for_conservative_scan) {
179 for (auto& custom_range : *s_custom_ranges_for_conservative_scan) {
180 for (size_t i = 0; i < (custom_range.value / sizeof(FlatPtr)); ++i) {
181 add_possible_value(custom_range.key[i]);
182 }
183 }
184 }
185
186 HashTable<HeapBlock*> all_live_heap_blocks;
187 for_each_block([&](auto& block) {
188 all_live_heap_blocks.set(&block);
189 return IterationDecision::Continue;
190 });
191
192 for (auto possible_pointer : possible_pointers) {
193 if (!possible_pointer)
194 continue;
195 dbgln_if(HEAP_DEBUG, " ? {}", (void const*)possible_pointer);
196 auto* possible_heap_block = HeapBlock::from_cell(reinterpret_cast<Cell const*>(possible_pointer));
197 if (all_live_heap_blocks.contains(possible_heap_block)) {
198 if (auto* cell = possible_heap_block->cell_from_possible_pointer(possible_pointer)) {
199 if (cell->state() == Cell::State::Live) {
200 dbgln_if(HEAP_DEBUG, " ?-> {}", (void const*)cell);
201 roots.set(cell);
202 } else {
203 dbgln_if(HEAP_DEBUG, " #-> {}", (void const*)cell);
204 }
205 }
206 }
207 }
208}
209
210class MarkingVisitor final : public Cell::Visitor {
211public:
212 explicit MarkingVisitor(HashTable<Cell*> const& roots)
213 {
214 for (auto* root : roots) {
215 visit(root);
216 }
217 }
218
219 virtual void visit_impl(Cell& cell) override
220 {
221 if (cell.is_marked())
222 return;
223 dbgln_if(HEAP_DEBUG, " ! {}", &cell);
224
225 cell.set_marked(true);
226 m_work_queue.append(cell);
227 }
228
229 void mark_all_live_cells()
230 {
231 while (!m_work_queue.is_empty()) {
232 m_work_queue.take_last().visit_edges(*this);
233 }
234 }
235
236private:
237 Vector<Cell&> m_work_queue;
238};
239
240void Heap::mark_live_cells(HashTable<Cell*> const& roots)
241{
242 dbgln_if(HEAP_DEBUG, "mark_live_cells:");
243
244 MarkingVisitor visitor(roots);
245 visitor.mark_all_live_cells();
246
247 for (auto& inverse_root : m_uprooted_cells)
248 inverse_root->set_marked(false);
249
250 m_uprooted_cells.clear();
251}
252
253bool Heap::cell_must_survive_garbage_collection(Cell const& cell)
254{
255 if (!cell.overrides_must_survive_garbage_collection({}))
256 return false;
257 return cell.must_survive_garbage_collection();
258}
259
260void Heap::finalize_unmarked_cells()
261{
262 for_each_block([&](auto& block) {
263 block.template for_each_cell_in_state<Cell::State::Live>([](Cell* cell) {
264 if (!cell->is_marked() && !cell_must_survive_garbage_collection(*cell))
265 cell->finalize();
266 });
267 return IterationDecision::Continue;
268 });
269}
270
271void Heap::sweep_dead_cells(bool print_report, Core::ElapsedTimer const& measurement_timer)
272{
273 dbgln_if(HEAP_DEBUG, "sweep_dead_cells:");
274 Vector<HeapBlock*, 32> empty_blocks;
275 Vector<HeapBlock*, 32> full_blocks_that_became_usable;
276
277 size_t collected_cells = 0;
278 size_t live_cells = 0;
279 size_t collected_cell_bytes = 0;
280 size_t live_cell_bytes = 0;
281
282 for_each_block([&](auto& block) {
283 bool block_has_live_cells = false;
284 bool block_was_full = block.is_full();
285 block.template for_each_cell_in_state<Cell::State::Live>([&](Cell* cell) {
286 if (!cell->is_marked() && !cell_must_survive_garbage_collection(*cell)) {
287 dbgln_if(HEAP_DEBUG, " ~ {}", cell);
288 block.deallocate(cell);
289 ++collected_cells;
290 collected_cell_bytes += block.cell_size();
291 } else {
292 cell->set_marked(false);
293 block_has_live_cells = true;
294 ++live_cells;
295 live_cell_bytes += block.cell_size();
296 }
297 });
298 if (!block_has_live_cells)
299 empty_blocks.append(&block);
300 else if (block_was_full != block.is_full())
301 full_blocks_that_became_usable.append(&block);
302 return IterationDecision::Continue;
303 });
304
305 for (auto& weak_container : m_weak_containers)
306 weak_container.remove_dead_cells({});
307
308 for (auto* block : empty_blocks) {
309 dbgln_if(HEAP_DEBUG, " - HeapBlock empty @ {}: cell_size={}", block, block->cell_size());
310 allocator_for_size(block->cell_size()).block_did_become_empty({}, *block);
311 }
312
313 for (auto* block : full_blocks_that_became_usable) {
314 dbgln_if(HEAP_DEBUG, " - HeapBlock usable again @ {}: cell_size={}", block, block->cell_size());
315 allocator_for_size(block->cell_size()).block_did_become_usable({}, *block);
316 }
317
318 if constexpr (HEAP_DEBUG) {
319 for_each_block([&](auto& block) {
320 dbgln(" > Live HeapBlock @ {}: cell_size={}", &block, block.cell_size());
321 return IterationDecision::Continue;
322 });
323 }
324
325 if (print_report) {
326 Time const time_spent = measurement_timer.elapsed_time();
327 size_t live_block_count = 0;
328 for_each_block([&](auto&) {
329 ++live_block_count;
330 return IterationDecision::Continue;
331 });
332
333 dbgln("Garbage collection report");
334 dbgln("=============================================");
335 dbgln(" Time spent: {} ms", time_spent.to_milliseconds());
336 dbgln(" Live cells: {} ({} bytes)", live_cells, live_cell_bytes);
337 dbgln("Collected cells: {} ({} bytes)", collected_cells, collected_cell_bytes);
338 dbgln(" Live blocks: {} ({} bytes)", live_block_count, live_block_count * HeapBlock::block_size);
339 dbgln(" Freed blocks: {} ({} bytes)", empty_blocks.size(), empty_blocks.size() * HeapBlock::block_size);
340 dbgln("=============================================");
341 }
342}
343
344void Heap::did_create_handle(Badge<HandleImpl>, HandleImpl& impl)
345{
346 VERIFY(!m_handles.contains(impl));
347 m_handles.append(impl);
348}
349
350void Heap::did_destroy_handle(Badge<HandleImpl>, HandleImpl& impl)
351{
352 VERIFY(m_handles.contains(impl));
353 m_handles.remove(impl);
354}
355
356void Heap::did_create_marked_vector(Badge<MarkedVectorBase>, MarkedVectorBase& vector)
357{
358 VERIFY(!m_marked_vectors.contains(vector));
359 m_marked_vectors.append(vector);
360}
361
362void Heap::did_destroy_marked_vector(Badge<MarkedVectorBase>, MarkedVectorBase& vector)
363{
364 VERIFY(m_marked_vectors.contains(vector));
365 m_marked_vectors.remove(vector);
366}
367
368void Heap::did_create_weak_container(Badge<WeakContainer>, WeakContainer& set)
369{
370 VERIFY(!m_weak_containers.contains(set));
371 m_weak_containers.append(set);
372}
373
374void Heap::did_destroy_weak_container(Badge<WeakContainer>, WeakContainer& set)
375{
376 VERIFY(m_weak_containers.contains(set));
377 m_weak_containers.remove(set);
378}
379
380void Heap::defer_gc(Badge<DeferGC>)
381{
382 ++m_gc_deferrals;
383}
384
385void Heap::undefer_gc(Badge<DeferGC>)
386{
387 VERIFY(m_gc_deferrals > 0);
388 --m_gc_deferrals;
389
390 if (!m_gc_deferrals) {
391 if (m_should_gc_when_deferral_ends)
392 collect_garbage();
393 m_should_gc_when_deferral_ends = false;
394 }
395}
396
397void Heap::uproot_cell(Cell* cell)
398{
399 m_uprooted_cells.append(cell);
400}
401
402void register_safe_function_closure(void* base, size_t size)
403{
404 if (!s_custom_ranges_for_conservative_scan) {
405 // FIXME: This per-thread HashMap is currently leaked on thread exit.
406 s_custom_ranges_for_conservative_scan = new HashMap<FlatPtr*, size_t>;
407 }
408 auto result = s_custom_ranges_for_conservative_scan->set(reinterpret_cast<FlatPtr*>(base), size);
409 VERIFY(result == AK::HashSetResult::InsertedNewEntry);
410}
411
412void unregister_safe_function_closure(void* base, size_t)
413{
414 VERIFY(s_custom_ranges_for_conservative_scan);
415 bool did_remove = s_custom_ranges_for_conservative_scan->remove(reinterpret_cast<FlatPtr*>(base));
416 VERIFY(did_remove);
417}
418
419}