jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2020, the SerenityOS developers.
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7#include <AK/QuickSort.h>
8#include <AK/ScopedValueRollback.h>
9#include <AK/TemporaryChange.h>
10#include <LibGUI/TextEditor.h>
11#include <LibGfx/Font/Font.h>
12#include <LibGfx/Palette.h>
13#include <Shell/NodeVisitor.h>
14#include <Shell/Parser.h>
15#include <Shell/SyntaxHighlighter.h>
16
17namespace Shell {
18
19enum class AugmentedTokenKind : u32 {
20 __TokenTypeCount = (u32)AST::Node::Kind::__Count,
21 OpenParen,
22 CloseParen,
23};
24
25class HighlightVisitor : public AST::NodeVisitor {
26public:
27 HighlightVisitor(Vector<GUI::TextDocumentSpan>& spans, Gfx::Palette const& palette, const GUI::TextDocument& document)
28 : m_spans(spans)
29 , m_palette(palette)
30 , m_document(document)
31 {
32 }
33
34private:
35 AST::Position::Line offset_line(const AST::Position::Line& line, size_t offset)
36 {
37 // We need to look at the line(s) above.
38 AST::Position::Line new_line { line };
39 while (new_line.line_column < offset) {
40 offset -= new_line.line_column;
41 --offset;
42
43 if (new_line.line_number == 0)
44 break;
45 --new_line.line_number;
46
47 auto& line = m_document.line(new_line.line_number);
48 new_line.line_column = line.length();
49 }
50 if (offset > 0)
51 new_line.line_column -= offset;
52
53 return new_line;
54 }
55 void set_offset_range_end(GUI::TextRange& range, const AST::Position::Line& line, size_t offset = 0)
56 {
57 auto new_line = offset_line(line, offset);
58 range.set_end({ new_line.line_number, new_line.line_column });
59 }
60 void set_offset_range_start(GUI::TextRange& range, const AST::Position::Line& line, size_t offset = 0)
61 {
62 auto new_line = offset_line(line, offset);
63 range.set_start({ new_line.line_number, new_line.line_column });
64 }
65
66 GUI::TextDocumentSpan& span_for_node(const AST::Node* node)
67 {
68 GUI::TextDocumentSpan span;
69 set_offset_range_start(span.range, node->position().start_line);
70 set_offset_range_end(span.range, node->position().end_line);
71 span.data = static_cast<u64>(node->kind());
72 span.is_skippable = false;
73 m_spans.append(move(span));
74
75 return m_spans.last();
76 }
77
78 virtual void visit(const AST::PathRedirectionNode* node) override
79 {
80 if (node->path()->is_bareword()) {
81 auto& span = span_for_node(node->path());
82 span.attributes.color = m_palette.link();
83 span.attributes.underline = true;
84 } else {
85 NodeVisitor::visit(node);
86 }
87 }
88 virtual void visit(const AST::And* node) override
89 {
90 {
91 ScopedValueRollback first_in_command { m_is_first_in_command };
92 node->left()->visit(*this);
93 }
94 {
95 ScopedValueRollback first_in_command { m_is_first_in_command };
96 node->right()->visit(*this);
97 }
98
99 auto& span = span_for_node(node);
100 set_offset_range_start(span.range, node->and_position().start_line);
101 set_offset_range_end(span.range, node->and_position().end_line);
102 span.attributes.color = m_palette.syntax_punctuation();
103 span.attributes.bold = true;
104 }
105 virtual void visit(const AST::ListConcatenate* node) override
106 {
107 NodeVisitor::visit(node);
108 }
109 virtual void visit(const AST::Background* node) override
110 {
111 NodeVisitor::visit(node);
112
113 auto& span = span_for_node(node);
114 set_offset_range_start(span.range, node->position().end_line, 1);
115 span.attributes.color = m_palette.syntax_punctuation();
116 span.attributes.bold = true;
117 }
118 virtual void visit(const AST::BraceExpansion* node) override
119 {
120 NodeVisitor::visit(node);
121 }
122 virtual void visit(const AST::BarewordLiteral* node) override
123 {
124 NodeVisitor::visit(node);
125
126 auto& span = span_for_node(node);
127 if (m_is_first_in_command) {
128 span.attributes.color = m_palette.syntax_keyword();
129 span.attributes.bold = true;
130 m_is_first_in_command = false;
131 } else if (node->text().starts_with('-')) {
132 span.attributes.color = m_palette.syntax_preprocessor_statement();
133 } else {
134 span.attributes.color = m_palette.base_text();
135 }
136 }
137 virtual void visit(const AST::CastToCommand* node) override
138 {
139 NodeVisitor::visit(node);
140 }
141 virtual void visit(const AST::CastToList* node) override
142 {
143 NodeVisitor::visit(node);
144
145 auto& start_span = span_for_node(node);
146 start_span.attributes.color = m_palette.syntax_punctuation();
147 start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
148 start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
149
150 auto& end_span = span_for_node(node);
151 end_span.attributes.color = m_palette.syntax_punctuation();
152 set_offset_range_start(end_span.range, node->position().end_line, 1);
153 end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
154 }
155 virtual void visit(const AST::CloseFdRedirection* node) override
156 {
157 NodeVisitor::visit(node);
158 }
159 virtual void visit(const AST::CommandLiteral* node) override
160 {
161 NodeVisitor::visit(node);
162 }
163 virtual void visit(const AST::Comment* node) override
164 {
165 NodeVisitor::visit(node);
166
167 auto& span = span_for_node(node);
168 span.attributes.color = m_palette.syntax_comment();
169 }
170 virtual void visit(const AST::ContinuationControl* node) override
171 {
172 NodeVisitor::visit(node);
173
174 auto& span = span_for_node(node);
175 span.attributes.color = m_palette.syntax_control_keyword();
176 }
177 virtual void visit(const AST::DynamicEvaluate* node) override
178 {
179 NodeVisitor::visit(node);
180
181 auto& start_span = span_for_node(node);
182 start_span.attributes.color = m_palette.syntax_punctuation();
183 start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 1 });
184 }
185 virtual void visit(const AST::DoubleQuotedString* node) override
186 {
187 NodeVisitor::visit(node);
188
189 auto& start_span = span_for_node(node);
190 start_span.attributes.color = m_palette.syntax_string();
191 start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 1 });
192 start_span.is_skippable = true;
193
194 auto& end_span = span_for_node(node);
195 set_offset_range_start(end_span.range, node->position().end_line, 1);
196 end_span.attributes.color = m_palette.syntax_string();
197 end_span.is_skippable = true;
198
199 if (m_is_first_in_command) {
200 start_span.attributes.bold = true;
201 end_span.attributes.bold = true;
202 }
203 m_is_first_in_command = false;
204 }
205 virtual void visit(const AST::Fd2FdRedirection* node) override
206 {
207 NodeVisitor::visit(node);
208 }
209 virtual void visit(const AST::FunctionDeclaration* node) override
210 {
211 NodeVisitor::visit(node);
212
213 // fn name
214 auto& name_span = span_for_node(node);
215 set_offset_range_start(name_span.range, node->name().position.start_line);
216 set_offset_range_end(name_span.range, node->name().position.end_line);
217 name_span.attributes.color = m_palette.syntax_identifier();
218
219 // arguments
220 for (auto& arg : node->arguments()) {
221 auto& name_span = span_for_node(node);
222 set_offset_range_start(name_span.range, arg.position.start_line);
223 set_offset_range_end(name_span.range, arg.position.end_line);
224 name_span.attributes.color = m_palette.syntax_identifier();
225 }
226 }
227 virtual void visit(const AST::ForLoop* node) override
228 {
229 // The iterated expression is an expression, not a command.
230 m_is_first_in_command = false;
231 NodeVisitor::visit(node);
232
233 // "for"
234 auto& for_span = span_for_node(node);
235 // FIXME: "fo\\\nr" is valid too
236 for_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 3 });
237 for_span.attributes.color = m_palette.syntax_keyword();
238
239 // "in"
240 if (auto maybe_position = node->in_keyword_position(); maybe_position.has_value()) {
241 auto& position = maybe_position.value();
242
243 auto& in_span = span_for_node(node);
244 set_offset_range_start(in_span.range, position.start_line);
245 set_offset_range_end(in_span.range, position.end_line);
246 in_span.attributes.color = m_palette.syntax_keyword();
247 }
248
249 // "index"
250 if (auto maybe_position = node->index_keyword_position(); maybe_position.has_value()) {
251 auto& position = maybe_position.value();
252
253 auto& index_span = span_for_node(node);
254 set_offset_range_start(index_span.range, position.start_line);
255 set_offset_range_end(index_span.range, position.end_line);
256 index_span.attributes.color = m_palette.syntax_keyword();
257 }
258
259 // variables
260 if (auto maybe_variable = node->variable(); maybe_variable.has_value()) {
261 auto& position = maybe_variable->position;
262
263 auto& variable_span = span_for_node(node);
264 set_offset_range_start(variable_span.range, position.start_line);
265 set_offset_range_end(variable_span.range, position.end_line);
266 variable_span.attributes.color = m_palette.syntax_identifier();
267 }
268
269 if (auto maybe_variable = node->index_variable(); maybe_variable.has_value()) {
270 auto& position = maybe_variable->position;
271
272 auto& variable_span = span_for_node(node);
273 set_offset_range_start(variable_span.range, position.start_line);
274 set_offset_range_end(variable_span.range, position.end_line);
275 variable_span.attributes.color = m_palette.syntax_identifier();
276 }
277 }
278 virtual void visit(const AST::Glob* node) override
279 {
280 NodeVisitor::visit(node);
281
282 auto& span = span_for_node(node);
283 span.attributes.color = m_palette.syntax_preprocessor_value();
284 }
285 virtual void visit(const AST::Execute* node) override
286 {
287 TemporaryChange first { m_is_first_in_command, true };
288 NodeVisitor::visit(node);
289
290 if (node->does_capture_stdout()) {
291 auto& start_span = span_for_node(node);
292 start_span.attributes.color = m_palette.syntax_punctuation();
293 start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
294 start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
295
296 auto& end_span = span_for_node(node);
297 end_span.attributes.color = m_palette.syntax_punctuation();
298 set_offset_range_start(end_span.range, node->position().end_line, 1);
299 end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
300 }
301 }
302 virtual void visit(const AST::IfCond* node) override
303 {
304 m_is_first_in_command = false;
305 NodeVisitor::visit(node);
306
307 // "if"
308 auto& if_span = span_for_node(node);
309 // FIXME: "i\\\nf" is valid too
310 if_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
311 if_span.attributes.color = m_palette.syntax_keyword();
312
313 // "else"
314 if (auto maybe_position = node->else_position(); maybe_position.has_value()) {
315 auto& position = maybe_position.value();
316
317 auto& else_span = span_for_node(node);
318 set_offset_range_start(else_span.range, position.start_line);
319 set_offset_range_end(else_span.range, position.end_line);
320 else_span.attributes.color = m_palette.syntax_keyword();
321 }
322 }
323
324 virtual void visit(const AST::ImmediateExpression* node) override
325 {
326 TemporaryChange first { m_is_first_in_command, false };
327 NodeVisitor::visit(node);
328
329 // ${
330 auto& start_span = span_for_node(node);
331 start_span.attributes.color = m_palette.syntax_punctuation();
332 start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
333 start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
334
335 // Function name
336 auto& name_span = span_for_node(node);
337 name_span.attributes.color = m_palette.syntax_preprocessor_statement(); // Closest thing we have to this
338 set_offset_range_start(name_span.range, node->function_position().start_line);
339 set_offset_range_end(name_span.range, node->function_position().end_line);
340
341 // }
342 auto& end_span = span_for_node(node);
343 end_span.attributes.color = m_palette.syntax_punctuation();
344 set_offset_range_start(end_span.range, node->position().end_line, 1);
345 end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
346 }
347
348 virtual void visit(const AST::Join* node) override
349 {
350 NodeVisitor::visit(node);
351 }
352 virtual void visit(const AST::MatchExpr* node) override
353 {
354 // The matched expression is an expression, not a command.
355 m_is_first_in_command = false;
356 NodeVisitor::visit(node);
357
358 // "match"
359 auto& match_expr = span_for_node(node);
360 // FIXME: "mat\\\nch" is valid too
361 match_expr.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 5 });
362 match_expr.attributes.color = m_palette.syntax_keyword();
363
364 // "as"
365 if (auto maybe_position = node->as_position(); maybe_position.has_value()) {
366 auto& position = maybe_position.value();
367
368 auto& as_span = span_for_node(node);
369 as_span.range.set_start({ position.start_line.line_number, position.start_line.line_column });
370 as_span.range.set_end({ position.end_line.line_number, position.end_line.line_column + 1 });
371 as_span.attributes.color = m_palette.syntax_keyword();
372 }
373 }
374 virtual void visit(const AST::Or* node) override
375 {
376 {
377 ScopedValueRollback first_in_command { m_is_first_in_command };
378 node->left()->visit(*this);
379 }
380 {
381 ScopedValueRollback first_in_command { m_is_first_in_command };
382 node->right()->visit(*this);
383 }
384
385 auto& span = span_for_node(node);
386 set_offset_range_start(span.range, node->or_position().start_line);
387 set_offset_range_end(span.range, node->or_position().end_line);
388 span.attributes.color = m_palette.syntax_punctuation();
389 span.attributes.bold = true;
390 }
391 virtual void visit(const AST::Pipe* node) override
392 {
393 NodeVisitor::visit(node);
394 }
395 virtual void visit(const AST::Range* node) override
396 {
397 NodeVisitor::visit(node);
398
399 auto& start_span = span_for_node(node->start());
400 auto& start_position = node->start()->position();
401 set_offset_range_start(start_span.range, start_position.start_line, 1);
402 start_span.range.set_end({ start_position.start_line.line_number, start_position.start_line.line_column + 1 });
403 start_span.attributes.color = m_palette.syntax_punctuation();
404
405 auto& end_span = span_for_node(node->start());
406 auto& end_position = node->end()->position();
407 set_offset_range_start(end_span.range, end_position.end_line, 1);
408 start_span.range.set_end({ end_position.start_line.line_number, end_position.start_line.line_column + 1 });
409
410 end_span.attributes.color = m_palette.syntax_punctuation();
411 }
412 virtual void visit(const AST::ReadRedirection* node) override
413 {
414 NodeVisitor::visit(node);
415 }
416 virtual void visit(const AST::ReadWriteRedirection* node) override
417 {
418 NodeVisitor::visit(node);
419 }
420 virtual void visit(const AST::Sequence* node) override
421 {
422 for (auto& entry : node->entries()) {
423 ScopedValueRollback first_in_command { m_is_first_in_command };
424 entry->visit(*this);
425 }
426
427 for (auto& position : node->separator_positions()) {
428 if (position.start_offset == position.end_offset)
429 continue;
430 auto& span = span_for_node(node);
431 set_offset_range_start(span.range, position.start_line);
432 set_offset_range_end(span.range, position.end_line);
433 span.attributes.color = m_palette.syntax_punctuation();
434 span.attributes.bold = true;
435 span.is_skippable = true;
436 }
437 }
438 virtual void visit(const AST::Subshell* node) override
439 {
440 NodeVisitor::visit(node);
441 }
442 virtual void visit(const AST::SimpleVariable* node) override
443 {
444 NodeVisitor::visit(node);
445
446 auto& span = span_for_node(node);
447 span.attributes.color = m_palette.syntax_identifier();
448 }
449 virtual void visit(const AST::SpecialVariable* node) override
450 {
451 NodeVisitor::visit(node);
452
453 auto& span = span_for_node(node);
454 span.attributes.color = m_palette.syntax_identifier();
455 }
456 virtual void visit(const AST::Juxtaposition* node) override
457 {
458 NodeVisitor::visit(node);
459 }
460 virtual void visit(const AST::StringLiteral* node) override
461 {
462 NodeVisitor::visit(node);
463
464 if (node->text().is_empty())
465 return;
466
467 auto& span = span_for_node(node);
468 span.attributes.color = m_palette.syntax_string();
469 if (m_is_first_in_command)
470 span.attributes.bold = true;
471 m_is_first_in_command = false;
472 }
473 virtual void visit(const AST::StringPartCompose* node) override
474 {
475 NodeVisitor::visit(node);
476 }
477 virtual void visit(const AST::SyntaxError* node) override
478 {
479 NodeVisitor::visit(node);
480
481 auto& span = span_for_node(node);
482 span.attributes.underline = true;
483 span.attributes.background_color = Color(Color::NamedColor::MidRed).lightened(1.3f).with_alpha(128);
484 span.attributes.color = m_palette.base_text();
485 }
486 virtual void visit(const AST::Tilde* node) override
487 {
488 NodeVisitor::visit(node);
489
490 auto& span = span_for_node(node);
491 span.attributes.color = m_palette.link();
492 }
493 virtual void visit(const AST::VariableDeclarations* node) override
494 {
495 TemporaryChange first_in_command { m_is_first_in_command, false };
496 for (auto& decl : node->variables()) {
497 auto& name_span = span_for_node(decl.name);
498 name_span.attributes.color = m_palette.syntax_identifier();
499
500 decl.value->visit(*this);
501
502 auto& start_span = span_for_node(decl.name);
503 start_span.range.set_start({ decl.name->position().end_line.line_number, decl.name->position().end_line.line_column });
504 start_span.range.set_end({ decl.value->position().start_line.line_number, decl.value->position().start_line.line_column + 1 });
505 start_span.attributes.color = m_palette.syntax_punctuation();
506 start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
507 }
508 }
509 virtual void visit(const AST::WriteAppendRedirection* node) override
510 {
511 NodeVisitor::visit(node);
512 }
513 virtual void visit(const AST::WriteRedirection* node) override
514 {
515 NodeVisitor::visit(node);
516 }
517
518 Vector<GUI::TextDocumentSpan>& m_spans;
519 Gfx::Palette const& m_palette;
520 const GUI::TextDocument& m_document;
521 bool m_is_first_in_command { false };
522};
523
524bool SyntaxHighlighter::is_identifier(u64 token) const
525{
526 if (!token)
527 return false;
528
529 auto kind = (size_t)token;
530 return kind == (size_t)AST::Node::Kind::BarewordLiteral
531 || kind == (size_t)AST::Node::Kind::StringLiteral
532 || kind == (size_t)AST::Node::Kind::Tilde;
533}
534
535bool SyntaxHighlighter::is_navigatable(u64) const
536{
537 return false;
538}
539
540void SyntaxHighlighter::rehighlight(Palette const& palette)
541{
542 auto text = m_client->get_text();
543
544 Parser parser(text);
545 auto ast = parser.parse();
546
547 Vector<GUI::TextDocumentSpan> spans;
548 HighlightVisitor visitor { spans, palette, m_client->get_document() };
549
550 if (ast)
551 ast->visit(visitor);
552
553 quick_sort(spans, [](auto& a, auto& b) { return a.range.start() < b.range.start() && a.range.end() < b.range.end(); });
554
555 if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
556 for (auto& span : spans) {
557 dbgln("Kind {}, range {}.", span.data, span.range);
558 }
559 }
560
561 m_client->do_set_spans(move(spans));
562 m_has_brace_buddies = false;
563 highlight_matching_token_pair();
564 m_client->do_update();
565}
566
567Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
568{
569 static Vector<MatchingTokenPair> pairs;
570 if (pairs.is_empty()) {
571 pairs.append({
572 static_cast<u64>(AugmentedTokenKind::OpenParen),
573 static_cast<u64>(AugmentedTokenKind::CloseParen),
574 });
575 }
576 return pairs;
577}
578
579bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
580{
581 return token0 == token1;
582}
583
584}