Something went wrong. Try again.
A post-modern development environment.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663mod codegen;mod docgen;mod helpers;mod path;
use std::{env, error::Error};
type DynError = Box<dyn Error>;
pub mod tasks { use crate::codegen::code_gen; use crate::DynError; use std::collections::HashSet;
pub fn docgen() -> Result<(), DynError> { use crate::docgen::*; write(TYPABLE_COMMANDS_MD_OUTPUT, &typable_commands()?); write(STATIC_COMMANDS_MD_OUTPUT, &static_commands()?); write(LANG_SUPPORT_MD_OUTPUT, &lang_features()?); Ok(()) }
pub fn querycheck(languages: impl Iterator<Item = String>) -> Result<(), DynError> { use helix_core::syntax::LanguageData;
let languages_to_check: HashSet<_> = languages.collect(); let loader = helix_core::config::default_lang_loader(); for (_language, lang_data) in loader.languages() { if !languages_to_check.is_empty() && !languages_to_check.contains(&lang_data.config().language_id) { continue; } let config = lang_data.config(); let Some(syntax_config) = LanguageData::compile_syntax_config(config, &loader)? else { continue; }; let grammar = syntax_config.grammar; LanguageData::compile_indent_query(grammar, config)?; LanguageData::compile_textobject_query(grammar, config)?; LanguageData::compile_tag_query(grammar, config)?; LanguageData::compile_rainbow_query(grammar, config)?; }
println!("Query check succeeded");
Ok(()) }
pub fn codegen() { code_gen() }
pub fn install_steel() { std::process::Command::new("cargo") .args([ "install", "--git", "https://github.com/mattwparas/steel.git", "steel-interpreter", "steel-language-server", "cargo-steel-lib", "--locked", "--force", ]) .spawn() .unwrap() .wait() .unwrap();
std::process::Command::new("cargo") .args([ "install", "--git", "https://github.com/mattwparas/steel.git", "steel-forge", "--locked", "--force", ]) .spawn() .unwrap() .wait() .unwrap();
println!("----------------------------"); println!("=> Finished installing steel"); println!("----------------------------"); println!("Warming up `forge`...");
let forge = std::process::Command::new("forge") .args(["pkg", "refresh"]) .spawn() .unwrap() .wait();
match forge { Ok(_) => { println!("Done."); println!("----------------------------"); } Err(e) => { println!("Error calling forge !!! Make sure that ~/.cargo/bin or $CARGO_HOME/bin are loaded into the path."); println!("----------------------------"); panic!("{:?}", e) } }
code_gen();
std::process::Command::new("cargo") .args([ "install", "--path", "helix-term", "--features", "steel,git", "--locked", "--force", ]) .spawn() .unwrap() .wait() .unwrap(); }
pub fn indentcheck(languages: impl Iterator<Item = String>) -> Result<(), DynError> { use helix_core::{ indent::{ is_opaque_interior, is_outdent_token_at, treesitter_indent_for_pos, IndentStyle, }, Syntax, }; use helix_stdx::rope::RopeSliceExt; use ropey::Rope;
let filter: HashSet<String> = languages.collect(); let loader = helix_core::config::default_lang_loader(); let corpus = crate::path::tests_indent(); let tab_width = 4; let mut errors = 0usize; let mut over_notes = 0usize;
let mut entries: Vec<_> = std::fs::read_dir(&corpus)? .filter_map(Result::ok) .map(|e| e.path()) .filter(|p| p.is_file()) .collect(); entries.sort();
for path in entries { let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(""); if !filter.is_empty() && !filter.contains(stem) { continue; } let file = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
// Corpus files are named <language-id>.<ext>; resolve the language by id. let language = match loader .languages() .find(|(_, data)| data.config().language_id == stem) { Some((language, _)) => language, None => { return Err(format!( "{file}: no configured language with id '{stem}' (corpus files are named <language-id>.<ext>)" ) .into()) } };
let config = loader.language(language).config(); let indent_style = IndentStyle::from_str( &config .indent .as_ref() .ok_or_else(|| format!("{file}: language '{stem}' has no indent config"))? .unit, ); // Lines that are commented out are skipped: they self-document edge // cases (e.g. known indent limitations) without being asserted on. let comment_tokens: Vec<String> = config.comment_tokens.clone().unwrap_or_default();
let doc = Rope::from_reader(&mut std::fs::File::open(&path)?)?; let text = doc.slice(..); let syntax = Syntax::new(text, language, &loader) .map_err(|e| format!("{file}: failed to parse: {e:?}"))?; let indent_query = loader .indent_query(language) .ok_or_else(|| format!("{file}: language '{stem}' has no indent query"))?;
for i in 0..doc.len_lines() { let line = text.line(i); let Some(pos) = line.first_non_whitespace_char() else { continue; };
let trimmed = line.slice(pos..).to_string(); if comment_tokens .iter() .any(|tok| trimmed.starts_with(tok.as_str())) { continue; }
let suggested = treesitter_indent_for_pos( indent_query, &syntax, &loader, tab_width, indent_style.indent_width(tab_width), text, i, text.line_to_char(i) + pos, false, ) .unwrap() .to_string(&indent_style, tab_width);
let actual = line .get_slice(..pos) .map(|s| s.to_string()) .unwrap_or_default();
if actual != suggested { errors += 1; println!( "{file}:{}: reindent expected {} columns, computed {}", i + 1, actual.chars().count(), suggested.chars().count(), ); }
// Typing direction: simulate pressing Enter at the end of this line and check the indent computed for the next line. // - under-indent (computed < canonical) is always a failure: nothing pulls the line further in, so the user is left // under-indented. // - over-indent (computed > canonical) is only acceptable when the next line's leading token is @outdent (a closing // bracket, case/else/except keyword, ...): entering it dedents the line. An over-indent on a plain statement (e.g. // a line after `return` that should leave the block but doesn't) has nothing to correct it and is a real failure. if i + 1 < doc.len_lines() { if let Some(next_pos) = text.line(i + 1).first_non_whitespace_char() { let next = text.line(i + 1); let next_trim = next.slice(next_pos..).to_string(); // Lines inside an @opaque body (string/comment) carry literal leading whitespace, not code indent — don't // assert a typing indent for them. let next_byte = text.char_to_byte(text.line_to_char(i + 1) + next_pos) as u32;
let next_is_opaque = is_opaque_interior(indent_query, &syntax, text, next_byte); let next_is_comment = next_is_opaque || comment_tokens .iter() .any(|tok| next_trim.starts_with(tok.as_str()));
if !next_is_comment { let typed = treesitter_indent_for_pos( indent_query, &syntax, &loader, tab_width, indent_style.indent_width(tab_width), text, i, text.line_to_char(i + 1) - 1, true, ) .unwrap() .to_string(&indent_style, tab_width);
let next_actual = next .get_slice(..next_pos) .map(|s| s.to_string()) .unwrap_or_default();
let typed_cols = typed.chars().count(); let want_cols = next_actual.chars().count(); let leading_outdent = || { let byte = text.char_to_byte(text.line_to_char(i + 1) + next_pos); is_outdent_token_at(indent_query, &syntax, text, byte as u32) };
if typed_cols < want_cols { errors += 1; println!( "{file}:{}: typing under-indents: computed {} columns, expected {} | {}", i + 2, typed_cols, want_cols, next_trim.trim_end(), ); } else if typed_cols > want_cols && !leading_outdent() { // Over-indents are reported but not failed: every over-indent sits at a legitimate dedent point, and // for indent-delimited languages (python after a block) the editor genuinely cannot know how far to // dedent. Surfaced so a regression shows up in the diff; gate on under-indents only. over_notes += 1; println!( "{file}:{}: note: typing over-indents: computed {} columns, expected {} (no leading outdent; review) | {}", i + 2, typed_cols, want_cols, next_trim.trim_end(), ); } } } } } }
if over_notes > 0 { println!("Indent check: {over_notes} typing over-indent note(s) (not failures; review for regressions)"); } match errors { 0 => { println!("Indent check succeeded"); Ok(()) } n => Err(format!("Indent check failed: {n} line(s) with wrong indentation").into()), } }
pub fn themecheck(themes: impl Iterator<Item = String>) -> Result<(), DynError> { use helix_view::theme::Loader;
let themes_to_check: HashSet<_> = themes.collect();
let theme_names = [ vec!["default".to_string(), "base16_default".to_string()], Loader::read_names(&crate::path::themes()), ] .concat(); let loader = Loader::new(&[crate::path::runtime()]); let mut errors_present = false;
for name in theme_names { if !themes_to_check.is_empty() && !themes_to_check.contains(&name) { continue; }
let (_, warnings) = loader.load_with_warnings(&name).unwrap();
if !warnings.is_empty() { errors_present = true; println!("Theme '{name}' loaded with errors:"); for warning in warnings { println!("\t* {}", warning); } } }
match errors_present { true => Err("Errors found when loading bundled themes".into()), false => { println!("Theme check successful!"); Ok(()) } } }
pub fn highlightcheck(args: impl Iterator<Item = String>) -> Result<(), DynError> { use helix_core::syntax::{HighlightEvent, Loader, Syntax}; use helix_core::Language; use ropey::Rope;
// The highlighter yields a `Highlight` index into the loader's scope // list. Feed it the capture names actually used across the bundled // queries, so every capture maps to itself and we read back the exact // `@capture` that won — no hand-maintained scope list. (Inheritance is // irrelevant here: scanning every language's queries unions all names.) fn capture_scopes() -> Vec<String> { let mut set = std::collections::BTreeSet::new(); let Ok(langs) = std::fs::read_dir(crate::path::ts_queries()) else { return Vec::new(); }; for lang in langs.filter_map(Result::ok) { for q in ["highlights.scm", "locals.scm"] { let Ok(text) = std::fs::read_to_string(lang.path().join(q)) else { continue; }; let b = text.as_bytes(); let mut i = 0; while i < b.len() { if b[i] != b'@' { i += 1; continue; } let s = i + 1; let mut j = s; while j < b.len() && (b[j].is_ascii_alphanumeric() || matches!(b[j], b'.' | b'_' | b'-')) { j += 1; } let name = &text[s..j]; i = j; if name.is_empty() || name.starts_with('_') { continue; } // Locals: the highlight applied to a resolved reference is // the class after `local.definition.`; the rest are structural. if let Some(class) = name.strip_prefix("local.definition.") { if !class.is_empty() { set.insert(class.to_string()); } } else if !name.starts_with("local.") && name != "local" { set.insert(name.to_string()); } } } } set.into_iter().collect() }
// Winning (top-of-stack) capture per byte range, via the real highlighter. fn spans( loader: &Loader, scopes: &[String], language: Language, source: &str, ) -> Option<Vec<(usize, usize, String)>> { let rope = Rope::from_str(source); // None when the grammar isn't built, so callers can skip rather than // panic (a no-language run only needs the corpus' grammars built). let syntax = Syntax::new(rope.slice(..), language, loader).ok()?; let mut hl = syntax.highlighter(rope.slice(..), loader, ..); let mut active: Vec<u32> = Vec::new(); let mut start: u32 = 0; let len = source.len() as u32; let mut out = Vec::new(); loop { let off = hl.next_event_offset(); let cur = if off == u32::MAX { len } else { off }; if cur > start { if let Some(idx) = active.last() { out.push((start as usize, cur as usize, scopes[*idx as usize].clone())); } } if off == u32::MAX { break; } let (event, highlights) = hl.advance(); let v: Vec<u32> = highlights.map(|h| h.get()).collect(); match event { HighlightEvent::Refresh => active = v, HighlightEvent::Push => active.extend(v), } start = off; } Some(out) }
let scopes = capture_scopes(); let loader = helix_core::config::default_lang_loader(); loader.set_scopes(scopes.clone());
let args: Vec<String> = args.collect();
// `--dump <language> <file>`: print the winning capture per span, for // discovering the exact `@capture` names when authoring assertions. if args.first().map(String::as_str) == Some("--dump") { let lang = args .get(1) .ok_or("usage: highlight-check --dump <language> <file>")?; let file = args .get(2) .ok_or("usage: highlight-check --dump <language> <file>")?; let language = loader .language_for_name(lang.as_str()) .ok_or_else(|| format!("unknown language '{lang}'"))?; let source = std::fs::read_to_string(file)?; let sp = spans(&loader, &scopes, language, &source) .ok_or_else(|| format!("could not highlight (grammar for '{lang}' not built?)"))?; for (s, e, scope) in sp { let text = &source[s..e]; if !text.trim().is_empty() { println!("{scope}\t{text:?}"); } } return Ok(()); }
// Corpus mode: nvim-treesitter-style highlight tests laid out under // tests/query/highlights/<language-id>/<name>.<ext>. A comment line // below the code carries caret assertions whose columns line up with the // tokens above: // // foo(bar) // // ^ @function // // ^^^ @variable // // Each `^` checks the *winning* capture at the column above it; the // expected `@capture` must match exactly (a leading `!` negates). This // catches precedence bugs query-check cannot — e.g. a call captured on a // wrapper node that the inner `(identifier) @variable` then wins. let filter: HashSet<String> = args.into_iter().collect(); let root = crate::path::tests_highlight(); let mut files: Vec<(String, std::path::PathBuf)> = Vec::new(); for lang_dir in std::fs::read_dir(&root)?.filter_map(Result::ok) { if !lang_dir.path().is_dir() { continue; } let lang = lang_dir.file_name().to_string_lossy().into_owned(); if !filter.is_empty() && !filter.contains(&lang) { continue; } for f in std::fs::read_dir(lang_dir.path())?.filter_map(Result::ok) { if f.path().is_file() { files.push((lang.clone(), f.path())); } } } files.sort();
let mut errors = 0usize; let mut checks = 0usize; for (lang, path) in files { let language = match loader .languages() .find(|(_, d)| d.config().language_id == lang) { Some((language, _)) => language, None => { return Err( format!("{}: no configured language '{lang}'", path.display()).into(), ) } }; let source = std::fs::read_to_string(&path)?; let Some(sp) = spans(&loader, &scopes, language, &source) else { eprintln!("{}: skipped ('{lang}' grammar not built)", path.display()); continue; }; let at = |byte: usize| -> Option<&str> { sp.iter() .find(|(s, e, _)| *s <= byte && byte < *e) .map(|(_, _, n)| n.as_str()) };
let lines: Vec<&str> = source.lines().collect(); let mut line_start = Vec::with_capacity(lines.len()); let mut off = 0usize; for l in &lines { line_start.push(off); off += l.len() + 1; }
let rel = path.strip_prefix(&root).unwrap_or(&path).display(); let mut base: Option<usize> = None; for (li, line) in lines.iter().enumerate() { // An assertion line is a comment whose first `^` is preceded only // by the comment leader (no alphanumerics), so code like `a ^ b` // is never mistaken for one. let marker = line .find('^') .filter(|&c| !line[..c].bytes().any(|b| b.is_ascii_alphanumeric())); let Some(col) = marker else { if !line.trim().is_empty() { base = Some(li); } continue; }; let Some(brow) = base else { continue }; let len = line[col..].bytes().take_while(|b| *b == b'^').count(); let mut exp = line[col + len..].trim(); let negate = exp.starts_with('!'); if negate { exp = exp[1..].trim_start(); } // First whitespace-delimited token after the caret, with a // leading `@` stripped — so a trailing comment close (OCaml // `*)`, HTML `-->`) isn't swallowed into the expected name. let exp = exp .trim_start_matches('@') .split_whitespace() .next() .unwrap_or(""); if exp.is_empty() { continue; } let code = lines[brow]; for c in col..col + len { // Skip carets that fall past the code line or over whitespace. match code.as_bytes().get(c) { None => continue, Some(b) if b.is_ascii_whitespace() => continue, _ => {} } checks += 1; let got = at(line_start[brow] + c); if (got == Some(exp)) != negate { continue; } errors += 1; eprintln!( "{rel}:{}:{}: expected {}{} but got {}", brow + 1, c + 1, if negate { "not " } else { "" }, exp, got.unwrap_or("<unhighlighted>") ); } } }
if errors > 0 { return Err( format!("Highlight check failed: {errors} of {checks} assertion(s) wrong").into(), ); } println!("Highlight check succeeded ({checks} assertions)"); Ok(()) }
pub fn print_help() { println!( "Usage: Run with `cargo xtask <task>`, eg. `cargo xtask docgen`.
Tasks: steel Install steel and helix with steel enabled code-gen Generate any code used for steel docgen Generate files to be included in the mdbook output. query-check [languages] Check that tree-sitter queries are valid for the given languages, or all languages if none are specified. indent-check [languages] Check indentation for the corpus files in tests/indent/ (named <language-id>.<ext>) against the configured grammars, for the given languages, or all corpus files if none are specified. highlight-check [languages] Check highlight queries against the real highlighter using the nvim-treesitter-style tests under tests/query/highlights/ <language-id>/<name>.<ext> (caret comment lines like `// ^^^ @capture` assert the winning capture at the column above), for the given languages, or all if none are specified. `highlight-check --dump <language> <file>` instead prints the winning capture per span for an arbitrary input file. theme-check [themes] Check that the theme files in runtime/themes/ are valid for the given themes, or all themes if none are specified." ); }}
fn main() -> Result<(), DynError> { let mut args = env::args().skip(1); let task = args.next(); match task { None => tasks::print_help(), Some(t) => match t.as_str() { "docgen" => tasks::docgen()?, "code-gen" => tasks::codegen(), "steel" => tasks::install_steel(), "query-check" => tasks::querycheck(args)?, "indent-check" => tasks::indentcheck(args)?, "highlight-check" => tasks::highlightcheck(args)?, "theme-check" => tasks::themecheck(args)?, invalid => return Err(format!("Invalid task name: {}", invalid).into()), }, }; Ok(())}