Commit
Author: Kevin Schoon [me@kevinschoon.com]
Hash: f13cda2754ddcb3bf30b30eb611be68e13d0d0c6
Timestamp: Sun, 26 May 2024 17:20:00 +0000 (8 months ago)

+529 -369 +/-12 browse
add anchor links in headers rendered from markdown
1diff --git a/ayllu/src/highlight.rs b/ayllu/src/highlight.rs
2new file mode 100644
3index 0000000..d53c791
4--- /dev/null
5+++ b/ayllu/src/highlight.rs
6 @@ -0,0 +1,348 @@
7+ use std::collections::HashMap;
8+
9+ use std::fs;
10+ use std::io::{Cursor, Error, Write};
11+ use std::mem;
12+ use std::path::Path;
13+ use std::sync::RwLock;
14+
15+ use comrak::adapters::SyntaxHighlighterAdapter;
16+ use lazy_static::lazy_static;
17+ use log::debug;
18+ use tera::escape_html;
19+ use tree_sitter::Language;
20+ use tree_sitter_highlight::{HighlightConfiguration, Highlighter as TSHighlighter, HtmlRenderer};
21+
22+ use crate::config::TreeSitterParser;
23+ use crate::languages::{Hint, LANGUAGE_TABLE};
24+
25+ lazy_static! {
26+ // global containing all language parsers and syntax highlighter definitions
27+ // this needs to be initialized one time at startup
28+ static ref LANGUAGES: RwLock<HashMap<Hint, Language>> = RwLock::new(HashMap::new());
29+ // highlighter queries
30+ static ref HIGHLIGHTS: RwLock<HashMap<Hint, String>> = RwLock::new(HashMap::new());
31+ // local queries
32+ static ref LOCALS: RwLock<HashMap<Hint, String>> = RwLock::new(HashMap::new());
33+ // injection queries
34+ static ref INJECTIONS: RwLock<HashMap<Hint, String>> = RwLock::new(HashMap::new());
35+ }
36+
37+ unsafe fn load_language(path: &str, hint: &Hint) -> Language {
38+ let lib = libloading::Library::new(path).unwrap();
39+ // rewrite any dash in the module name to use snakecase as is the convention
40+ // for tree-sitter function names.
41+ let method_name = format!("tree_sitter_{}", hint.0.to_lowercase().replace('-', "_"));
42+ // NOTE: maybe, probably? some security auditing that needs to happen here?
43+ debug!("attempting to load method: {}", method_name);
44+ let func: libloading::Symbol<unsafe extern "C" fn() -> Language> =
45+ lib.get(method_name.as_bytes()).unwrap();
46+ let language = func();
47+ mem::forget(lib);
48+ language
49+ }
50+
51+ pub struct Loader {
52+ // typically /usr/lib
53+ lib_path: String,
54+ // typically /usr/share/tree-sitter
55+ query_path: String,
56+ dynamic_ok: bool,
57+ extra_queries_path: Option<String>,
58+ extra_parsers: Vec<TreeSitterParser>,
59+ }
60+
61+ impl Loader {
62+ pub fn new(lib_path: &str, query_path: &str) -> Self {
63+ Loader {
64+ lib_path: lib_path.to_string(),
65+ query_path: query_path.to_string(),
66+ dynamic_ok: false,
67+ extra_queries_path: None,
68+ extra_parsers: Vec::new(),
69+ }
70+ }
71+
72+ pub fn dynamic(mut self, value: bool) -> Self {
73+ self.dynamic_ok = value;
74+ self
75+ }
76+
77+ pub fn extra_queries(mut self, path: &str) -> Self {
78+ self.extra_queries_path = Some(path.to_string());
79+ self
80+ }
81+
82+ pub fn parsers(mut self, parsers: Vec<TreeSitterParser>) -> Self {
83+ self.extra_parsers = parsers.clone();
84+ self
85+ }
86+
87+ fn try_query_load(&self, path: &str, hint: &Hint) -> Result<(), Error> {
88+ let query_base_path = Path::new(path).join(hint.0.to_string().to_lowercase());
89+ if let Ok(queries) = fs::read_dir(query_base_path) {
90+ for query in queries {
91+ let query = query?;
92+ match query.file_name().into_string().unwrap().as_str() {
93+ "highlights.scm" => {
94+ let highlight_scm = fs::read_to_string(query.path())?;
95+ HIGHLIGHTS
96+ .write()
97+ .unwrap()
98+ .insert(hint.clone(), highlight_scm);
99+ debug!("loaded [{:?}] {}", hint, query.path().display());
100+ }
101+ "locals.scm" => {
102+ let locals_scm = fs::read_to_string(query.path())?;
103+ LOCALS.write().unwrap().insert(hint.clone(), locals_scm);
104+ debug!("loaded [{:?}] {}", hint, query.path().display());
105+ }
106+ "injections.scm" => {
107+ let injections_scm = fs::read_to_string(query.path())?;
108+ INJECTIONS
109+ .write()
110+ .unwrap()
111+ .insert(hint.clone(), injections_scm);
112+ debug!("loaded [{:?}] {}", hint, query.path().display());
113+ }
114+ name => {
115+ debug!("ignoring query file: {}", name)
116+ }
117+ }
118+ }
119+ }
120+ Ok(())
121+ }
122+
123+ pub fn load(&self) -> Result<(), Error> {
124+ let modules = fs::read_dir(Path::new(&self.lib_path))?;
125+ for module in modules {
126+ let fp = module?;
127+ let file_name = fp.file_name().into_string().unwrap();
128+ if file_name.starts_with("libtree-sitter-") && file_name.ends_with(".so") {
129+ let language_name = file_name.replace("libtree-sitter-", "").replace(".so", "");
130+
131+ let hint = Hint(language_name);
132+ let language = unsafe { load_language(fp.path().to_str().unwrap(), &hint) };
133+ LANGUAGES.write().unwrap().insert(hint.clone(), language);
134+ debug!(
135+ "loaded tree-sitter shared module: [{:?}] {}",
136+ hint,
137+ fp.path().display()
138+ );
139+ self.try_query_load(self.query_path.as_str(), &hint)?;
140+ // override any base queries with additional queries
141+ self.extra_queries_path
142+ .as_ref()
143+ .map(|path| self.try_query_load(path.as_str(), &hint))
144+ .transpose()?;
145+ }
146+ }
147+ for parser in &self.extra_parsers {
148+ let hint = Hint(parser.language.clone());
149+ let language = unsafe { load_language(&parser.shared_object, &hint) };
150+ LANGUAGES.write().unwrap().insert(hint.clone(), language);
151+ parser
152+ .highlight_query
153+ .as_ref()
154+ .map(|path| {
155+ let highlight_scm = fs::read_to_string(path)?;
156+ HIGHLIGHTS
157+ .write()
158+ .unwrap()
159+ .insert(hint.clone(), highlight_scm);
160+ Ok::<(), Error>(())
161+ })
162+ .transpose()?;
163+ parser
164+ .locals_query
165+ .as_ref()
166+ .map(|path| {
167+ let locals_scm = fs::read_to_string(path)?;
168+ LOCALS.write().unwrap().insert(hint.clone(), locals_scm);
169+ Ok::<(), Error>(())
170+ })
171+ .transpose()?;
172+ parser
173+ .injections_query
174+ .as_ref()
175+ .map(|path| {
176+ let injections_scm = fs::read_to_string(path)?;
177+ INJECTIONS
178+ .write()
179+ .unwrap()
180+ .insert(hint.clone(), injections_scm);
181+ Ok::<(), Error>(())
182+ })
183+ .transpose()?;
184+ }
185+ Ok(())
186+ }
187+ }
188+
189+ fn render_lines(lines: Vec<&str>, show_line_numbers: bool) -> String {
190+ let buf = Vec::new();
191+ let mut file = Cursor::new(buf);
192+ write!(&mut file, "<table class=\"code\">").unwrap();
193+ for (i, line) in lines.into_iter().enumerate() {
194+ if show_line_numbers {
195+ write!(&mut file, "<tr><td class=line-number>{:?}</td>", i + 1).unwrap();
196+ }
197+ write!(&mut file, "<td class=line>{}</td></tr>", line).unwrap();
198+ }
199+ write!(&mut file, "</table>").unwrap();
200+ let bytes = file.into_inner();
201+ String::from_utf8(bytes).unwrap()
202+ }
203+
204+ #[derive(Clone, Debug)]
205+ pub struct Highlighter {
206+ names: Vec<String>,
207+ classes: Vec<String>,
208+ }
209+
210+ impl Highlighter {
211+ pub fn new(names: &[String]) -> Self {
212+ let mut classes: Vec<String> = Vec::new();
213+
214+ for name in names.iter() {
215+ classes.push(format!("class=\"ts_{}\"", name.replace('.', "_")));
216+ }
217+ Self {
218+ names: names.to_vec(),
219+ classes,
220+ }
221+ }
222+
223+ pub fn highlight(
224+ &self,
225+ code: &str,
226+ filepath: Option<&str>,
227+ alias: Option<&str>,
228+ hint: Option<Hint>,
229+ show_line_numbers: bool,
230+ ) -> (Option<Hint>, String) {
231+ let buf = Vec::new();
232+ let mut file = Cursor::new(buf);
233+ write!(&mut file, "<table>").unwrap();
234+
235+ let hint = match hint {
236+ Some(hint) => Some(hint),
237+ None => LANGUAGE_TABLE.guess(code, alias, filepath),
238+ };
239+
240+ debug!("language hint: {:?}", hint);
241+
242+ match hint {
243+ Some(hint) => match (
244+ LANGUAGES.read().unwrap().get(&hint),
245+ HIGHLIGHTS.read().unwrap().get(&hint),
246+ ) {
247+ (Some(language), Some(syntax)) => {
248+ debug!("painting syntax for language: {:?}", hint);
249+ let mut highlighter = TSHighlighter::new();
250+ let injections = INJECTIONS
251+ .read()
252+ .unwrap()
253+ .get(&hint)
254+ .cloned()
255+ .unwrap_or_default();
256+ let locals = LOCALS
257+ .read()
258+ .unwrap()
259+ .get(&hint)
260+ .cloned()
261+ .unwrap_or_default();
262+ let mut config =
263+ HighlightConfiguration::new(*language, syntax, &injections, &locals)
264+ .unwrap();
265+
266+ config.configure(&self.names);
267+
268+ let code = code.as_bytes();
269+
270+ let events = highlighter
271+ .highlight(&config, code, None, |_| None)
272+ .unwrap();
273+
274+ let mut renderer = HtmlRenderer::new();
275+
276+ renderer
277+ .render(events, code, &move |highlight| {
278+ let ret = match self.classes.get(highlight.0) {
279+ Some(name) => name.as_bytes(),
280+ None => "".as_bytes(),
281+ };
282+ ret
283+ })
284+ .unwrap();
285+
286+ (
287+ Some(hint.clone()),
288+ render_lines(renderer.lines().collect(), show_line_numbers),
289+ )
290+ }
291+ _ => {
292+ debug!("cannot paint syntax for language: {:?}", hint);
293+ (
294+ None,
295+ render_lines(escape_html(code).lines().collect(), show_line_numbers),
296+ )
297+ }
298+ },
299+ None => {
300+ debug!("cannot determine language");
301+ (
302+ None,
303+ render_lines(code.to_string().lines().collect(), show_line_numbers),
304+ )
305+ }
306+ }
307+ }
308+
309+ pub fn highlight_vec(
310+ &self,
311+ content: Vec<u8>,
312+ filepath: Option<&str>,
313+ alias: Option<&str>,
314+ hint: Option<Hint>,
315+ show_line_numbers: bool,
316+ ) -> (Option<Hint>, String) {
317+ let content = String::from_utf8(content).unwrap();
318+ self.highlight(content.as_str(), filepath, alias, hint, show_line_numbers)
319+ }
320+ }
321+
322+ #[derive(Debug, Clone)]
323+ pub struct TreeSitterAdapter(pub Highlighter);
324+
325+ impl TreeSitterAdapter {}
326+
327+ impl SyntaxHighlighterAdapter for TreeSitterAdapter {
328+ fn write_highlighted(
329+ &self,
330+ output: &mut dyn std::io::Write,
331+ lang: Option<&str>,
332+ code: &str,
333+ ) -> std::io::Result<()> {
334+ let (_, highlighted) = self.0.highlight(code, None, lang, None, false);
335+ output.write_all(highlighted.as_bytes()).unwrap();
336+ Ok(())
337+ }
338+
339+ fn write_pre_tag(
340+ &self,
341+ _: &mut dyn std::io::Write,
342+ _: std::collections::HashMap<String, String>,
343+ ) -> std::io::Result<()> {
344+ Ok(())
345+ }
346+
347+ fn write_code_tag(
348+ &self,
349+ _: &mut dyn std::io::Write,
350+ _: std::collections::HashMap<String, String>,
351+ ) -> std::io::Result<()> {
352+ Ok(())
353+ }
354+ }
355 diff --git a/ayllu/src/main.rs b/ayllu/src/main.rs
356index fceb0c5..2a7989d 100644
357--- a/ayllu/src/main.rs
358+++ b/ayllu/src/main.rs
359 @@ -16,6 +16,8 @@ mod config;
360 mod database_ext;
361 mod languages;
362 mod license;
363+ mod readme;
364+ mod highlight;
365 mod time;
366
367 #[derive(Parser, Debug)]
368 diff --git a/ayllu/src/readme.rs b/ayllu/src/readme.rs
369new file mode 100644
370index 0000000..64f7248
371--- /dev/null
372+++ b/ayllu/src/readme.rs
373 @@ -0,0 +1,143 @@
374+ use crate::highlight::TreeSitterAdapter;
375+ use comrak::{
376+ adapters::HeadingAdapter, markdown_to_html_with_plugins, Anchorizer, ComrakOptions,
377+ ComrakPlugins,
378+ };
379+
380+ // TODO: Make configurable as part of the loaded theme
381+ const ARROW_SYMBOL_SVG: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" class="icon" width="24" height="24" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .75.75 0 0 1 .018-1.042.75.75 0 0 1 1.042-.018 2 2 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.75.75 0 0 1-1.042-.018.75.75 0 0 1-.018-1.042m-4.69 9.64a2 2 0 0 0 2.83 0l1.25-1.25a.75.75 0 0 1 1.042.018.75.75 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .75.75 0 0 1-.018 1.042.75.75 0 0 1-1.042.018 2 2 0 0 0-2.83 0l-2.5 2.5a2 2 0 0 0 0 2.83"></path></svg>"#;
382+
383+ struct HeadingWriter;
384+
385+ impl HeadingAdapter for HeadingWriter {
386+ fn enter(
387+ &self,
388+ output: &mut dyn std::io::prelude::Write,
389+ heading: &comrak::adapters::HeadingMeta,
390+ _sourcepos: Option<comrak::nodes::Sourcepos>,
391+ ) -> std::io::Result<()> {
392+ let mut anchorizer = Anchorizer::new();
393+ let id = anchorizer.anchorize(heading.content.clone());
394+ let level = heading.level;
395+ output.write_all(format!(
396+ "<header><a class=\"icon-link\" href=\"#{id}\">{ARROW_SYMBOL_SVG}</a><h{level} id={id} dir=auto>"
397+ ).as_bytes())
398+ }
399+
400+ fn exit(
401+ &self,
402+ output: &mut dyn std::io::prelude::Write,
403+ heading: &comrak::adapters::HeadingMeta,
404+ ) -> std::io::Result<()> {
405+ let level = heading.level;
406+ output.write_all(format!("</h{level}></header>").as_bytes())
407+ }
408+ }
409+
410+ /// Responsible for transforming markdown based readme files into HTML
411+ pub struct Renderer<'a> {
412+ /// base URL used for re-writes
413+ pub origin: &'a str,
414+ /// collection name
415+ pub collection: &'a str,
416+ /// project name
417+ pub name: &'a str,
418+ /// enable comrak unsafe markdown option
419+ pub allow_unsafe_markdown: bool,
420+ }
421+
422+ impl Renderer<'_> {
423+ pub fn render(self, adapter: TreeSitterAdapter, readme: &str) -> String {
424+ let mut opts = ComrakOptions::default();
425+ opts.extension.description_lists = true;
426+ opts.extension.footnotes = true;
427+ opts.extension.strikethrough = true;
428+ opts.extension.superscript = true;
429+ opts.extension.table = true;
430+ opts.extension.tasklist = true;
431+
432+ opts.parse.smart = true;
433+ opts.parse.relaxed_tasklist_matching = true;
434+ // allow raw html in the markdown documents
435+ opts.render.unsafe_ = self.allow_unsafe_markdown;
436+ // not relevent since we paint ourselves I think
437+ // opts.render.github_pre_lang = false;
438+
439+ let mut plugins = ComrakPlugins::default();
440+ plugins.render.codefence_syntax_highlighter = Some(&adapter);
441+ plugins.render.heading_adapter = Some(&HeadingWriter {});
442+ markdown_to_html_with_plugins(readme, &opts, &plugins)
443+ }
444+ }
445+
446+ #[cfg(test)]
447+
448+ mod tests {
449+
450+ use super::*;
451+
452+ use crate::highlight::Highlighter;
453+
454+ #[test]
455+ fn renderer_basic() {
456+ let renderer = Renderer {
457+ origin: "https://ayllu-forge.org",
458+ collection: "projects",
459+ name: "ayllu",
460+ allow_unsafe_markdown: true,
461+ };
462+
463+ let highlighter = Highlighter::new(&[]);
464+ let adapter = TreeSitterAdapter(highlighter);
465+
466+ let result = renderer.render(adapter, "# Hello World");
467+ assert!(result
468+ .contains("<h1 id=hello-world dir=auto><a class=\"anchor\" href=\"#hello-world\">"));
469+ }
470+
471+ #[test]
472+ fn rewrite_links() {
473+
474+ // TODO
475+
476+ /*
477+ let renderer = Renderer {
478+ origin: "https://ayllu-forge.org",
479+ collection: "projects",
480+ name: "ayllu",
481+ allow_unsafe_markdown: true,
482+ };
483+
484+ let highlighter = Highlighter::new(&[]);
485+ let adapter = TreeSitterAdapter(highlighter);
486+
487+ let result = renderer.render(
488+ adapter,
489+ r#"
490+ # Ayllu
491+
492+ ## Hyper Performant Hackable Code Forge Built on Open Standards
493+
494+ [fuu]("/fuu")
495+ [bar](./bar)
496+ [baz]("./aaaa/baz/")
497+ [qux]("./bbbb/cccc/qux/")
498+
499+ ### Fascinating Content!
500+
501+ ddddeeeeffffggg
502+
503+ 111111111111111111111111111111111111111111111
504+
505+ "#,
506+ );
507+
508+
509+ assert!(result.contains("https://ayllu-forge.org/projects/ayllu/render/fuu"));
510+ assert!(result.contains("https://ayllu-forge.org/projects/ayllu/render/bar"));
511+ assert!(result.contains("https://ayllu-forge.org/projects/ayllu/render/aaaa/baz"));
512+ assert!(result.contains("https://ayllu-forge.org/projects/ayllu/render/bbbb/cccc/qux"));
513+
514+ */
515+ }
516+ }
517 diff --git a/ayllu/src/web2/highlight.rs b/ayllu/src/web2/highlight.rs
518deleted file mode 100644
519index c276560..0000000
520--- a/ayllu/src/web2/highlight.rs
521+++ /dev/null
522 @@ -1,354 +0,0 @@
523- use std::collections::HashMap;
524-
525- use std::fs;
526- use std::io::{Cursor, Error, Write};
527- use std::mem;
528- use std::path::Path;
529- use std::sync::RwLock;
530-
531- use comrak::adapters::SyntaxHighlighterAdapter;
532- use lazy_static::lazy_static;
533- use log::debug;
534- use tera::escape_html;
535- use tree_sitter::Language;
536- use tree_sitter_highlight::{HighlightConfiguration, Highlighter as TSHighlighter, HtmlRenderer};
537-
538- use crate::config::TreeSitterParser;
539- use crate::languages::{Hint, LANGUAGE_TABLE};
540-
541- lazy_static! {
542- // global containing all language parsers and syntax highlighter definitions
543- // this needs to be initialized one time at startup
544- static ref LANGUAGES: RwLock<HashMap<Hint, Language>> = RwLock::new(HashMap::new());
545- // highlighter queries
546- static ref HIGHLIGHTS: RwLock<HashMap<Hint, String>> = RwLock::new(HashMap::new());
547- // local queries
548- static ref LOCALS: RwLock<HashMap<Hint, String>> = RwLock::new(HashMap::new());
549- // injection queries
550- static ref INJECTIONS: RwLock<HashMap<Hint, String>> = RwLock::new(HashMap::new());
551- }
552-
553- unsafe fn load_language(path: &str, hint: &Hint) -> Language {
554- let lib = libloading::Library::new(path).unwrap();
555- // rewrite any dash in the module name to use snakecase as is the convention
556- // for tree-sitter function names.
557- let method_name = format!("tree_sitter_{}", hint.0.to_lowercase().replace('-', "_"));
558- // NOTE: maybe, probably? some security auditing that needs to happen here?
559- debug!("attempting to load method: {}", method_name);
560- let func: libloading::Symbol<unsafe extern "C" fn() -> Language> =
561- lib.get(method_name.as_bytes()).unwrap();
562- let language = func();
563- mem::forget(lib);
564- language
565- }
566-
567- pub struct Loader {
568- // typically /usr/lib
569- lib_path: String,
570- // typically /usr/share/tree-sitter
571- query_path: String,
572- dynamic_ok: bool,
573- extra_queries_path: Option<String>,
574- extra_parsers: Vec<TreeSitterParser>,
575- }
576-
577- impl Loader {
578- pub fn new(lib_path: &str, query_path: &str) -> Self {
579- Loader {
580- lib_path: lib_path.to_string(),
581- query_path: query_path.to_string(),
582- dynamic_ok: false,
583- extra_queries_path: None,
584- extra_parsers: Vec::new(),
585- }
586- }
587-
588- pub fn dynamic(mut self, value: bool) -> Self {
589- self.dynamic_ok = value;
590- self
591- }
592-
593- pub fn extra_queries(mut self, path: &str) -> Self {
594- self.extra_queries_path = Some(path.to_string());
595- self
596- }
597-
598- pub fn parsers(mut self, parsers: Vec<TreeSitterParser>) -> Self {
599- self.extra_parsers = parsers.clone();
600- self
601- }
602-
603- fn try_query_load(&self, path: &str, hint: &Hint) -> Result<(), Error> {
604- let query_base_path = Path::new(path).join(hint.0.to_string().to_lowercase());
605- if let Ok(queries) = fs::read_dir(query_base_path) {
606- for query in queries {
607- let query = query?;
608- match query.file_name().into_string().unwrap().as_str() {
609- "highlights.scm" => {
610- let highlight_scm = fs::read_to_string(query.path())?;
611- HIGHLIGHTS
612- .write()
613- .unwrap()
614- .insert(hint.clone(), highlight_scm);
615- debug!("loaded [{:?}] {}", hint, query.path().display());
616- }
617- "locals.scm" => {
618- let locals_scm = fs::read_to_string(query.path())?;
619- LOCALS.write().unwrap().insert(hint.clone(), locals_scm);
620- debug!("loaded [{:?}] {}", hint, query.path().display());
621- }
622- "injections.scm" => {
623- let injections_scm = fs::read_to_string(query.path())?;
624- INJECTIONS
625- .write()
626- .unwrap()
627- .insert(hint.clone(), injections_scm);
628- debug!("loaded [{:?}] {}", hint, query.path().display());
629- }
630- name => {
631- debug!("ignoring query file: {}", name)
632- }
633- }
634- }
635- }
636- Ok(())
637- }
638-
639- pub fn load(&self) -> Result<(), Error> {
640- let modules = fs::read_dir(Path::new(&self.lib_path))?;
641- for module in modules {
642- let fp = module?;
643- let file_name = fp.file_name().into_string().unwrap();
644- if file_name.starts_with("libtree-sitter-") && file_name.ends_with(".so") {
645- let language_name = file_name.replace("libtree-sitter-", "").replace(".so", "");
646-
647- let hint = Hint(language_name);
648- let language = unsafe { load_language(fp.path().to_str().unwrap(), &hint) };
649- LANGUAGES.write().unwrap().insert(hint.clone(), language);
650- debug!(
651- "loaded tree-sitter shared module: [{:?}] {}",
652- hint,
653- fp.path().display()
654- );
655- self.try_query_load(self.query_path.as_str(), &hint)?;
656- // override any base queries with additional queries
657- self.extra_queries_path
658- .as_ref()
659- .map(|path| self.try_query_load(path.as_str(), &hint))
660- .transpose()?;
661- }
662- }
663- for parser in &self.extra_parsers {
664- let hint = Hint(parser.language.clone());
665- let language = unsafe { load_language(&parser.shared_object, &hint) };
666- LANGUAGES.write().unwrap().insert(hint.clone(), language);
667- parser
668- .highlight_query
669- .as_ref()
670- .map(|path| {
671- let highlight_scm = fs::read_to_string(path)?;
672- HIGHLIGHTS
673- .write()
674- .unwrap()
675- .insert(hint.clone(), highlight_scm);
676- Ok::<(), Error>(())
677- })
678- .transpose()?;
679- parser
680- .locals_query
681- .as_ref()
682- .map(|path| {
683- let locals_scm = fs::read_to_string(path)?;
684- LOCALS.write().unwrap().insert(hint.clone(), locals_scm);
685- Ok::<(), Error>(())
686- })
687- .transpose()?;
688- parser
689- .injections_query
690- .as_ref()
691- .map(|path| {
692- let injections_scm = fs::read_to_string(path)?;
693- INJECTIONS
694- .write()
695- .unwrap()
696- .insert(hint.clone(), injections_scm);
697- Ok::<(), Error>(())
698- })
699- .transpose()?;
700- }
701- Ok(())
702- }
703- }
704-
705- fn render_lines(lines: Vec<&str>, show_line_numbers: bool) -> String {
706- let buf = Vec::new();
707- let mut file = Cursor::new(buf);
708- write!(&mut file, "<table class=\"code\">").unwrap();
709- for (i, line) in lines.into_iter().enumerate() {
710- if show_line_numbers {
711- write!(&mut file, "<tr><td class=line-number>{:?}</td>", i + 1).unwrap();
712- }
713- write!(&mut file, "<td class=line>{}</td></tr>", line).unwrap();
714- }
715- write!(&mut file, "</table>").unwrap();
716- let bytes = file.into_inner();
717- String::from_utf8(bytes).unwrap()
718- }
719-
720- #[derive(Clone, Debug)]
721- pub struct Highlighter {
722- names: Vec<String>,
723- classes: Vec<String>,
724- }
725-
726- impl Highlighter {
727- pub fn new(names: &[String]) -> Self {
728- let mut classes: Vec<String> = Vec::new();
729-
730- for name in names.iter() {
731- classes.push(format!("class=\"ts_{}\"", name.replace('.', "_")));
732- }
733- Self {
734- names: names.to_vec(),
735- classes,
736- }
737- }
738-
739- pub fn highlight(
740- &self,
741- code: &str,
742- filepath: Option<&str>,
743- alias: Option<&str>,
744- hint: Option<Hint>,
745- show_line_numbers: bool,
746- ) -> (Option<Hint>, String) {
747- let buf = Vec::new();
748- let mut file = Cursor::new(buf);
749- write!(&mut file, "<table>").unwrap();
750-
751- let hint = match hint {
752- Some(hint) => Some(hint),
753- None => LANGUAGE_TABLE.guess(code, alias, filepath),
754- };
755-
756- debug!("language hint: {:?}", hint);
757-
758- match hint {
759- Some(hint) => match (
760- LANGUAGES.read().unwrap().get(&hint),
761- HIGHLIGHTS.read().unwrap().get(&hint),
762- ) {
763- (Some(language), Some(syntax)) => {
764- debug!("painting syntax for language: {:?}", hint);
765- let mut highlighter = TSHighlighter::new();
766- let injections = INJECTIONS
767- .read()
768- .unwrap()
769- .get(&hint)
770- .cloned()
771- .unwrap_or_default();
772- let locals = LOCALS
773- .read()
774- .unwrap()
775- .get(&hint)
776- .cloned()
777- .unwrap_or_default();
778- let mut config =
779- HighlightConfiguration::new(*language, syntax, &injections, &locals)
780- .unwrap();
781-
782- config.configure(&self.names);
783-
784- let code = code.as_bytes();
785-
786- let events = highlighter
787- .highlight(&config, code, None, |_| None)
788- .unwrap();
789-
790- let mut renderer = HtmlRenderer::new();
791-
792- renderer
793- .render(events, code, &move |highlight| {
794- let ret = match self.classes.get(highlight.0) {
795- Some(name) => name.as_bytes(),
796- None => "".as_bytes(),
797- };
798- ret
799- })
800- .unwrap();
801-
802- (
803- Some(hint.clone()),
804- render_lines(renderer.lines().collect(), show_line_numbers),
805- )
806- }
807- _ => {
808- debug!("cannot paint syntax for language: {:?}", hint);
809- (
810- None,
811- render_lines(escape_html(code).lines().collect(), show_line_numbers),
812- )
813- }
814- },
815- None => {
816- debug!("cannot determine language");
817- (
818- None,
819- render_lines(code.to_string().lines().collect(), show_line_numbers),
820- )
821- }
822- }
823- }
824-
825- pub fn highlight_vec(
826- &self,
827- content: Vec<u8>,
828- filepath: Option<&str>,
829- alias: Option<&str>,
830- hint: Option<Hint>,
831- show_line_numbers: bool,
832- ) -> (Option<Hint>, String) {
833- let content = String::from_utf8(content).unwrap();
834- self.highlight(content.as_str(), filepath, alias, hint, show_line_numbers)
835- }
836- }
837-
838- #[derive(Debug, Clone)]
839- pub struct TreeSitterAdapter {
840- highlighter: Highlighter,
841- }
842-
843- impl TreeSitterAdapter {
844- pub fn new(highlighter: Highlighter) -> Self {
845- TreeSitterAdapter { highlighter }
846- }
847- }
848-
849- impl SyntaxHighlighterAdapter for TreeSitterAdapter {
850- fn write_highlighted(
851- &self,
852- output: &mut dyn std::io::Write,
853- lang: Option<&str>,
854- code: &str,
855- ) -> std::io::Result<()> {
856- let (_, highlighted) = self.highlighter.highlight(code, None, lang, None, false);
857- output.write_all(highlighted.as_bytes()).unwrap();
858- Ok(())
859- }
860-
861- fn write_pre_tag(
862- &self,
863- _: &mut dyn std::io::Write,
864- _: std::collections::HashMap<String, String>,
865- ) -> std::io::Result<()> {
866- Ok(())
867- }
868-
869- fn write_code_tag(
870- &self,
871- _: &mut dyn std::io::Write,
872- _: std::collections::HashMap<String, String>,
873- ) -> std::io::Result<()> {
874- Ok(())
875- }
876- }
877 diff --git a/ayllu/src/web2/mod.rs b/ayllu/src/web2/mod.rs
878index 3d0d739..6e95d71 100644
879--- a/ayllu/src/web2/mod.rs
880+++ b/ayllu/src/web2/mod.rs
881 @@ -2,7 +2,6 @@ mod charts;
882 mod config;
883 mod error;
884 mod extractors;
885- mod highlight;
886 mod middleware;
887 mod navigation;
888 mod routes;
889 diff --git a/ayllu/src/web2/routes/about.rs b/ayllu/src/web2/routes/about.rs
890index 86c3983..6860e55 100644
891--- a/ayllu/src/web2/routes/about.rs
892+++ b/ayllu/src/web2/routes/about.rs
893 @@ -3,8 +3,8 @@ use comrak::{markdown_to_html_with_plugins, ComrakOptions, ComrakPlugins};
894 use axum::{extract::Extension, response::Html};
895
896 use crate::config::Config;
897+ use crate::highlight::TreeSitterAdapter;
898 use crate::web2::error::Error;
899- use crate::web2::highlight::TreeSitterAdapter;
900 use crate::web2::middleware::template::Template;
901 use crate::web2::navigation;
902
903 diff --git a/ayllu/src/web2/routes/blame.rs b/ayllu/src/web2/routes/blame.rs
904index 4ff451b..23ed548 100644
905--- a/ayllu/src/web2/routes/blame.rs
906+++ b/ayllu/src/web2/routes/blame.rs
907 @@ -5,7 +5,7 @@ use axum::{extract::Extension, response::Html};
908 use crate::config::Config;
909 use crate::time as timeutil;
910 use crate::web2::error::Error;
911- use crate::web2::highlight::Highlighter;
912+ use crate::highlight::Highlighter;
913 use crate::web2::middleware::repository::Preamble;
914 use crate::web2::middleware::template::Template;
915 use crate::web2::navigation;
916 diff --git a/ayllu/src/web2/routes/blob.rs b/ayllu/src/web2/routes/blob.rs
917index 0f7d68b..39f6e3b 100644
918--- a/ayllu/src/web2/routes/blob.rs
919+++ b/ayllu/src/web2/routes/blob.rs
920 @@ -12,7 +12,7 @@ use axum::{
921 use crate::config::Config;
922 use crate::languages::LANGUAGE_TABLE;
923 use crate::web2::error::Error;
924- use crate::web2::highlight::{Highlighter, TreeSitterAdapter};
925+ use crate::highlight::{Highlighter, TreeSitterAdapter};
926 use crate::web2::middleware::repository::Preamble;
927 use crate::web2::middleware::template::Template;
928 use crate::web2::navigation;
929 diff --git a/ayllu/src/web2/routes/commit.rs b/ayllu/src/web2/routes/commit.rs
930index 95f730a..9204fa2 100644
931--- a/ayllu/src/web2/routes/commit.rs
932+++ b/ayllu/src/web2/routes/commit.rs
933 @@ -3,8 +3,8 @@ use axum::{
934 response::Html,
935 };
936
937+ use crate::highlight::Highlighter;
938 use crate::web2::error::Error;
939- use crate::web2::highlight::Highlighter;
940 use crate::web2::middleware::repository::Preamble;
941 use crate::web2::middleware::template::Template;
942 use crate::web2::navigation;
943 diff --git a/ayllu/src/web2/routes/repo.rs b/ayllu/src/web2/routes/repo.rs
944index b804033..b55f3cc 100644
945--- a/ayllu/src/web2/routes/repo.rs
946+++ b/ayllu/src/web2/routes/repo.rs
947 @@ -6,7 +6,6 @@ use axum::{
948 extract::{Extension, OriginalUri},
949 response::Html,
950 };
951- use comrak::{markdown_to_html_with_plugins, ComrakPlugins};
952 use serde::Serialize;
953 use tera::{to_value, Filter};
954 use time::Duration;
955 @@ -17,10 +16,11 @@ use crate::database_ext::{
956 langauges::LanguagesExt,
957 stats::{Aggregation, StatsExt},
958 };
959+ use crate::highlight::TreeSitterAdapter;
960 use crate::license;
961+ use crate::readme::Renderer;
962 use crate::web2::charts;
963 use crate::web2::error::Error;
964- use crate::web2::highlight::TreeSitterAdapter;
965 use crate::web2::middleware::repository::Preamble;
966 use crate::web2::middleware::rpc_initiator::Initiator;
967 use crate::web2::middleware::template::Template;
968 @@ -216,12 +216,18 @@ pub async fn serve(
969 .await?;
970 let chart_builder = charts::Renderer::new((320, 250));
971
972- let mut plugins = ComrakPlugins::default();
973- plugins.render.codefence_syntax_highlighter = Some(&adapter);
974-
975- let readme = readme.map_or(String::new(), |readme| {
976- markdown_to_html_with_plugins(readme.as_str(), &cfg.markdown_render_options(), &plugins)
977- });
978+ let readme = match readme {
979+ Some(readme) => {
980+ let renderer = Renderer {
981+ origin: &cfg.origin,
982+ collection: &preamble.collection_name,
983+ name: &preamble.repo_name,
984+ allow_unsafe_markdown: cfg.web.unsafe_markdown,
985+ };
986+ renderer.render(adapter, &readme)
987+ }
988+ None => String::new(),
989+ };
990
991 // Merge everything together and render
992 templates.register_filter(
993 diff --git a/ayllu/src/web2/server.rs b/ayllu/src/web2/server.rs
994index 8cfc4cd..6bd546d 100644
995--- a/ayllu/src/web2/server.rs
996+++ b/ayllu/src/web2/server.rs
997 @@ -17,7 +17,7 @@ use tracing::{Level, Span};
998
999 use crate::config::Config;
1000 use crate::languages::{Hint, LANGUAGE_TABLE};
1001- use crate::web2::highlight::{Highlighter, Loader, TreeSitterAdapter};
1002+ use crate::highlight::{Highlighter, Loader, TreeSitterAdapter};
1003 use crate::web2::middleware::error;
1004 use crate::web2::middleware::repository;
1005 use crate::web2::middleware::rpc_initiator;
1006 @@ -87,7 +87,7 @@ pub async fn serve(cfg: &Config) -> Result<(), Box<dyn Error>> {
1007 }
1008
1009 let highlighter = Highlighter::new(&keywords);
1010- let adapter = TreeSitterAdapter::new(highlighter.clone());
1011+ let adapter = TreeSitterAdapter(highlighter.clone());
1012
1013 let db = Builder::default()
1014 .url(&cfg.database.path)
1015 diff --git a/ayllu/themes/default/readme.css b/ayllu/themes/default/readme.css
1016index 99039f2..43ff353 100644
1017--- a/ayllu/themes/default/readme.css
1018+++ b/ayllu/themes/default/readme.css
1019 @@ -21,3 +21,19 @@
1020 margin-top: 1em;
1021 text-align: left;
1022 }
1023+
1024+ .readme > header {
1025+ display: flex;
1026+ align-items: center;
1027+ }
1028+
1029+ .icon-link {
1030+ display: flex;
1031+ align-items: center;
1032+ }
1033+
1034+ .icon {
1035+ width: 24px;
1036+ height: 24px;
1037+ fill: currentColor;
1038+ }