From f711f0097302fd5fb454b446c8d4f3ee467ad6f9 Mon Sep 17 00:00:00 2001 From: Domenic Quirl Date: Thu, 11 Feb 2021 17:28:13 +0100 Subject: [PATCH] document the `green` module --- src/green.rs | 4 ++ src/green/builder.rs | 112 +++++++++++++++++++++++++++++++++++-------- src/green/element.rs | 2 +- src/green/node.rs | 15 +++--- src/green/token.rs | 22 ++++----- src/lib.rs | 2 +- src/utility_types.rs | 3 ++ 7 files changed, 120 insertions(+), 40 deletions(-) diff --git a/src/green.rs b/src/green.rs index 519fb71..ee352ac 100644 --- a/src/green.rs +++ b/src/green.rs @@ -1,3 +1,7 @@ +//! Implementation of the inner, "green" tree. +//! The [`GreenNodeBuilder`] is the main entry point to constructing [`GreenNode`]s and +//! [`GreenToken`]s. + mod builder; mod element; mod node; diff --git a/src/green/builder.rs b/src/green/builder.rs index ba5f3b0..d72a5f3 100644 --- a/src/green/builder.rs +++ b/src/green/builder.rs @@ -18,6 +18,8 @@ use super::{node::GreenNodeHead, token::GreenTokenData}; /// this node into the cache. const CHILDREN_CACHE_THRESHOLD: usize = 3; +/// A `NodeCache` deduplicates identical tokens and small nodes during tree construction. +/// You can re-use the same cache for multiple similar trees with [`GreenNodeBuilder::with_cache`]. #[derive(Debug)] pub struct NodeCache<'i, I = Rodeo> { nodes: FxHashMap, @@ -26,6 +28,27 @@ pub struct NodeCache<'i, I = Rodeo> { } impl NodeCache<'static, Rodeo> { + /// Constructs a new, empty cache. + /// + /// By default, this will also create a default interner to deduplicate source text (strings) across + /// tokens. To re-use an existing interner, see [`with_interner`](NodeCache::with_interner). + /// # Examples + /// ``` + /// # use cstree::*; + /// # const ROOT: SyntaxKind = SyntaxKind(0); + /// # const INT: SyntaxKind = SyntaxKind(1); + /// # fn parse(b: &mut GreenNodeBuilder, s: &str) {} + /// let mut cache = NodeCache::new(); + /// let mut builder = GreenNodeBuilder::with_cache(&mut cache); + /// # builder.start_node(ROOT); + /// # builder.token(INT, "42"); + /// # builder.finish_node(); + /// parse(&mut builder, "42"); + /// let (tree, _) = builder.finish(); + /// assert_eq!(tree.kind(), ROOT); + /// let int = tree.children().next().unwrap(); + /// assert_eq!(int.kind(), INT); + /// ``` pub fn new() -> Self { Self { nodes: FxHashMap::default(), @@ -49,6 +72,27 @@ impl<'i, I> NodeCache<'i, I> where I: Interner, { + /// Constructs a new, empty cache that will use the given interner to deduplicate source text + /// (strings) across tokens. + /// # Examples + /// ``` + /// # use cstree::*; + /// # use lasso::Rodeo; + /// # const ROOT: SyntaxKind = SyntaxKind(0); + /// # const INT: SyntaxKind = SyntaxKind(1); + /// # fn parse(b: &mut GreenNodeBuilder, s: &str) {} + /// let mut interner = Rodeo::new(); + /// let mut cache = NodeCache::with_interner(&mut interner); + /// let mut builder = GreenNodeBuilder::with_cache(&mut cache); + /// # builder.start_node(ROOT); + /// # builder.token(INT, "42"); + /// # builder.finish_node(); + /// parse(&mut builder, "42"); + /// let (tree, _) = builder.finish(); + /// assert_eq!(tree.kind(), ROOT); + /// let int = tree.children().next().unwrap(); + /// assert_eq!(int.kind(), INT); + /// ``` pub fn with_interner(interner: &'i mut I) -> Self { Self { nodes: FxHashMap::default(), @@ -183,11 +227,32 @@ impl Default for MaybeOwned<'_, T> { } } -/// A checkpoint for maybe wrapping a node. See `GreenNodeBuilder::checkpoint` for details. +/// A checkpoint for maybe wrapping a node. See [`GreenNodeBuilder::checkpoint`] for details. #[derive(Clone, Copy, Debug)] pub struct Checkpoint(usize); -/// A builder for a green tree. +/// A builder for green trees. +/// Construct with [`new`](GreenNodeBuilder::new) or [`with_cache`](GreenNodeBuilder::with_cache). To +/// add tree nodes, start them with [`start_node`](GreenNodeBuilder::start_node), add +/// [`token`](GreenNodeBuilder::token)s and then [`finish_node`](GreenNodeBuilder::finish_node). When +/// the whole tree is constructed, call [`finish`](GreenNodeBuilder::finish) to obtain the root. +/// +/// # Examples +/// ``` +/// # use cstree::*; +/// # const ROOT: SyntaxKind = SyntaxKind(0); +/// # const INT: SyntaxKind = SyntaxKind(1); +/// let mut builder = GreenNodeBuilder::new(); +/// builder.start_node(ROOT); +/// builder.token(INT, "42"); +/// builder.finish_node(); +/// let (tree, interner) = builder.finish(); +/// assert_eq!(tree.kind(), ROOT); +/// let int = tree.children().next().unwrap(); +/// assert_eq!(int.kind(), INT); +/// let resolver = interner.unwrap().into_resolver(); +/// assert_eq!(int.as_token().unwrap().text(&resolver), "42"); +/// ``` #[derive(Debug)] pub struct GreenNodeBuilder<'cache, 'interner, I = Rodeo> { cache: MaybeOwned<'cache, NodeCache<'interner, I>>, @@ -196,7 +261,7 @@ pub struct GreenNodeBuilder<'cache, 'interner, I = Rodeo> { } impl GreenNodeBuilder<'static, 'static, Rodeo> { - /// Creates new builder. + /// Creates new builder with an empty [`NodeCache`]. pub fn new() -> Self { Self { cache: MaybeOwned::Owned(NodeCache::new()), @@ -216,8 +281,8 @@ impl<'cache, 'interner, I> GreenNodeBuilder<'cache, 'interner, I> where I: Interner, { - /// Reusing `NodeCache` between different `GreenNodeBuilder`s saves memory. - /// It allows to structurally share underlying trees. + /// Reusing a [`NodeCache`] between multiple builders saves memory, as it allows to structurally + /// share underlying trees. pub fn with_cache(cache: &'cache mut NodeCache<'interner, I>) -> Self { Self { cache: MaybeOwned::Borrowed(cache), @@ -226,22 +291,21 @@ where } } - /// Adds new token to the current branch. + /// Add new token to the current branch. #[inline] pub fn token(&mut self, kind: SyntaxKind, text: &str) { let token = self.cache.token(kind, text); self.children.push(token.into()); } - /// Start new node and make it current. + /// Start new node of the given `kind` and make it current. #[inline] pub fn start_node(&mut self, kind: SyntaxKind) { let len = self.children.len(); self.parents.push((kind, len)); } - /// Finish current branch and restore previous - /// branch as current. + /// Finish the current branch and restore the previous branch as current. #[inline] pub fn finish_node(&mut self) { let (kind, first_child) = self.parents.pop().unwrap(); @@ -250,12 +314,13 @@ where self.children.push(node.into()); } - /// Prepare for maybe wrapping the next node. - /// The way wrapping works is that you first of all get a checkpoint, - /// then you place all tokens you want to wrap, and then *maybe* call - /// `start_node_at`. - /// Example: - /// ```rust + /// Prepare for maybe wrapping the next node with a surrounding node. + /// + /// The way wrapping works is that you first get a checkpoint, then you add nodes and tokens as + /// normal, and then you *maybe* call [`start_node_at`](GreenNodeBuilder::start_node_at). + /// + /// # Examples + /// ``` /// # use cstree::{GreenNodeBuilder, SyntaxKind}; /// # const PLUS: SyntaxKind = SyntaxKind(0); /// # const OPERATION: SyntaxKind = SyntaxKind(1); @@ -280,8 +345,8 @@ where Checkpoint(self.children.len()) } - /// Wrap the previous branch marked by `checkpoint` in a new branch and - /// make it current. + /// Wrap the previous branch marked by [`checkpoint`](GreenNodeBuilder::checkpoint) in a new + /// branch and make it current. #[inline] pub fn start_node_at(&mut self, checkpoint: Checkpoint, kind: SyntaxKind) { let Checkpoint(checkpoint) = checkpoint; @@ -300,9 +365,16 @@ where self.parents.push((kind, checkpoint)); } - /// Complete tree building. Make sure that - /// `start_node_at` and `finish_node` calls - /// are paired! + /// Complete building the tree. + /// + /// Make sure that calls to [`start_node`](GreenNodeBuilder::start_node) / + /// [`start_node_at`](GreenNodeBuilder::start_node_at) and + /// [`finish_node`](GreenNodeBuilder::finish_node) are balanced, i.e. that every started node has + /// been completed! + /// + /// If this builder was constructed with [`new`](GreenNodeBuilder::new), this method returns the + /// interner used to deduplicate source text (strings) as its second return value to allow + /// resolving tree tokens back to text and re-using the interner to build additonal trees. #[inline] pub fn finish(mut self) -> (GreenNode, Option) { assert_eq!(self.children.len(), 1); diff --git a/src/green/element.rs b/src/green/element.rs index 2ed2fca..3a62f47 100644 --- a/src/green/element.rs +++ b/src/green/element.rs @@ -1,6 +1,6 @@ use std::{fmt, hash, mem}; -// NOTE: From `thin_dst`: +// NOTE from `thin_dst`: // This MUST be size=1 such that pointer math actually advances the pointer. type ErasedPtr = *const u8; diff --git a/src/green/node.rs b/src/green/node.rs index 13f245d..46a4296 100644 --- a/src/green/node.rs +++ b/src/green/node.rs @@ -12,7 +12,7 @@ use crate::{ TextSize, }; -#[repr(align(2))] // NB: this is an at-least annotation +#[repr(align(2))] //to use 1 bit for pointer tagging. NB: this is an at-least annotation #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub(super) struct GreenNodeHead { kind: SyntaxKind, @@ -40,8 +40,8 @@ impl GreenNodeHead { } } -/// Internal node in the immutable tree. -/// It has other nodes and tokens as children. +/// Internal node in the immutable "green" tree. +/// It contains other nodes and tokens as its children. #[derive(Clone)] pub struct GreenNode { pub(super) data: ThinArc, @@ -54,7 +54,7 @@ impl std::fmt::Debug for GreenNode { } impl GreenNode { - /// Creates new Node. + /// Creates a new Node. #[inline] pub fn new(kind: SyntaxKind, children: I) -> GreenNode where @@ -103,19 +103,19 @@ impl GreenNode { } } - /// Kind of this node. + /// [`SyntaxKind`] of this node. #[inline] pub fn kind(&self) -> SyntaxKind { self.data.header.header.kind } - /// Returns the length of the text covered by this node. + /// Returns the length of text covered by this node. #[inline] pub fn text_len(&self) -> TextSize { self.data.header.header.text_len } - /// Children of this node. + /// Iterator over all children of this node. #[inline] pub fn children(&self) -> Children<'_> { Children { @@ -139,6 +139,7 @@ impl PartialEq for GreenNode { impl Eq for GreenNode {} +/// An iterator over a [`GreenNode`]'s children. #[derive(Debug, Clone)] pub struct Children<'a> { inner: slice::Iter<'a, PackedGreenElement>, diff --git a/src/green/token.rs b/src/green/token.rs index 0fda7e5..a59039e 100644 --- a/src/green/token.rs +++ b/src/green/token.rs @@ -4,15 +4,15 @@ use std::{fmt, hash, mem::ManuallyDrop, ptr}; use crate::{green::SyntaxKind, interning::Resolver, TextSize}; use lasso::Spur; -#[repr(align(2))] // NB: this is an at-least annotation +#[repr(align(2))] // to use 1 bit for pointer tagging. NB: this is an at-least annotation #[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)] -pub struct GreenTokenData { - pub kind: SyntaxKind, - pub text: Spur, - pub text_len: TextSize, +pub(super) struct GreenTokenData { + pub(super) kind: SyntaxKind, + pub(super) text: Spur, + pub(super) text_len: TextSize, } -/// Leaf node in the immutable tree. +/// Leaf node in the immutable "green" tree. pub struct GreenToken { ptr: ptr::NonNull, } @@ -39,9 +39,9 @@ impl GreenToken { unsafe { &*Self::remove_tag(self.ptr).as_ptr() } } - /// Creates new Token. + /// Creates a new Token. #[inline] - pub fn new(data: GreenTokenData) -> GreenToken { + pub(super) fn new(data: GreenTokenData) -> GreenToken { let ptr = Arc::into_raw(Arc::new(data)); let ptr = ptr::NonNull::new(ptr as *mut _).unwrap(); GreenToken { @@ -49,13 +49,13 @@ impl GreenToken { } } - /// Kind of this Token. + /// [`SyntaxKind`] of this Token. #[inline] pub fn kind(&self) -> SyntaxKind { self.data().kind } - /// Text of this Token. + /// The original source text of this Token. #[inline] pub fn text<'i, I>(&self, resolver: &'i I) -> &'i str where @@ -64,7 +64,7 @@ impl GreenToken { resolver.resolve(&self.data().text) } - /// Returns the length of the text covered by this token. + /// Returns the length of text covered by this token. #[inline] pub fn text_len(&self) -> TextSize { self.data().text_len diff --git a/src/lib.rs b/src/lib.rs index 676b641..b801b93 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,7 +47,7 @@ // missing_debug_implementations, unconditional_recursion, future_incompatible, - // missing_docs, + //missing_docs, )] #![deny(unsafe_code)] diff --git a/src/utility_types.rs b/src/utility_types.rs index c907ce0..41ea079 100644 --- a/src/utility_types.rs +++ b/src/utility_types.rs @@ -1,3 +1,6 @@ +/// Convenience type to represent tree elements which may either be a node or a token. +/// +/// Used for both red and green tree, references to elements, ... #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum NodeOrToken { Node(N),