mirror of
https://github.com/RGBCube/cstree
synced 2025-07-28 09:37:44 +00:00
add NodeCache::with_interner
and make it pub
This commit is contained in:
parent
695d72e8ee
commit
de6386c883
5 changed files with 97 additions and 38 deletions
|
@ -13,10 +13,7 @@
|
||||||
//! - "+" Token(Add)
|
//! - "+" Token(Add)
|
||||||
//! - "4" Token(Number)
|
//! - "4" Token(Number)
|
||||||
|
|
||||||
use cstree::{
|
use cstree::{interning::Resolver, GreenNodeBuilder, NodeOrToken};
|
||||||
interning::{Reader, Resolver},
|
|
||||||
GreenNodeBuilder, NodeOrToken,
|
|
||||||
};
|
|
||||||
use std::iter::Peekable;
|
use std::iter::Peekable;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
|
@ -66,7 +63,7 @@ type SyntaxElement = cstree::NodeOrToken<SyntaxNode, SyntaxToken>;
|
||||||
type SyntaxElementRef<'a> = cstree::NodeOrToken<&'a SyntaxNode, &'a SyntaxToken>;
|
type SyntaxElementRef<'a> = cstree::NodeOrToken<&'a SyntaxNode, &'a SyntaxToken>;
|
||||||
|
|
||||||
struct Parser<'input, I: Iterator<Item = (SyntaxKind, &'input str)>> {
|
struct Parser<'input, I: Iterator<Item = (SyntaxKind, &'input str)>> {
|
||||||
builder: GreenNodeBuilder<'static>,
|
builder: GreenNodeBuilder<'static, 'static>,
|
||||||
iter: Peekable<I>,
|
iter: Peekable<I>,
|
||||||
}
|
}
|
||||||
impl<'input, I: Iterator<Item = (SyntaxKind, &'input str)>> Parser<'input, I> {
|
impl<'input, I: Iterator<Item = (SyntaxKind, &'input str)>> Parser<'input, I> {
|
||||||
|
|
|
@ -62,10 +62,7 @@ impl cstree::Language for Lang {
|
||||||
|
|
||||||
/// GreenNode is an immutable tree, which is cheap to change,
|
/// GreenNode is an immutable tree, which is cheap to change,
|
||||||
/// but doesn't contain offsets and parent pointers.
|
/// but doesn't contain offsets and parent pointers.
|
||||||
use cstree::{
|
use cstree::{interning::Resolver, GreenNode};
|
||||||
interning::{Reader, Resolver},
|
|
||||||
GreenNode,
|
|
||||||
};
|
|
||||||
|
|
||||||
/// You can construct GreenNodes by hand, but a builder
|
/// You can construct GreenNodes by hand, but a builder
|
||||||
/// is helpful for top-down parsers: it maintains a stack
|
/// is helpful for top-down parsers: it maintains a stack
|
||||||
|
@ -91,7 +88,7 @@ fn parse(text: &str) -> Parse<impl Resolver> {
|
||||||
/// in *reverse* order.
|
/// in *reverse* order.
|
||||||
tokens: Vec<(SyntaxKind, &'input str)>,
|
tokens: Vec<(SyntaxKind, &'input str)>,
|
||||||
/// the in-progress tree.
|
/// the in-progress tree.
|
||||||
builder: GreenNodeBuilder<'static>,
|
builder: GreenNodeBuilder<'static, 'static>,
|
||||||
/// the list of syntax errors we've accumulated
|
/// the list of syntax errors we've accumulated
|
||||||
/// so far.
|
/// so far.
|
||||||
errors: Vec<String>,
|
errors: Vec<String>,
|
||||||
|
|
|
@ -19,29 +19,42 @@ use super::{node::GreenNodeHead, token::GreenTokenData};
|
||||||
const CHILDREN_CACHE_THRESHOLD: usize = 3;
|
const CHILDREN_CACHE_THRESHOLD: usize = 3;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct NodeCache {
|
pub struct NodeCache<'i, I = Rodeo<Spur, FxBuildHasher>> {
|
||||||
nodes: FxHashMap<GreenNodeHead, GreenNode>,
|
nodes: FxHashMap<GreenNodeHead, GreenNode>,
|
||||||
tokens: FxHashMap<GreenTokenData, GreenToken>,
|
tokens: FxHashMap<GreenTokenData, GreenToken>,
|
||||||
interner: Rodeo<Spur, FxBuildHasher>,
|
interner: MaybeOwned<'i, I>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NodeCache {
|
impl NodeCache<'static, Rodeo<Spur, FxBuildHasher>> {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
nodes: FxHashMap::default(),
|
nodes: FxHashMap::default(),
|
||||||
tokens: FxHashMap::default(),
|
tokens: FxHashMap::default(),
|
||||||
interner: Rodeo::with_capacity_and_hasher(
|
interner: MaybeOwned::Owned(Rodeo::with_capacity_and_hasher(
|
||||||
// capacity values suggested by author of `lasso`
|
// capacity values suggested by author of `lasso`
|
||||||
Capacity::new(512, unsafe { NonZeroUsize::new_unchecked(4096) }),
|
Capacity::new(512, unsafe { NonZeroUsize::new_unchecked(4096) }),
|
||||||
FxBuildHasher::default(),
|
FxBuildHasher::default(),
|
||||||
),
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'i, I> NodeCache<'i, I>
|
||||||
|
where
|
||||||
|
I: Interner,
|
||||||
|
{
|
||||||
|
pub fn with_interner(interner: &'i mut I) -> Self {
|
||||||
|
Self {
|
||||||
|
nodes: FxHashMap::default(),
|
||||||
|
tokens: FxHashMap::default(),
|
||||||
|
interner: MaybeOwned::Borrowed(interner),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn node<I>(&mut self, kind: SyntaxKind, children: I) -> GreenNode
|
fn node<It>(&mut self, kind: SyntaxKind, children: It) -> GreenNode
|
||||||
where
|
where
|
||||||
I: IntoIterator<Item = GreenElement>,
|
It: IntoIterator<Item = GreenElement>,
|
||||||
I::IntoIter: ExactSizeIterator,
|
It::IntoIter: ExactSizeIterator,
|
||||||
{
|
{
|
||||||
let children = children.into_iter();
|
let children = children.into_iter();
|
||||||
|
|
||||||
|
@ -61,10 +74,10 @@ impl NodeCache {
|
||||||
|
|
||||||
/// Creates a [`GreenNode`] by looking inside the cache or inserting
|
/// Creates a [`GreenNode`] by looking inside the cache or inserting
|
||||||
/// a new node into the cache if it's a cache miss.
|
/// a new node into the cache if it's a cache miss.
|
||||||
fn get_cached_node<I>(&mut self, kind: SyntaxKind, children: I) -> GreenNode
|
fn get_cached_node<It>(&mut self, kind: SyntaxKind, children: It) -> GreenNode
|
||||||
where
|
where
|
||||||
I: IntoIterator<Item = GreenElement>,
|
It: IntoIterator<Item = GreenElement>,
|
||||||
I::IntoIter: ExactSizeIterator,
|
It::IntoIter: ExactSizeIterator,
|
||||||
{
|
{
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct ChildrenIter {
|
struct ChildrenIter {
|
||||||
|
@ -132,6 +145,15 @@ enum MaybeOwned<'a, T> {
|
||||||
Borrowed(&'a mut T),
|
Borrowed(&'a mut T),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl<T> MaybeOwned<'_, T> {
|
||||||
|
fn as_owned(self) -> Option<T> {
|
||||||
|
match self {
|
||||||
|
MaybeOwned::Owned(owned) => Some(owned),
|
||||||
|
MaybeOwned::Borrowed(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<T> std::ops::Deref for MaybeOwned<'_, T> {
|
impl<T> std::ops::Deref for MaybeOwned<'_, T> {
|
||||||
type Target = T;
|
type Target = T;
|
||||||
|
|
||||||
|
@ -164,26 +186,31 @@ pub struct Checkpoint(usize);
|
||||||
|
|
||||||
/// A builder for a green tree.
|
/// A builder for a green tree.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct GreenNodeBuilder<'cache> {
|
pub struct GreenNodeBuilder<'cache, 'interner, I = Rodeo<Spur, FxBuildHasher>> {
|
||||||
cache: MaybeOwned<'cache, NodeCache>,
|
cache: MaybeOwned<'cache, NodeCache<'interner, I>>,
|
||||||
parents: Vec<(SyntaxKind, usize)>,
|
parents: Vec<(SyntaxKind, usize)>,
|
||||||
children: Vec<GreenElement>,
|
children: Vec<GreenElement>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl GreenNodeBuilder<'_> {
|
impl GreenNodeBuilder<'static, 'static, Rodeo<Spur, FxBuildHasher>> {
|
||||||
/// Creates new builder.
|
/// Creates new builder.
|
||||||
pub fn new() -> GreenNodeBuilder<'static> {
|
pub fn new() -> Self {
|
||||||
GreenNodeBuilder {
|
Self {
|
||||||
cache: MaybeOwned::Owned(NodeCache::new()),
|
cache: MaybeOwned::Owned(NodeCache::new()),
|
||||||
parents: Vec::with_capacity(8),
|
parents: Vec::with_capacity(8),
|
||||||
children: Vec::with_capacity(8),
|
children: Vec::with_capacity(8),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'cache, 'interner, I> GreenNodeBuilder<'cache, 'interner, I>
|
||||||
|
where
|
||||||
|
I: Interner,
|
||||||
|
{
|
||||||
/// Reusing `NodeCache` between different `GreenNodeBuilder`s saves memory.
|
/// Reusing `NodeCache` between different `GreenNodeBuilder`s saves memory.
|
||||||
/// It allows to structurally share underlying trees.
|
/// It allows to structurally share underlying trees.
|
||||||
pub fn with_cache(cache: &mut NodeCache) -> GreenNodeBuilder<'_> {
|
pub fn with_cache(cache: &'cache mut NodeCache<'interner, I>) -> Self {
|
||||||
GreenNodeBuilder {
|
Self {
|
||||||
cache: MaybeOwned::Borrowed(cache),
|
cache: MaybeOwned::Borrowed(cache),
|
||||||
parents: Vec::with_capacity(8),
|
parents: Vec::with_capacity(8),
|
||||||
children: Vec::with_capacity(8),
|
children: Vec::with_capacity(8),
|
||||||
|
@ -268,15 +295,12 @@ impl GreenNodeBuilder<'_> {
|
||||||
/// `start_node_at` and `finish_node` calls
|
/// `start_node_at` and `finish_node` calls
|
||||||
/// are paired!
|
/// are paired!
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn finish(mut self) -> (GreenNode, Option<impl Interner<Spur>>) {
|
pub fn finish(mut self) -> (GreenNode, Option<I>) {
|
||||||
assert_eq!(self.children.len(), 1);
|
assert_eq!(self.children.len(), 1);
|
||||||
let resolver = match self.cache {
|
let resolver = self.cache.as_owned().and_then(|cache| cache.interner.as_owned());
|
||||||
MaybeOwned::Owned(cache) => Some(cache.interner),
|
|
||||||
MaybeOwned::Borrowed(_) => None,
|
|
||||||
};
|
|
||||||
match self.children.pop().unwrap() {
|
match self.children.pop().unwrap() {
|
||||||
NodeOrToken::Node(node) => (node, resolver),
|
NodeOrToken::Node(node) => (node, resolver),
|
||||||
NodeOrToken::Token(_) => panic!(),
|
NodeOrToken::Token(_) => panic!("called `finish` on a `GreenNodeBuilder` which only contained a token"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,7 +53,7 @@ use std::fmt;
|
||||||
pub use text_size::{TextLen, TextRange, TextSize};
|
pub use text_size::{TextLen, TextRange, TextSize};
|
||||||
|
|
||||||
pub use crate::{
|
pub use crate::{
|
||||||
green::{Checkpoint, Children, GreenNode, GreenNodeBuilder, GreenToken, SyntaxKind},
|
green::{Checkpoint, Children, GreenNode, GreenNodeBuilder, GreenToken, NodeCache, SyntaxKind},
|
||||||
syntax::{SyntaxElement, SyntaxElementChildren, SyntaxElementRef, SyntaxNode, SyntaxNodeChildren, SyntaxToken},
|
syntax::{SyntaxElement, SyntaxElementChildren, SyntaxElementRef, SyntaxNode, SyntaxNodeChildren, SyntaxToken},
|
||||||
syntax_text::SyntaxText,
|
syntax_text::SyntaxText,
|
||||||
utility_types::{Direction, NodeOrToken, TokenAtOffset, WalkEvent},
|
utility_types::{Direction, NodeOrToken, TokenAtOffset, WalkEvent},
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
mod common;
|
mod common;
|
||||||
|
|
||||||
use common::TestLang;
|
use common::TestLang;
|
||||||
use cstree::{GreenNodeBuilder, SyntaxKind, SyntaxNode, TextRange};
|
use cstree::{GreenNodeBuilder, NodeCache, SyntaxKind, SyntaxNode, TextRange};
|
||||||
use lasso::Resolver;
|
use lasso::{Interner, Resolver, Rodeo};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
enum Element<'s> {
|
enum Element<'s> {
|
||||||
|
@ -26,7 +26,21 @@ fn build_tree<D>(root: &Element<'_>) -> (SyntaxNode<TestLang, D>, impl Resolver)
|
||||||
(SyntaxNode::new_root(node), interner.unwrap())
|
(SyntaxNode::new_root(node), interner.unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_recursive(root: &Element<'_>, builder: &mut GreenNodeBuilder, mut from: u16) -> u16 {
|
fn build_tree_with_cache<'c, 'i, D, I>(root: &Element<'_>, cache: &'c mut NodeCache<'i, I>) -> SyntaxNode<TestLang, D>
|
||||||
|
where
|
||||||
|
I: Interner,
|
||||||
|
{
|
||||||
|
let mut builder = GreenNodeBuilder::with_cache(cache);
|
||||||
|
build_recursive(root, &mut builder, 0);
|
||||||
|
let (node, interner) = builder.finish();
|
||||||
|
assert!(interner.is_none());
|
||||||
|
SyntaxNode::new_root(node)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_recursive<'c, 'i, I>(root: &Element<'_>, builder: &mut GreenNodeBuilder<'c, 'i, I>, mut from: u16) -> u16
|
||||||
|
where
|
||||||
|
I: Interner,
|
||||||
|
{
|
||||||
match root {
|
match root {
|
||||||
Element::Node(children) => {
|
Element::Node(children) => {
|
||||||
builder.start_node(SyntaxKind(from));
|
builder.start_node(SyntaxKind(from));
|
||||||
|
@ -98,3 +112,30 @@ fn data() {
|
||||||
assert_eq!(node2.get_data(), None);
|
assert_eq!(node2.get_data(), None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn with_interner() {
|
||||||
|
let mut interner = Rodeo::new();
|
||||||
|
let mut cache = NodeCache::with_interner(&mut interner);
|
||||||
|
let tree = two_level_tree();
|
||||||
|
let tree = build_tree_with_cache::<(), _>(&tree, &mut cache);
|
||||||
|
let resolver = interner;
|
||||||
|
|
||||||
|
assert_eq!(tree.syntax_kind(), SyntaxKind(0));
|
||||||
|
assert_eq!(tree.kind(), SyntaxKind(0));
|
||||||
|
{
|
||||||
|
let leaf1_0 = tree.children().nth(1).unwrap().children_with_tokens().nth(0).unwrap();
|
||||||
|
let leaf1_0 = leaf1_0.into_token().unwrap();
|
||||||
|
assert_eq!(leaf1_0.syntax_kind(), SyntaxKind(5));
|
||||||
|
assert_eq!(leaf1_0.kind(), SyntaxKind(5));
|
||||||
|
assert_eq!(leaf1_0.text(&resolver), "1.0");
|
||||||
|
assert_eq!(leaf1_0.text_range(), TextRange::at(6.into(), 3.into()));
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let node2 = tree.children().nth(2).unwrap();
|
||||||
|
assert_eq!(node2.syntax_kind(), SyntaxKind(6));
|
||||||
|
assert_eq!(node2.kind(), SyntaxKind(6));
|
||||||
|
assert_eq!(node2.children_with_tokens().count(), 3);
|
||||||
|
assert_eq!(node2.text(&resolver), "2.02.12.2");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue