RGBCube/cstree
1
Fork 0
mirror of https://github.com/RGBCube/cstree synced 2025-07-27 09:07:44 +00:00
Code Issues Activity

Merge pull request #31 from domenicquirl/interning-exports

Browse source
This commit is contained in:
DQ 2021-09-17 17:29:21 +02:00 • committed by GitHub
commit 08d381612c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 464 additions and 230 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Cargo.toml
View file

@ -15,7 +15,7 @@ readme = "README.md"
debug = true
[dependencies]
lasso = { version = "0.6", features = ["inline-more"] }
lasso = { version = "0.6", features = ["inline-more", "multi-threaded"] }
text-size = "1.1.0"
fxhash = "0.2.1"
parking_lot = "0.11.2"
@ -41,8 +41,8 @@ name = "main"
harness = false
[features]
default = []
serde1 = ["serde"]
default = []
serialize = ["serde", "lasso/serialize"]
[package.metadata.docs.rs]
features = ["serde1"]
features = ["serialize"]

4
src/green.rs
View file

@ -5,6 +5,7 @@
mod builder;
mod element;
mod interner;
mod iter;
mod node;
mod token;
@ -14,7 +15,8 @@ use self::element::{GreenElement, PackedGreenElement};
pub use self::{
builder::{Checkpoint, GreenNodeBuilder, NodeCache},
interner::TokenInterner,
node::{Children, GreenNode},
iter::GreenNodeChildren,
node::GreenNode,
token::GreenToken,
};

20
src/green/builder.rs
View file

@ -356,6 +356,26 @@ where
}
}
/// Shortcut to construct a builder that uses an existing interner.
///
/// This is equivalent to using [`from_cache`](GreenNodeBuilder::from_cache) with a node cache
/// obtained from [`NodeCache::with_interner`].
#[inline]
pub fn with_interner(interner: &'interner mut I) -> Self {
let cache = NodeCache::with_interner(interner);
Self::from_cache(cache)
}
/// Shortcut to construct a builder that uses an existing interner.
///
/// This is equivalent to using [`from_cache`](GreenNodeBuilder::from_cache) with a node cache
/// obtained from [`NodeCache::from_interner`].
#[inline]
pub fn from_interner(interner: I) -> Self {
let cache = NodeCache::from_interner(interner);
Self::from_cache(cache)
}
/// Get a reference to the interner used to deduplicate source text (strings).
///
/// This is the same interner as used by the underlying [`NodeCache`].

32
src/green/interner.rs
View file

@ -1,12 +1,14 @@
use std::num::NonZeroUsize;
use crate::interning::{
Capacity, Interner, IntoReader, IntoReaderAndResolver, IntoResolver, Key, Reader, Resolver, Rodeo,
};
use fxhash::FxBuildHasher;
use lasso::{Capacity, Interner, IntoReader, IntoReaderAndResolver, IntoResolver, Reader, Resolver, Rodeo, Spur};
/// The default [`Interner`] used to deduplicate green token strings.
#[derive(Debug)]
pub struct TokenInterner {
rodeo: Rodeo<Spur, FxBuildHasher>,
rodeo: Rodeo,
}
impl TokenInterner {
@ -23,22 +25,22 @@ impl TokenInterner {
impl Resolver for TokenInterner {
#[inline]
fn resolve<'a>(&'a self, key: &Spur) -> &'a str {
fn resolve<'a>(&'a self, key: &Key) -> &'a str {
self.rodeo.resolve(key)
}
#[inline]
fn try_resolve<'a>(&'a self, key: &Spur) -> Option<&'a str> {
fn try_resolve<'a>(&'a self, key: &Key) -> Option<&'a str> {
self.rodeo.try_resolve(key)
}
#[inline]
unsafe fn resolve_unchecked<'a>(&'a self, key: &Spur) -> &'a str {
unsafe fn resolve_unchecked<'a>(&'a self, key: &Key) -> &'a str {
self.rodeo.resolve_unchecked(key)
}
#[inline]
fn contains_key(&self, key: &Spur) -> bool {
fn contains_key(&self, key: &Key) -> bool {
self.rodeo.contains_key(key)
}
@ -50,7 +52,7 @@ impl Resolver for TokenInterner {
impl Reader for TokenInterner {
#[inline]
fn get(&self, val: &str) -> Option<Spur> {
fn get(&self, val: &str) -> Option<Key> {
self.rodeo.get(val)
}
@ -61,7 +63,7 @@ impl Reader for TokenInterner {
}
impl IntoResolver for TokenInterner {
type Resolver = <Rodeo<Spur, FxBuildHasher> as IntoResolver>::Resolver;
type Resolver = <Rodeo as IntoResolver>::Resolver;
#[inline]
fn into_resolver(self) -> Self::Resolver
@ -76,34 +78,34 @@ impl IntoResolver for TokenInterner {
where
Self: 'static,
{
Rodeo::<Spur, FxBuildHasher>::into_resolver_boxed(Box::new(self.rodeo))
Rodeo::into_resolver_boxed(Box::new(self.rodeo))
}
}
impl Interner for TokenInterner {
#[inline]
fn get_or_intern(&mut self, val: &str) -> Spur {
fn get_or_intern(&mut self, val: &str) -> Key {
self.rodeo.get_or_intern(val)
}
#[inline]
fn try_get_or_intern(&mut self, val: &str) -> lasso::LassoResult<Spur> {
fn try_get_or_intern(&mut self, val: &str) -> lasso::LassoResult<Key> {
self.rodeo.try_get_or_intern(val)
}
#[inline]
fn get_or_intern_static(&mut self, val: &'static str) -> Spur {
fn get_or_intern_static(&mut self, val: &'static str) -> Key {
self.rodeo.get_or_intern_static(val)
}
#[inline]
fn try_get_or_intern_static(&mut self, val: &'static str) -> lasso::LassoResult<Spur> {
fn try_get_or_intern_static(&mut self, val: &'static str) -> lasso::LassoResult<Key> {
self.rodeo.try_get_or_intern_static(val)
}
}
impl IntoReader for TokenInterner {
type Reader = <Rodeo<Spur, FxBuildHasher> as IntoReader>::Reader;
type Reader = <Rodeo as IntoReader>::Reader;
#[inline]
fn into_reader(self) -> Self::Reader
@ -117,7 +119,7 @@ impl IntoReader for TokenInterner {
where
Self: 'static,
{
Rodeo::<Spur, FxBuildHasher>::into_reader_boxed(Box::new(self.rodeo))
Rodeo::into_reader_boxed(Box::new(self.rodeo))
}
}

92
src/green/iter.rs Normal file
View file

@ -0,0 +1,92 @@
//! Green tree iterators.
use std::{iter::FusedIterator, slice};
use super::{element::PackedGreenElement, GreenElementRef};
/// An iterator over a [`GreenNode`](crate::GreenNode)'s children.
#[derive(Debug, Clone)]
pub struct GreenNodeChildren<'a> {
pub(super) inner: slice::Iter<'a, PackedGreenElement>,
}
// NB: forward everything stable that iter::Slice specializes as of Rust 1.39.0
impl ExactSizeIterator for GreenNodeChildren<'_> {
#[inline(always)]
fn len(&self) -> usize {
self.inner.len()
}
}
impl<'a> Iterator for GreenNodeChildren<'a> {
type Item = GreenElementRef<'a>;
#[inline]
fn next(&mut self) -> Option<GreenElementRef<'a>> {
self.inner.next().map(PackedGreenElement::as_ref)
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
#[inline]
fn count(self) -> usize
where
Self: Sized,
{
self.inner.count()
}
#[inline]
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.inner.nth(n).map(PackedGreenElement::as_ref)
}
#[inline]
fn last(mut self) -> Option<Self::Item>
where
Self: Sized,
{
self.next_back()
}
#[inline]
fn fold<Acc, Fold>(self, init: Acc, mut f: Fold) -> Acc
where
Fold: FnMut(Acc, Self::Item) -> Acc,
{
let mut accum = init;
for x in self {
accum = f(accum, x);
}
accum
}
}
impl<'a> DoubleEndedIterator for GreenNodeChildren<'a> {
#[inline]
fn next_back(&mut self) -> Option<Self::Item> {
self.inner.next_back().map(PackedGreenElement::as_ref)
}
#[inline]
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.inner.nth_back(n).map(PackedGreenElement::as_ref)
}
#[inline]
fn rfold<Acc, Fold>(mut self, init: Acc, mut f: Fold) -> Acc
where
Fold: FnMut(Acc, Self::Item) -> Acc,
{
let mut accum = init;
while let Some(x) = self.next_back() {
accum = f(accum, x);
}
accum
}
}
impl FusedIterator for GreenNodeChildren<'_> {}

94
src/green/node.rs
View file

@ -1,13 +1,12 @@
use std::{
hash::{Hash, Hasher},
iter::FusedIterator,
slice,
};
use fxhash::FxHasher32;
use crate::{
green::{GreenElement, GreenElementRef, PackedGreenElement, SyntaxKind},
green::{iter::GreenNodeChildren, GreenElement, PackedGreenElement, SyntaxKind},
TextSize,
};
use triomphe::{Arc, HeaderWithLength, ThinArc};
@ -135,8 +134,8 @@ impl GreenNode {
/// Iterator over all children of this node.
#[inline]
pub fn children(&self) -> Children<'_> {
Children {
pub fn children(&self) -> GreenNodeChildren<'_> {
GreenNodeChildren {
inner: self.data.slice.iter(),
}
}
@ -156,90 +155,3 @@ impl PartialEq for GreenNode {
}
impl Eq for GreenNode {}
/// An iterator over a [`GreenNode`]'s children.
#[derive(Debug, Clone)]
pub struct Children<'a> {
inner: slice::Iter<'a, PackedGreenElement>,
}
// NB: forward everything stable that iter::Slice specializes as of Rust 1.39.0
impl ExactSizeIterator for Children<'_> {
#[inline(always)]
fn len(&self) -> usize {
self.inner.len()
}
}
impl<'a> Iterator for Children<'a> {
type Item = GreenElementRef<'a>;
#[inline]
fn next(&mut self) -> Option<GreenElementRef<'a>> {
self.inner.next().map(PackedGreenElement::as_ref)
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
#[inline]
fn count(self) -> usize
where
Self: Sized,
{
self.inner.count()
}
#[inline]
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.inner.nth(n).map(PackedGreenElement::as_ref)
}
#[inline]
fn last(mut self) -> Option<Self::Item>
where
Self: Sized,
{
self.next_back()
}
#[inline]
fn fold<Acc, Fold>(self, init: Acc, mut f: Fold) -> Acc
where
Fold: FnMut(Acc, Self::Item) -> Acc,
{
let mut accum = init;
for x in self {
accum = f(accum, x);
}
accum
}
}
impl<'a> DoubleEndedIterator for Children<'a> {
#[inline]
fn next_back(&mut self) -> Option<Self::Item> {
self.inner.next_back().map(PackedGreenElement::as_ref)
}
#[inline]
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.inner.nth_back(n).map(PackedGreenElement::as_ref)
}
#[inline]
fn rfold<Acc, Fold>(mut self, init: Acc, mut f: Fold) -> Acc
where
Fold: FnMut(Acc, Self::Item) -> Acc,
{
let mut accum = init;
while let Some(x) = self.next_back() {
accum = f(accum, x);
}
accum
}
}
impl FusedIterator for Children<'_> {}

15
src/green/token.rs
View file

@ -1,14 +1,17 @@
use std::{fmt, hash, mem::ManuallyDrop, ptr};
use crate::{green::SyntaxKind, interning::Resolver, TextSize};
use lasso::Spur;
use crate::{
green::SyntaxKind,
interning::{Key, Resolver},
TextSize,
};
use triomphe::Arc;
#[repr(align(2))] // to use 1 bit for pointer tagging. NB: this is an at-least annotation
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
pub(super) struct GreenTokenData {
pub(super) kind: SyntaxKind,
pub(super) text: Spur,
pub(super) text: Key,
pub(super) text_len: TextSize,
}
@ -70,8 +73,12 @@ impl GreenToken {
self.data().text_len
}
/// Returns the interned key of text covered by this token.
/// This key may be used for comparisons with other keys of strings interned by the same interner.
///
/// See also [`text`](GreenToken::text).
#[inline]
pub(crate) fn text_key(&self) -> Spur {
pub fn text_key(&self) -> Key {
self.data().text
}
}

47
src/interning.rs Normal file
View file

@ -0,0 +1,47 @@
//! Types and Traits for efficient String storage and deduplication.
//!
//! Interning functionality is provided by the [`lasso`](lasso) crate.
pub use fxhash::FxBuildHasher as Hasher;
pub use crate::green::TokenInterner;
/// The index type for all interners. Each key represents
pub type Key = lasso::Spur;
pub use lasso::{Interner, IntoReader, IntoReaderAndResolver, IntoResolver, Reader, Resolver};
/// A string interner that caches strings quickly with a minimal memory footprint, returning a unique key to re-access
/// it with `O(1)` times. By default, `Rodeo` uses an [`fxhash`] [`Hasher`].
pub type Rodeo<S = Hasher> = lasso::Rodeo<Key, S>;
/// Constructs a new, single-threaded interner.
///
/// If you need the interner to be multi-threaded, see [`new_threaded_interner`].
#[inline]
pub fn new_interner() -> Rodeo {
Rodeo::with_hasher(Hasher::default())
}
/// A string interner that caches strings quickly with a minimal memory footprint, returning a unique key to re-access
/// it with `O(1)` times. By default, `ThreadedRodeo` uses an [`fxhash`] [`Hasher`].
pub type ThreadedRodeo<S = Hasher> = lasso::ThreadedRodeo<Key, S>;
/// Constructs a new interner that can be used across multiple threads.
#[inline]
pub fn new_threaded_interner() -> ThreadedRodeo {
ThreadedRodeo::with_hasher(Hasher::default())
}
/// A read-only view of a [`Rodeo`] or [`ThreadedRodeo`] that allows contention-free access to interned strings, both
/// key to string resolution and string to key lookups.
///
/// The hasher is the same as the Rodeo or ThreadedRodeo that created it.
/// Can be acquired with the `into_reader` methods (see also [`IntoReader`]).
pub type RodeoReader<S = Hasher> = lasso::RodeoReader<Key, S>;
/// A read-only view of a [`Rodeo`] or [`ThreadedRodeo`] that allows contention-free access to interned strings with
/// only key to string resolution.
///
/// Can be acquired with the `into_resolver` methods (see also [`IntoResolver`]).
pub type RodeoResolver = lasso::RodeoResolver<Key>;
pub use lasso::{Capacity, Iter, LassoError, LassoErrorKind, LassoResult, MemoryLimits, Strings};

15
src/lib.rs
View file

@ -49,26 +49,23 @@
#[allow(unsafe_code)]
mod green;
#[allow(unsafe_code)]
pub mod syntax;
mod syntax;
#[cfg(feature = "serde1")]
#[cfg(feature = "serialize")]
mod serde_impls;
#[allow(missing_docs)]
mod utility_types;
/// Types and Traits for efficient String storage and deduplication.
pub mod interning {
pub use crate::green::TokenInterner;
pub use lasso::{Interner, IntoReader, IntoReaderAndResolver, IntoResolver, Reader, Resolver};
}
pub mod interning;
use std::fmt;
// Reexport types for working with strings.
pub use text_size::{TextLen, TextRange, TextSize};
#[doc(inline)]
pub use crate::syntax::*;
pub use crate::{
green::{Checkpoint, Children, GreenNode, GreenNodeBuilder, GreenToken, NodeCache, SyntaxKind},
syntax::*,
green::{Checkpoint, GreenNode, GreenNodeBuilder, GreenNodeChildren, GreenToken, NodeCache, SyntaxKind},
utility_types::{Direction, NodeOrToken, TokenAtOffset, WalkEvent},
};
pub use triomphe::Arc;

164
src/syntax/iter.rs Normal file
View file

@ -0,0 +1,164 @@
//! Red tree iterators.
use std::iter::FusedIterator;
use text_size::TextSize;
use crate::{green::GreenElementRef, GreenNodeChildren, Language, SyntaxElementRef, SyntaxNode};
#[derive(Clone, Debug)]
struct Iter<'n> {
green: GreenNodeChildren<'n>,
offset: TextSize,
index: usize,
}
impl<'n> Iter<'n> {
fn new<L: Language, D>(parent: &'n SyntaxNode<L, D>) -> Self {
let offset = parent.text_range().start();
let green: GreenNodeChildren<'_> = parent.green().children();
Iter {
green,
offset,
index: 0,
}
}
}
impl<'n> Iterator for Iter<'n> {
type Item = (GreenElementRef<'n>, usize, TextSize);
#[inline(always)]
fn next(&mut self) -> Option<Self::Item> {
self.green.next().map(|element| {
let offset = self.offset;
let index = self.index;
self.offset += element.text_len();
self.index += 1;
(element, index, offset)
})
}
#[inline(always)]
fn size_hint(&self) -> (usize, Option<usize>) {
self.green.size_hint()
}
#[inline(always)]
fn count(self) -> usize
where
Self: Sized,
{
self.green.count()
}
}
impl<'n> ExactSizeIterator for Iter<'n> {
#[inline(always)]
fn len(&self) -> usize {
self.green.len()
}
}
impl<'n> FusedIterator for Iter<'n> {}
/// An iterator over the child nodes of a [`SyntaxNode`].
#[derive(Clone, Debug)]
pub struct SyntaxNodeChildren<'n, L: Language, D: 'static = ()> {
inner: Iter<'n>,
parent: &'n SyntaxNode<L, D>,
}
impl<'n, L: Language, D> SyntaxNodeChildren<'n, L, D> {
#[inline]
pub(super) fn new(parent: &'n SyntaxNode<L, D>) -> Self {
Self {
inner: Iter::new(parent),
parent,
}
}
}
impl<'n, L: Language, D> Iterator for SyntaxNodeChildren<'n, L, D> {
type Item = &'n SyntaxNode<L, D>;
#[inline(always)]
fn next(&mut self) -> Option<Self::Item> {
for (element, index, offset) in &mut self.inner {
if let Some(&node) = element.as_node() {
return Some(self.parent.get_or_add_node(node, index, offset).as_node().unwrap());
}
}
None
}
#[inline(always)]
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
#[inline(always)]
fn count(self) -> usize
where
Self: Sized,
{
self.inner.count()
}
}
impl<'n, L: Language, D> ExactSizeIterator for SyntaxNodeChildren<'n, L, D> {
#[inline(always)]
fn len(&self) -> usize {
self.inner.len()
}
}
impl<'n, L: Language, D> FusedIterator for SyntaxNodeChildren<'n, L, D> {}
/// An iterator over the children of a [`SyntaxNode`].
#[derive(Clone, Debug)]
pub struct SyntaxElementChildren<'n, L: Language, D: 'static = ()> {
inner: Iter<'n>,
parent: &'n SyntaxNode<L, D>,
}
impl<'n, L: Language, D> SyntaxElementChildren<'n, L, D> {
#[inline]
pub(super) fn new(parent: &'n SyntaxNode<L, D>) -> Self {
Self {
inner: Iter::new(parent),
parent,
}
}
}
impl<'n, L: Language, D> Iterator for SyntaxElementChildren<'n, L, D> {
type Item = SyntaxElementRef<'n, L, D>;
#[inline(always)]
fn next(&mut self) -> Option<Self::Item> {
let parent = self.parent;
self.inner
.next()
.map(|(green, index, offset)| parent.get_or_add_element(green, index, offset))
}
#[inline(always)]
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
#[inline(always)]
fn count(self) -> usize
where
Self: Sized,
{
self.inner.count()
}
}
impl<'n, L: Language, D> ExactSizeIterator for SyntaxElementChildren<'n, L, D> {
#[inline(always)]
fn len(&self) -> usize {
self.inner.len()
}
}
impl<'n, L: Language, D> FusedIterator for SyntaxElementChildren<'n, L, D> {}

4
src/syntax/mod.rs
View file

@ -8,11 +8,13 @@
mod element;
pub use element::{SyntaxElement, SyntaxElementRef};
mod node;
pub use node::{SyntaxElementChildren, SyntaxNode, SyntaxNodeChildren};
pub use node::SyntaxNode;
mod token;
pub use token::SyntaxToken;
mod resolved;
pub use resolved::{ResolvedElement, ResolvedElementRef, ResolvedNode, ResolvedToken};
mod iter;
pub use iter::{SyntaxElementChildren, SyntaxNodeChildren};
mod text;
pub use text::SyntaxText;

103
src/syntax/node.rs
View file

@ -1,5 +1,5 @@
use super::*;
#[cfg(feature = "serde1")]
#[cfg(feature = "serialize")]
use crate::serde_impls::{SerializeWithData, SerializeWithResolver};
use crate::{
green::{GreenElementRef, SyntaxKind},
@ -507,7 +507,12 @@ impl<L: Language, D> SyntaxNode<L, D> {
}
#[inline(always)]
fn get_or_add_node(&self, node: &GreenNode, index: usize, offset: TextSize) -> SyntaxElementRef<'_, L, D> {
pub(super) fn get_or_add_node(
&self,
node: &GreenNode,
index: usize,
offset: TextSize,
) -> SyntaxElementRef<'_, L, D> {
if let Some(elem) = self.read(index) {
debug_assert_eq!(elem.text_range().start(), offset);
return elem;
@ -520,7 +525,7 @@ impl<L: Language, D> SyntaxNode<L, D> {
}
#[inline(always)]
fn get_or_add_element(
pub(super) fn get_or_add_element(
&self,
element: GreenElementRef<'_>,
index: usize,
@ -946,7 +951,7 @@ impl<L: Language, D> SyntaxNode<L, D> {
}
}
#[cfg(feature = "serde1")]
#[cfg(feature = "serialize")]
impl<L, D> SyntaxNode<L, D>
where
L: Language,
@ -973,96 +978,6 @@ where
}
}
#[derive(Clone, Debug)]
struct Iter<'n> {
green: Children<'n>,
offset: TextSize,
index: usize,
}
impl<'n> Iter<'n> {
fn new<L: Language, D>(parent: &'n SyntaxNode<L, D>) -> Self {
let offset = parent.text_range().start();
let green: Children<'_> = parent.green().children();
Iter {
green,
offset,
index: 0,
}
}
#[inline(always)]
fn next(&mut self) -> Option<(GreenElementRef, usize, TextSize)> {
self.green.next().map(|element| {
let offset = self.offset;
let index = self.index;
self.offset += element.text_len();
self.index += 1;
(element, index, offset)
})
}
}
/// An iterator over the child nodes of a [`SyntaxNode`].
#[derive(Clone, Debug)]
pub struct SyntaxNodeChildren<'n, L: Language, D: 'static = ()> {
inner: Iter<'n>,
parent: &'n SyntaxNode<L, D>,
}
impl<'n, L: Language, D> SyntaxNodeChildren<'n, L, D> {
#[inline]
fn new(parent: &'n SyntaxNode<L, D>) -> Self {
Self {
inner: Iter::new(parent),
parent,
}
}
}
impl<'n, L: Language, D> Iterator for SyntaxNodeChildren<'n, L, D> {
type Item = &'n SyntaxNode<L, D>;
#[inline(always)]
fn next(&mut self) -> Option<Self::Item> {
while let Some((element, index, offset)) = self.inner.next() {
if let Some(&node) = element.as_node() {
return Some(self.parent.get_or_add_node(node, index, offset).as_node().unwrap());
}
}
None
}
}
/// An iterator over the children of a [`SyntaxNode`].
#[derive(Clone, Debug)]
pub struct SyntaxElementChildren<'n, L: Language, D: 'static = ()> {
inner: Iter<'n>,
parent: &'n SyntaxNode<L, D>,
}
impl<'n, L: Language, D> SyntaxElementChildren<'n, L, D> {
#[inline]
fn new(parent: &'n SyntaxNode<L, D>) -> Self {
Self {
inner: Iter::new(parent),
parent,
}
}
}
impl<'n, L: Language, D> Iterator for SyntaxElementChildren<'n, L, D> {
type Item = SyntaxElementRef<'n, L, D>;
#[inline(always)]
fn next(&mut self) -> Option<Self::Item> {
let parent = self.parent;
self.inner
.next()
.map(|(green, index, offset)| parent.get_or_add_element(green, index, offset))
}
}
impl GreenNode {
#[inline(always)]
fn children_from(

2
src/syntax/resolved.rs
View file

@ -214,7 +214,7 @@ impl<L: Language, D> fmt::Display for ResolvedToken<L, D> {
}
}
#[cfg(feature = "serde1")]
#[cfg(feature = "serialize")]
impl<L, D> ResolvedNode<L, D>
where
L: Language,

81
src/syntax/token.rs
View file

@ -9,7 +9,7 @@ use lasso::Resolver;
use text_size::{TextRange, TextSize};
use super::*;
use crate::{Direction, GreenNode, GreenToken, Language, SyntaxKind};
use crate::{interning::Key, Direction, GreenNode, GreenToken, Language, SyntaxKind};
/// Syntax tree token.
#[derive(Debug)]
@ -69,6 +69,7 @@ impl<L: Language, D> SyntaxToken<L, D> {
/// Returns this token's [`Debug`](fmt::Debug) representation as a string.
///
/// To avoid allocating for every token, see [`write_debug`](SyntaxToken::write_debug).
#[inline]
pub fn debug<R>(&self, resolver: &R) -> String
where
R: Resolver + ?Sized,
@ -182,16 +183,86 @@ impl<L: Language, D> SyntaxToken<L, D> {
/// This method is different from the `PartialEq` and `Eq` implementations in that it compares
/// the text and not the token position.
/// It is more efficient than comparing the result of
/// [`resolve_text`](SyntaxToken::resolve_text) because it compares the tokens' interned string
/// keys.
/// [`resolve_text`](SyntaxToken::resolve_text) because it compares the tokens' interned
/// [`text_key`s](SyntaxToken::text_key).
/// Therefore, it also does not require a [`Resolver`].
/// **Note** that the result of the comparison may be wrong when comparing two tokens from
/// different trees that use different interners.
#[inline]
pub fn text_eq(&self, other: &Self) -> bool {
self.green().text_key() == other.green().text_key()
self.text_key() == other.text_key()
}
/// Returns the interned key of text covered by this token.
/// This key may be used for comparisons with other keys of strings interned by the same interner.
///
/// See also [`resolve_text`](SyntaxToken::resolve_text) and [`text_eq`](SyntaxToken::text_eq).
///
/// # Examples
/// If you intern strings inside of your application, e.g. inside of a compiler, you can use
/// token's text keys to cross-reference between the syntax tree and the rest of your
/// implementation by re-using the interner in both.
/// ```
/// # use cstree::*;
/// # use cstree::interning::{Hasher, Rodeo, Key, new_interner};
/// # #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
/// # #[repr(u16)]
/// # enum SyntaxKind {
/// # ROOT,
/// # INT,
/// # }
/// # #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
/// # enum Lang {}
/// # impl cstree::Language for Lang {
/// # type Kind = SyntaxKind;
/// #
/// # fn kind_from_raw(raw: cstree::SyntaxKind) -> Self::Kind {
/// # assert!(raw.0 <= SyntaxKind::INT as u16);
/// # unsafe { std::mem::transmute::<u16, SyntaxKind>(raw.0) }
/// # }
/// #
/// # fn kind_to_raw(kind: Self::Kind) -> cstree::SyntaxKind {
/// # cstree::SyntaxKind(kind as u16)
/// # }
/// # }
/// # type SyntaxNode<L> = cstree::SyntaxNode<L, ()>;
/// # const ROOT: cstree::SyntaxKind = cstree::SyntaxKind(0);
/// # const IDENT: cstree::SyntaxKind = cstree::SyntaxKind(1);
/// # fn parse(b: &mut GreenNodeBuilder<Rodeo>, s: &str) {}
/// #
/// struct TypeTable {
/// // ...
/// }
/// impl TypeTable {
/// fn type_of(&self, ident: Key) -> &str {
/// // ...
/// # ""
/// }
/// }
/// # struct State {
/// # interner: Rodeo,
/// # type_table: TypeTable,
/// # }
/// # let interner = new_interner();
/// # let state = &mut State { interner, type_table: TypeTable{} };
/// let mut builder = GreenNodeBuilder::with_interner(&mut state.interner);
/// # let input = "";
/// # builder.start_node(ROOT);
/// # builder.token(IDENT, "x");
/// # builder.finish_node();
/// let tree = parse(&mut builder, "x");
/// # let tree = SyntaxNode::<Lang>::new_root(builder.finish().0);
/// let type_table = &state.type_table;
/// let ident = tree.children_with_tokens().next().unwrap().into_token().unwrap();
/// let typ = type_table.type_of(ident.text_key());
/// ```
#[inline]
pub fn text_key(&self) -> Key {
self.green().text_key()
}
/// Returns the unterlying green tree token of this token.
#[inline]
pub fn green(&self) -> &GreenToken {
self.parent
.green()
@ -242,6 +313,7 @@ impl<L: Language, D> SyntaxToken<L, D> {
/// Returns the next token in the tree.
/// This is not necessary a direct sibling of this token, but will always be further right in the tree.
#[inline]
pub fn next_token(&self) -> Option<&SyntaxToken<L, D>> {
match self.next_sibling_or_token() {
Some(element) => element.first_token(),
@ -255,6 +327,7 @@ impl<L: Language, D> SyntaxToken<L, D> {
/// Returns the previous token in the tree.
/// This is not necessary a direct sibling of this token, but will always be further left in the tree.
#[inline]
pub fn prev_token(&self) -> Option<&SyntaxToken<L, D>> {
match self.prev_sibling_or_token() {
Some(element) => element.last_token(),

2
tests/it/main.rs
View file

@ -1,7 +1,7 @@
mod basic;
mod regressions;
mod sendsync;
#[cfg(feature = "serde1")]
#[cfg(feature = "serialize")]
mod serde;
use cstree::{GreenNode, GreenNodeBuilder, Language, NodeCache, SyntaxKind};

11
tests/it/serde.rs
View file

@ -1,12 +1,13 @@
use crate::{build_recursive, build_tree_with_cache, ResolvedNode};
use super::{Element, SyntaxNode};
use cstree::{interning::IntoResolver, GreenNodeBuilder, NodeCache, NodeOrToken};
use cstree::{
interning::{new_interner, IntoResolver},
GreenNodeBuilder, NodeCache, NodeOrToken,
};
use serde_test::Token;
use std::fmt;
type Rodeo = lasso::Rodeo<lasso::Spur, fxhash::FxBuildHasher>;
/// Macro for generating a list of `serde_test::Token`s using a simpler DSL.
macro_rules! event_tokens {
($($name:ident($($token:tt)*)),*) => {
@ -237,7 +238,7 @@ fn attach_data(node: &SyntaxNode<String>) {
#[test]
fn serialize_tree_with_data_with_resolver() {
let mut interner = Rodeo::with_hasher(Default::default());
let mut interner = new_interner();
let mut cache = NodeCache::with_interner(&mut interner);
let root = three_level_tree();
@ -255,7 +256,7 @@ fn serialize_tree_with_data_with_resolver() {
#[test]
fn serialize_tree_with_resolver() {
let mut interner = Rodeo::with_hasher(Default::default());
let mut interner = new_interner();
let mut cache = NodeCache::with_interner(&mut interner);
let root = three_level_tree();