mirror of
https://github.com/RGBCube/cstree
synced 2025-07-27 09:07:44 +00:00
Merge pull request #18 from domenicquirl/utils
This commit is contained in:
DQ
GitHub
commit
45241f98c5
19 changed files with 215 additions and 371 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -2,3 +2,4 @@
|
|||
|
||||
target
|
||||
*checksum*
|
||||
Cargo.lock
|
||||
|
|
|
|||
339
Cargo.lock
generated
339
Cargo.lock
generated
|
|
@ -1,339 +0,0 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cstree"
|
||||
version = "0.0.2"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"fxhash",
|
||||
"lasso",
|
||||
"m_lexer",
|
||||
"parking_lot",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_test",
|
||||
"servo_arc",
|
||||
"text-size",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
|
||||
|
||||
[[package]]
|
||||
name = "lasso"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17823787ed7c3f2ce99d4865d41edd4407b2fb6d9e71d534ec69d832a3ec2df3"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "m_lexer"
|
||||
version = "0.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7e51ebf91162d585a5bae05e4779efc4a276171cb880d61dd6fab11c98467a7"
|
||||
dependencies = [
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
|
||||
|
||||
[[package]]
|
||||
name = "nodrop"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.1.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
"thread_local",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.120"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "166b2349061381baf54a58e4b13c89369feb0ef2eaa57198899e2312aac30aab"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.120"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ca2a8cb5805ce9e3b95435e3765b7b553cecc762d938d409434338386cb5775"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.61"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_test"
|
||||
version = "1.0.120"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dd7d96489b14fa2f4a89be299ac117c8023d1ead9aaee963a2dde72dad4d14b"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"nodrop",
|
||||
"serde",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.58"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "text-size"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "288cb548dbe72b652243ea797201f3d481a0609a967980fcc5b2315ea811560a"
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
edition = "2018"
|
||||
name = "cstree"
|
||||
version = "0.0.2"
|
||||
version = "0.2.0"
|
||||
authors = ["Domenic Quirl <DomenicQuirl@pm.me>", "Aleksey Kladov <aleksey.kladov@gmail.com>"]
|
||||
description = "Library for generic lossless syntax trees"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
|
@ -9,7 +9,7 @@ repository = "https://github.com/domenicquirl/cstree"
|
|||
readme = "README.md"
|
||||
|
||||
[dependencies]
|
||||
lasso = "0.4.1"
|
||||
lasso = "0.5"
|
||||
text-size = "1.0.0"
|
||||
fxhash= "0.2.1"
|
||||
servo_arc = { path = "vendor/servo_arc" }
|
||||
|
|
|
|||
|
|
@ -13,7 +13,10 @@
|
|||
//! - "+" Token(Add)
|
||||
//! - "4" Token(Number)
|
||||
|
||||
use cstree::{interning::Resolver, GreenNodeBuilder, NodeOrToken};
|
||||
use cstree::{
|
||||
interning::{IntoResolver, Resolver},
|
||||
GreenNodeBuilder, NodeOrToken,
|
||||
};
|
||||
use std::iter::Peekable;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
|
|
|
|||
|
|
@ -59,7 +59,10 @@ impl cstree::Language for Lang {
|
|||
/// offsets and parent pointers.
|
||||
/// cstree also deduplicates the actual source string in addition to the tree nodes, so we will need
|
||||
/// the Resolver to get the real text back from the interned representation.
|
||||
use cstree::{interning::Resolver, GreenNode};
|
||||
use cstree::{
|
||||
interning::{IntoResolver, Resolver},
|
||||
GreenNode,
|
||||
};
|
||||
|
||||
/// You can construct GreenNodes by hand, but a builder is helpful for top-down parsers: it maintains
|
||||
/// a stack of currently in-progress nodes.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
mod builder;
|
||||
mod element;
|
||||
mod interner;
|
||||
mod node;
|
||||
mod token;
|
||||
|
||||
|
|
@ -12,6 +13,7 @@ use self::element::{GreenElement, PackedGreenElement};
|
|||
|
||||
pub use self::{
|
||||
builder::{Checkpoint, GreenNodeBuilder, NodeCache},
|
||||
interner::TokenInterner,
|
||||
node::{Children, GreenNode},
|
||||
token::GreenToken,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
use std::{convert::TryFrom, num::NonZeroUsize};
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use fxhash::{FxBuildHasher, FxHashMap};
|
||||
use lasso::{Capacity, Rodeo, Spur};
|
||||
use fxhash::FxHashMap;
|
||||
use text_size::TextSize;
|
||||
|
||||
use crate::{
|
||||
green::{GreenElement, GreenNode, GreenToken, SyntaxKind},
|
||||
green::{interner::TokenInterner, GreenElement, GreenNode, GreenToken, SyntaxKind},
|
||||
interning::Interner,
|
||||
NodeOrToken,
|
||||
};
|
||||
|
|
@ -21,13 +20,13 @@ const CHILDREN_CACHE_THRESHOLD: usize = 3;
|
|||
/// A `NodeCache` deduplicates identical tokens and small nodes during tree construction.
|
||||
/// You can re-use the same cache for multiple similar trees with [`GreenNodeBuilder::with_cache`].
|
||||
#[derive(Debug)]
|
||||
pub struct NodeCache<'i, I = Rodeo<Spur, FxBuildHasher>> {
|
||||
pub struct NodeCache<'i, I = TokenInterner> {
|
||||
nodes: FxHashMap<GreenNodeHead, GreenNode>,
|
||||
tokens: FxHashMap<GreenTokenData, GreenToken>,
|
||||
interner: MaybeOwned<'i, I>,
|
||||
}
|
||||
|
||||
impl NodeCache<'static, Rodeo<Spur, FxBuildHasher>> {
|
||||
impl NodeCache<'static> {
|
||||
/// Constructs a new, empty cache.
|
||||
///
|
||||
/// By default, this will also create a default interner to deduplicate source text (strings) across
|
||||
|
|
@ -53,11 +52,7 @@ impl NodeCache<'static, Rodeo<Spur, FxBuildHasher>> {
|
|||
Self {
|
||||
nodes: FxHashMap::default(),
|
||||
tokens: FxHashMap::default(),
|
||||
interner: MaybeOwned::Owned(Rodeo::with_capacity_and_hasher(
|
||||
// capacity values suggested by author of `lasso`
|
||||
Capacity::new(512, unsafe { NonZeroUsize::new_unchecked(4096) }),
|
||||
FxBuildHasher::default(),
|
||||
)),
|
||||
interner: MaybeOwned::Owned(TokenInterner::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -77,7 +72,7 @@ where
|
|||
/// # Examples
|
||||
/// ```
|
||||
/// # use cstree::*;
|
||||
/// # use lasso::Rodeo;
|
||||
/// use lasso::Rodeo;
|
||||
/// # const ROOT: SyntaxKind = SyntaxKind(0);
|
||||
/// # const INT: SyntaxKind = SyntaxKind(1);
|
||||
/// # fn parse(b: &mut GreenNodeBuilder<Rodeo>, s: &str) {}
|
||||
|
|
@ -239,7 +234,7 @@ pub struct Checkpoint(usize);
|
|||
///
|
||||
/// # Examples
|
||||
/// ```
|
||||
/// # use cstree::*;
|
||||
/// # use cstree::{*, interning::IntoResolver};
|
||||
/// # const ROOT: SyntaxKind = SyntaxKind(0);
|
||||
/// # const INT: SyntaxKind = SyntaxKind(1);
|
||||
/// let mut builder = GreenNodeBuilder::new();
|
||||
|
|
@ -254,13 +249,13 @@ pub struct Checkpoint(usize);
|
|||
/// assert_eq!(int.as_token().unwrap().text(&resolver), "42");
|
||||
/// ```
|
||||
#[derive(Debug)]
|
||||
pub struct GreenNodeBuilder<'cache, 'interner, I = Rodeo<Spur, FxBuildHasher>> {
|
||||
pub struct GreenNodeBuilder<'cache, 'interner, I = TokenInterner> {
|
||||
cache: MaybeOwned<'cache, NodeCache<'interner, I>>,
|
||||
parents: Vec<(SyntaxKind, usize)>,
|
||||
children: Vec<GreenElement>,
|
||||
}
|
||||
|
||||
impl GreenNodeBuilder<'static, 'static, Rodeo<Spur, FxBuildHasher>> {
|
||||
impl GreenNodeBuilder<'static, 'static> {
|
||||
/// Creates new builder with an empty [`NodeCache`].
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ pub(super) type GreenElement = NodeOrToken<GreenNode, GreenToken>;
|
|||
pub(crate) type GreenElementRef<'a> = NodeOrToken<&'a GreenNode, &'a GreenToken>;
|
||||
|
||||
#[repr(transparent)]
|
||||
pub(super) struct PackedGreenElement {
|
||||
pub(crate) struct PackedGreenElement {
|
||||
ptr: ErasedPtr,
|
||||
}
|
||||
|
||||
|
|
@ -113,7 +113,7 @@ impl From<PackedGreenElement> for GreenElement {
|
|||
}
|
||||
|
||||
impl PackedGreenElement {
|
||||
fn is_node(&self) -> bool {
|
||||
pub(crate) fn is_node(&self) -> bool {
|
||||
self.ptr as usize & 1 == 0
|
||||
}
|
||||
|
||||
|
|
|
|||
123
src/green/interner.rs
Normal file
123
src/green/interner.rs
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
use std::num::NonZeroUsize;
|
||||
|
||||
use fxhash::FxBuildHasher;
|
||||
use lasso::{Capacity, Interner, IntoReader, IntoReaderAndResolver, IntoResolver, Reader, Resolver, Rodeo, Spur};
|
||||
|
||||
/// The default [`Interner`] used to deduplicate green token strings.
|
||||
pub struct TokenInterner {
|
||||
rodeo: Rodeo<Spur, FxBuildHasher>,
|
||||
}
|
||||
|
||||
impl TokenInterner {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
rodeo: Rodeo::with_capacity_and_hasher(
|
||||
// capacity values suggested by author of `lasso`
|
||||
Capacity::new(512, unsafe { NonZeroUsize::new_unchecked(4096) }),
|
||||
FxBuildHasher::default(),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolver for TokenInterner {
|
||||
#[inline]
|
||||
fn resolve<'a>(&'a self, key: &Spur) -> &'a str {
|
||||
self.rodeo.resolve(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn try_resolve<'a>(&'a self, key: &Spur) -> Option<&'a str> {
|
||||
self.rodeo.try_resolve(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn resolve_unchecked<'a>(&'a self, key: &Spur) -> &'a str {
|
||||
self.rodeo.resolve_unchecked(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn contains_key(&self, key: &Spur) -> bool {
|
||||
self.rodeo.contains_key(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn len(&self) -> usize {
|
||||
self.rodeo.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Reader for TokenInterner {
|
||||
#[inline]
|
||||
fn get(&self, val: &str) -> Option<Spur> {
|
||||
self.rodeo.get(val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn contains(&self, val: &str) -> bool {
|
||||
self.rodeo.contains(val)
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResolver for TokenInterner {
|
||||
type Resolver = <Rodeo<Spur, FxBuildHasher> as IntoResolver>::Resolver;
|
||||
|
||||
#[inline]
|
||||
fn into_resolver(self) -> Self::Resolver
|
||||
where
|
||||
Self: 'static,
|
||||
{
|
||||
self.rodeo.into_resolver()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_resolver_boxed(self: Box<Self>) -> Self::Resolver
|
||||
where
|
||||
Self: 'static,
|
||||
{
|
||||
Rodeo::<Spur, FxBuildHasher>::into_resolver_boxed(Box::new(self.rodeo))
|
||||
}
|
||||
}
|
||||
|
||||
impl Interner for TokenInterner {
|
||||
#[inline]
|
||||
fn get_or_intern(&mut self, val: &str) -> Spur {
|
||||
self.rodeo.get_or_intern(val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn try_get_or_intern(&mut self, val: &str) -> lasso::LassoResult<Spur> {
|
||||
self.rodeo.try_get_or_intern(val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_or_intern_static(&mut self, val: &'static str) -> Spur {
|
||||
self.rodeo.get_or_intern_static(val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn try_get_or_intern_static(&mut self, val: &'static str) -> lasso::LassoResult<Spur> {
|
||||
self.rodeo.try_get_or_intern_static(val)
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoReader for TokenInterner {
|
||||
type Reader = <Rodeo<Spur, FxBuildHasher> as IntoReader>::Reader;
|
||||
|
||||
#[inline]
|
||||
fn into_reader(self) -> Self::Reader
|
||||
where
|
||||
Self: 'static,
|
||||
{
|
||||
self.rodeo.into_reader()
|
||||
}
|
||||
|
||||
fn into_reader_boxed(self: Box<Self>) -> Self::Reader
|
||||
where
|
||||
Self: 'static,
|
||||
{
|
||||
Rodeo::<Spur, FxBuildHasher>::into_reader_boxed(Box::new(self.rodeo))
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoReaderAndResolver for TokenInterner {}
|
||||
|
|
@ -115,6 +115,11 @@ impl GreenNode {
|
|||
self.data.header.header.text_len
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn iter(&self) -> slice::Iter<'_, PackedGreenElement> {
|
||||
self.data.slice.iter()
|
||||
}
|
||||
|
||||
/// Iterator over all children of this node.
|
||||
#[inline]
|
||||
pub fn children(&self) -> Children<'_> {
|
||||
|
|
|
|||
|
|
@ -63,7 +63,8 @@ mod utility_types;
|
|||
|
||||
/// Types and Traits for efficient String storage and deduplication.
|
||||
pub mod interning {
|
||||
pub use lasso::{Interner, Reader, Resolver};
|
||||
pub use crate::green::TokenInterner;
|
||||
pub use lasso::{Interner, IntoReader, IntoReaderAndResolver, IntoResolver, Reader, Resolver};
|
||||
}
|
||||
use std::fmt;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
//! Serialization and Deserialization for syntax trees.
|
||||
|
||||
use crate::{interning::Resolver, GreenNodeBuilder, Language, NodeOrToken, SyntaxKind, SyntaxNode, WalkEvent};
|
||||
use crate::{
|
||||
interning::{IntoResolver, Resolver},
|
||||
GreenNodeBuilder, Language, NodeOrToken, SyntaxKind, SyntaxNode, WalkEvent,
|
||||
};
|
||||
use serde::{
|
||||
de::{Error, SeqAccess, Visitor},
|
||||
ser::SerializeTuple,
|
||||
|
|
|
|||
|
|
@ -443,7 +443,7 @@ impl<L: Language, D, R> SyntaxNode<L, D, R> {
|
|||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// # use cstree::*;
|
||||
/// # use cstree::{*, interning::TokenInterner};
|
||||
/// # #[allow(non_camel_case_types)]
|
||||
/// #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
/// #[repr(u16)]
|
||||
|
|
@ -468,7 +468,7 @@ impl<L: Language, D, R> SyntaxNode<L, D, R> {
|
|||
/// }
|
||||
/// # const ROOT: cstree::SyntaxKind = cstree::SyntaxKind(0);
|
||||
/// # const TOKEN: cstree::SyntaxKind = cstree::SyntaxKind(1);
|
||||
/// # type SyntaxNode<L> = cstree::SyntaxNode<L, (), lasso::Rodeo<lasso::Spur, fxhash::FxBuildHasher>>;
|
||||
/// # type SyntaxNode<L> = cstree::SyntaxNode<L, (), TokenInterner>;
|
||||
/// let mut builder = GreenNodeBuilder::new();
|
||||
/// builder.start_node(ROOT);
|
||||
/// builder.token(TOKEN, "content");
|
||||
|
|
@ -695,6 +695,20 @@ impl<L: Language, D, R> SyntaxNode<L, D, R> {
|
|||
}
|
||||
}
|
||||
|
||||
/// The number of child nodes (!) of this node.
|
||||
///
|
||||
/// If you want to also consider leafs, see [`arity_with_tokens`](SyntaxNode::arity_with_tokens).
|
||||
#[inline]
|
||||
pub fn arity(&self) -> usize {
|
||||
self.green().iter().filter(|&child| child.is_node()).count()
|
||||
}
|
||||
|
||||
/// The number of children of this node.
|
||||
#[inline]
|
||||
pub fn arity_with_tokens(&self) -> usize {
|
||||
self.data().children.len()
|
||||
}
|
||||
|
||||
/// Returns an iterator along the chain of parents of this node.
|
||||
#[inline]
|
||||
pub fn ancestors(&self) -> impl Iterator<Item = &SyntaxNode<L, D, R>> {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ use crate::{interning::Resolver, Language, SyntaxNode, SyntaxToken, TextRange, T
|
|||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// # use cstree::*;
|
||||
/// # use cstree::{*, interning::IntoResolver};
|
||||
/// # #[allow(non_camel_case_types)]
|
||||
/// # #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
/// # #[repr(u16)]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use std::fmt;
|
||||
|
||||
/// Convenience type to represent tree elements which may either be a node or a token.
|
||||
///
|
||||
/// Used for both red and green tree, references to elements, ...
|
||||
|
|
@ -53,6 +55,15 @@ impl<N: Clone, T: Clone> NodeOrToken<&N, &T> {
|
|||
}
|
||||
}
|
||||
|
||||
impl<N: fmt::Display, T: fmt::Display> fmt::Display for NodeOrToken<N, T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
NodeOrToken::Node(node) => node.fmt(f),
|
||||
NodeOrToken::Token(token) => token.fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum Direction {
|
||||
Next,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
mod common;
|
||||
|
||||
use common::{build_recursive, build_tree_with_cache, Element, SyntaxNode};
|
||||
use common::{
|
||||
build_recursive, build_tree_with_cache, Element, SyntaxElement, SyntaxElementRef, SyntaxNode, SyntaxToken,
|
||||
};
|
||||
use cstree::{GreenNodeBuilder, NodeCache, SyntaxKind, TextRange};
|
||||
use lasso::{Resolver, Rodeo};
|
||||
|
||||
|
|
@ -129,3 +131,15 @@ fn inline_resolver() {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_debug_display() {
|
||||
use std::fmt;
|
||||
fn f<T: fmt::Debug + fmt::Display>() {}
|
||||
|
||||
f::<SyntaxNode<(), lasso::Rodeo>>();
|
||||
f::<SyntaxToken<(), lasso::Rodeo>>();
|
||||
f::<SyntaxElement<(), lasso::Rodeo>>();
|
||||
f::<SyntaxElementRef<'static, (), lasso::Rodeo>>();
|
||||
f::<cstree::NodeOrToken<String, u128>>();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@ use cstree::{GreenNode, GreenNodeBuilder, Language, NodeCache, SyntaxKind};
|
|||
use lasso::Interner;
|
||||
|
||||
pub type SyntaxNode<D = (), R = ()> = cstree::SyntaxNode<TestLang, D, R>;
|
||||
pub type SyntaxToken<D = (), R = ()> = cstree::SyntaxToken<TestLang, D, R>;
|
||||
pub type SyntaxElement<D = (), R = ()> = cstree::SyntaxElement<TestLang, D, R>;
|
||||
pub type SyntaxElementRef<'a, D = (), R = ()> = cstree::SyntaxElementRef<'a, TestLang, D, R>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Element<'s> {
|
||||
|
|
|
|||
|
|
@ -7,8 +7,10 @@ use crossbeam_utils::thread::scope;
|
|||
use std::{thread, time::Duration};
|
||||
|
||||
use common::{build_recursive, Element, SyntaxNode};
|
||||
use cstree::GreenNodeBuilder;
|
||||
use lasso::Resolver;
|
||||
use cstree::{
|
||||
interning::{IntoResolver, Resolver},
|
||||
GreenNodeBuilder,
|
||||
};
|
||||
|
||||
fn build_tree<D>(root: &Element<'_>) -> SyntaxNode<D, impl Resolver> {
|
||||
let mut builder = GreenNodeBuilder::new();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
#![cfg(feature = "serde1")]
|
||||
|
||||
#[allow(unused)]
|
||||
mod common;
|
||||
|
||||
use common::{Element, SyntaxNode};
|
||||
use cstree::{GreenNodeBuilder, NodeCache, NodeOrToken};
|
||||
use lasso::Resolver;
|
||||
use cstree::{
|
||||
interning::{IntoResolver, Resolver},
|
||||
GreenNodeBuilder, NodeCache, NodeOrToken,
|
||||
};
|
||||
use serde_test::Token;
|
||||
use std::fmt;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue