proc_macro2/
fallback.rs

1#[cfg(wrap_proc_macro)]
2use crate::imp;
3#[cfg(span_locations)]
4use crate::location::LineColumn;
5use crate::parse::{self, Cursor};
6use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
7use crate::{Delimiter, Spacing, TokenTree};
8#[cfg(all(span_locations, not(fuzzing)))]
9use alloc::collections::BTreeMap;
10#[cfg(all(span_locations, not(fuzzing)))]
11use core::cell::RefCell;
12#[cfg(span_locations)]
13use core::cmp;
14use core::fmt::{self, Debug, Display, Write};
15use core::mem::ManuallyDrop;
16#[cfg(span_locations)]
17use core::ops::Range;
18use core::ops::RangeBounds;
19use core::ptr;
20use core::str;
21#[cfg(feature = "proc-macro")]
22use core::str::FromStr;
23use std::ffi::CStr;
24#[cfg(wrap_proc_macro)]
25use std::panic;
26#[cfg(procmacro2_semver_exempt)]
27use std::path::PathBuf;
28
29/// Force use of proc-macro2's fallback implementation of the API for now, even
30/// if the compiler's implementation is available.
31pub fn force() {
32    #[cfg(wrap_proc_macro)]
33    crate::detection::force_fallback();
34}
35
36/// Resume using the compiler's implementation of the proc macro API if it is
37/// available.
38pub fn unforce() {
39    #[cfg(wrap_proc_macro)]
40    crate::detection::unforce_fallback();
41}
42
43#[derive(Clone)]
44pub(crate) struct TokenStream {
45    inner: RcVec<TokenTree>,
46}
47
48#[derive(Debug)]
49pub(crate) struct LexError {
50    pub(crate) span: Span,
51}
52
53impl LexError {
54    pub(crate) fn span(&self) -> Span {
55        self.span
56    }
57
58    pub(crate) fn call_site() -> Self {
59        LexError {
60            span: Span::call_site(),
61        }
62    }
63}
64
65impl TokenStream {
66    pub(crate) fn new() -> Self {
67        TokenStream {
68            inner: RcVecBuilder::new().build(),
69        }
70    }
71
72    pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
73        // Create a dummy file & add it to the source map
74        let mut cursor = get_cursor(src);
75
76        // Strip a byte order mark if present
77        const BYTE_ORDER_MARK: &str = "\u{feff}";
78        if cursor.starts_with(BYTE_ORDER_MARK) {
79            cursor = cursor.advance(BYTE_ORDER_MARK.len());
80        }
81
82        parse::token_stream(cursor)
83    }
84
85    #[cfg(feature = "proc-macro")]
86    pub(crate) fn from_str_unchecked(src: &str) -> Self {
87        Self::from_str_checked(src).unwrap()
88    }
89
90    pub(crate) fn is_empty(&self) -> bool {
91        self.inner.len() == 0
92    }
93
94    fn take_inner(self) -> RcVecBuilder<TokenTree> {
95        let nodrop = ManuallyDrop::new(self);
96        unsafe { ptr::read(&nodrop.inner) }.make_owned()
97    }
98}
99
100fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
101    // https://github.com/dtolnay/proc-macro2/issues/235
102    match token {
103        TokenTree::Literal(crate::Literal {
104            #[cfg(wrap_proc_macro)]
105                inner: crate::imp::Literal::Fallback(literal),
106            #[cfg(not(wrap_proc_macro))]
107                inner: literal,
108            ..
109        }) if literal.repr.starts_with('-') => {
110            push_negative_literal(vec, literal);
111        }
112        _ => vec.push(token),
113    }
114
115    #[cold]
116    fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
117        literal.repr.remove(0);
118        let mut punct = crate::Punct::new('-', Spacing::Alone);
119        punct.set_span(crate::Span::_new_fallback(literal.span));
120        vec.push(TokenTree::Punct(punct));
121        vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal)));
122    }
123}
124
125// Nonrecursive to prevent stack overflow.
126impl Drop for TokenStream {
127    fn drop(&mut self) {
128        let mut stack = Vec::new();
129        let mut current = match self.inner.get_mut() {
130            Some(inner) => inner.take().into_iter(),
131            None => return,
132        };
133        loop {
134            while let Some(token) = current.next() {
135                let group = match token {
136                    TokenTree::Group(group) => group.inner,
137                    _ => continue,
138                };
139                #[cfg(wrap_proc_macro)]
140                let group = match group {
141                    crate::imp::Group::Fallback(group) => group,
142                    crate::imp::Group::Compiler(_) => continue,
143                };
144                let mut group = group;
145                if let Some(inner) = group.stream.inner.get_mut() {
146                    stack.push(current);
147                    current = inner.take().into_iter();
148                }
149            }
150            match stack.pop() {
151                Some(next) => current = next,
152                None => return,
153            }
154        }
155    }
156}
157
158pub(crate) struct TokenStreamBuilder {
159    inner: RcVecBuilder<TokenTree>,
160}
161
162impl TokenStreamBuilder {
163    pub(crate) fn new() -> Self {
164        TokenStreamBuilder {
165            inner: RcVecBuilder::new(),
166        }
167    }
168
169    pub(crate) fn with_capacity(cap: usize) -> Self {
170        TokenStreamBuilder {
171            inner: RcVecBuilder::with_capacity(cap),
172        }
173    }
174
175    pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
176        self.inner.push(tt);
177    }
178
179    pub(crate) fn build(self) -> TokenStream {
180        TokenStream {
181            inner: self.inner.build(),
182        }
183    }
184}
185
186#[cfg(span_locations)]
187fn get_cursor(src: &str) -> Cursor {
188    #[cfg(fuzzing)]
189    return Cursor { rest: src, off: 1 };
190
191    // Create a dummy file & add it to the source map
192    #[cfg(not(fuzzing))]
193    SOURCE_MAP.with(|sm| {
194        let mut sm = sm.borrow_mut();
195        let span = sm.add_file(src);
196        Cursor {
197            rest: src,
198            off: span.lo,
199        }
200    })
201}
202
203#[cfg(not(span_locations))]
204fn get_cursor(src: &str) -> Cursor {
205    Cursor { rest: src }
206}
207
208impl Display for LexError {
209    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
210        f.write_str("cannot parse string into token stream")
211    }
212}
213
214impl Display for TokenStream {
215    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
216        let mut joint = false;
217        for (i, tt) in self.inner.iter().enumerate() {
218            if i != 0 && !joint {
219                write!(f, " ")?;
220            }
221            joint = false;
222            match tt {
223                TokenTree::Group(tt) => Display::fmt(tt, f),
224                TokenTree::Ident(tt) => Display::fmt(tt, f),
225                TokenTree::Punct(tt) => {
226                    joint = tt.spacing() == Spacing::Joint;
227                    Display::fmt(tt, f)
228                }
229                TokenTree::Literal(tt) => Display::fmt(tt, f),
230            }?;
231        }
232
233        Ok(())
234    }
235}
236
237impl Debug for TokenStream {
238    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
239        f.write_str("TokenStream ")?;
240        f.debug_list().entries(self.clone()).finish()
241    }
242}
243
244#[cfg(feature = "proc-macro")]
245impl From<proc_macro::TokenStream> for TokenStream {
246    fn from(inner: proc_macro::TokenStream) -> Self {
247        TokenStream::from_str_unchecked(&inner.to_string())
248    }
249}
250
251#[cfg(feature = "proc-macro")]
252impl From<TokenStream> for proc_macro::TokenStream {
253    fn from(inner: TokenStream) -> Self {
254        proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
255    }
256}
257
258impl From<TokenTree> for TokenStream {
259    fn from(tree: TokenTree) -> Self {
260        let mut stream = RcVecBuilder::new();
261        push_token_from_proc_macro(stream.as_mut(), tree);
262        TokenStream {
263            inner: stream.build(),
264        }
265    }
266}
267
268impl FromIterator<TokenTree> for TokenStream {
269    fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
270        let mut stream = TokenStream::new();
271        stream.extend(tokens);
272        stream
273    }
274}
275
276impl FromIterator<TokenStream> for TokenStream {
277    fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
278        let mut v = RcVecBuilder::new();
279
280        for stream in streams {
281            v.extend(stream.take_inner());
282        }
283
284        TokenStream { inner: v.build() }
285    }
286}
287
288impl Extend<TokenTree> for TokenStream {
289    fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
290        let mut vec = self.inner.make_mut();
291        tokens
292            .into_iter()
293            .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token));
294    }
295}
296
297impl Extend<TokenStream> for TokenStream {
298    fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
299        self.inner.make_mut().extend(streams.into_iter().flatten());
300    }
301}
302
303pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
304
305impl IntoIterator for TokenStream {
306    type Item = TokenTree;
307    type IntoIter = TokenTreeIter;
308
309    fn into_iter(self) -> TokenTreeIter {
310        self.take_inner().into_iter()
311    }
312}
313
314#[cfg(procmacro2_semver_exempt)]
315#[derive(Clone, PartialEq, Eq)]
316pub(crate) struct SourceFile {
317    path: PathBuf,
318}
319
320#[cfg(procmacro2_semver_exempt)]
321impl SourceFile {
322    /// Get the path to this source file as a string.
323    pub(crate) fn path(&self) -> PathBuf {
324        self.path.clone()
325    }
326
327    pub(crate) fn is_real(&self) -> bool {
328        false
329    }
330}
331
332#[cfg(procmacro2_semver_exempt)]
333impl Debug for SourceFile {
334    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
335        f.debug_struct("SourceFile")
336            .field("path", &self.path())
337            .field("is_real", &self.is_real())
338            .finish()
339    }
340}
341
342#[cfg(all(span_locations, not(fuzzing)))]
343thread_local! {
344    static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
345        // Start with a single dummy file which all call_site() and def_site()
346        // spans reference.
347        files: vec![FileInfo {
348            source_text: String::new(),
349            span: Span { lo: 0, hi: 0 },
350            lines: vec![0],
351            char_index_to_byte_offset: BTreeMap::new(),
352        }],
353    });
354}
355
356#[cfg(span_locations)]
357pub(crate) fn invalidate_current_thread_spans() {
358    #[cfg(not(fuzzing))]
359    SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
360}
361
362#[cfg(all(span_locations, not(fuzzing)))]
363struct FileInfo {
364    source_text: String,
365    span: Span,
366    lines: Vec<usize>,
367    char_index_to_byte_offset: BTreeMap<usize, usize>,
368}
369
370#[cfg(all(span_locations, not(fuzzing)))]
371impl FileInfo {
372    fn offset_line_column(&self, offset: usize) -> LineColumn {
373        assert!(self.span_within(Span {
374            lo: offset as u32,
375            hi: offset as u32,
376        }));
377        let offset = offset - self.span.lo as usize;
378        match self.lines.binary_search(&offset) {
379            Ok(found) => LineColumn {
380                line: found + 1,
381                column: 0,
382            },
383            Err(idx) => LineColumn {
384                line: idx,
385                column: offset - self.lines[idx - 1],
386            },
387        }
388    }
389
390    fn span_within(&self, span: Span) -> bool {
391        span.lo >= self.span.lo && span.hi <= self.span.hi
392    }
393
394    fn byte_range(&mut self, span: Span) -> Range<usize> {
395        let lo_char = (span.lo - self.span.lo) as usize;
396
397        // Look up offset of the largest already-computed char index that is
398        // less than or equal to the current requested one. We resume counting
399        // chars from that point.
400        let (&last_char_index, &last_byte_offset) = self
401            .char_index_to_byte_offset
402            .range(..=lo_char)
403            .next_back()
404            .unwrap_or((&0, &0));
405
406        let lo_byte = if last_char_index == lo_char {
407            last_byte_offset
408        } else {
409            let total_byte_offset = match self.source_text[last_byte_offset..]
410                .char_indices()
411                .nth(lo_char - last_char_index)
412            {
413                Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
414                None => self.source_text.len(),
415            };
416            self.char_index_to_byte_offset
417                .insert(lo_char, total_byte_offset);
418            total_byte_offset
419        };
420
421        let trunc_lo = &self.source_text[lo_byte..];
422        let char_len = (span.hi - span.lo) as usize;
423        lo_byte..match trunc_lo.char_indices().nth(char_len) {
424            Some((offset, _ch)) => lo_byte + offset,
425            None => self.source_text.len(),
426        }
427    }
428
429    fn source_text(&mut self, span: Span) -> String {
430        let byte_range = self.byte_range(span);
431        self.source_text[byte_range].to_owned()
432    }
433}
434
435/// Computes the offsets of each line in the given source string
436/// and the total number of characters
437#[cfg(all(span_locations, not(fuzzing)))]
438fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
439    let mut lines = vec![0];
440    let mut total = 0;
441
442    for ch in s.chars() {
443        total += 1;
444        if ch == '\n' {
445            lines.push(total);
446        }
447    }
448
449    (total, lines)
450}
451
452#[cfg(all(span_locations, not(fuzzing)))]
453struct SourceMap {
454    files: Vec<FileInfo>,
455}
456
457#[cfg(all(span_locations, not(fuzzing)))]
458impl SourceMap {
459    fn next_start_pos(&self) -> u32 {
460        // Add 1 so there's always space between files.
461        //
462        // We'll always have at least 1 file, as we initialize our files list
463        // with a dummy file.
464        self.files.last().unwrap().span.hi + 1
465    }
466
467    fn add_file(&mut self, src: &str) -> Span {
468        let (len, lines) = lines_offsets(src);
469        let lo = self.next_start_pos();
470        let span = Span {
471            lo,
472            hi: lo + (len as u32),
473        };
474
475        self.files.push(FileInfo {
476            source_text: src.to_owned(),
477            span,
478            lines,
479            // Populated lazily by source_text().
480            char_index_to_byte_offset: BTreeMap::new(),
481        });
482
483        span
484    }
485
486    #[cfg(procmacro2_semver_exempt)]
487    fn filepath(&self, span: Span) -> PathBuf {
488        for (i, file) in self.files.iter().enumerate() {
489            if file.span_within(span) {
490                return PathBuf::from(if i == 0 {
491                    "<unspecified>".to_owned()
492                } else {
493                    format!("<parsed string {}>", i)
494                });
495            }
496        }
497        unreachable!("Invalid span with no related FileInfo!");
498    }
499
500    fn fileinfo(&self, span: Span) -> &FileInfo {
501        for file in &self.files {
502            if file.span_within(span) {
503                return file;
504            }
505        }
506        unreachable!("Invalid span with no related FileInfo!");
507    }
508
509    fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
510        for file in &mut self.files {
511            if file.span_within(span) {
512                return file;
513            }
514        }
515        unreachable!("Invalid span with no related FileInfo!");
516    }
517}
518
519#[derive(Clone, Copy, PartialEq, Eq)]
520pub(crate) struct Span {
521    #[cfg(span_locations)]
522    pub(crate) lo: u32,
523    #[cfg(span_locations)]
524    pub(crate) hi: u32,
525}
526
527impl Span {
528    #[cfg(not(span_locations))]
529    pub(crate) fn call_site() -> Self {
530        Span {}
531    }
532
533    #[cfg(span_locations)]
534    pub(crate) fn call_site() -> Self {
535        Span { lo: 0, hi: 0 }
536    }
537
538    pub(crate) fn mixed_site() -> Self {
539        Span::call_site()
540    }
541
542    #[cfg(procmacro2_semver_exempt)]
543    pub(crate) fn def_site() -> Self {
544        Span::call_site()
545    }
546
547    pub(crate) fn resolved_at(&self, _other: Span) -> Span {
548        // Stable spans consist only of line/column information, so
549        // `resolved_at` and `located_at` only select which span the
550        // caller wants line/column information from.
551        *self
552    }
553
554    pub(crate) fn located_at(&self, other: Span) -> Span {
555        other
556    }
557
558    #[cfg(procmacro2_semver_exempt)]
559    pub(crate) fn source_file(&self) -> SourceFile {
560        #[cfg(fuzzing)]
561        return SourceFile {
562            path: PathBuf::from("<unspecified>"),
563        };
564
565        #[cfg(not(fuzzing))]
566        SOURCE_MAP.with(|sm| {
567            let sm = sm.borrow();
568            let path = sm.filepath(*self);
569            SourceFile { path }
570        })
571    }
572
573    #[cfg(span_locations)]
574    pub(crate) fn byte_range(&self) -> Range<usize> {
575        #[cfg(fuzzing)]
576        return 0..0;
577
578        #[cfg(not(fuzzing))]
579        {
580            if self.is_call_site() {
581                0..0
582            } else {
583                SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
584            }
585        }
586    }
587
588    #[cfg(span_locations)]
589    pub(crate) fn start(&self) -> LineColumn {
590        #[cfg(fuzzing)]
591        return LineColumn { line: 0, column: 0 };
592
593        #[cfg(not(fuzzing))]
594        SOURCE_MAP.with(|sm| {
595            let sm = sm.borrow();
596            let fi = sm.fileinfo(*self);
597            fi.offset_line_column(self.lo as usize)
598        })
599    }
600
601    #[cfg(span_locations)]
602    pub(crate) fn end(&self) -> LineColumn {
603        #[cfg(fuzzing)]
604        return LineColumn { line: 0, column: 0 };
605
606        #[cfg(not(fuzzing))]
607        SOURCE_MAP.with(|sm| {
608            let sm = sm.borrow();
609            let fi = sm.fileinfo(*self);
610            fi.offset_line_column(self.hi as usize)
611        })
612    }
613
614    #[cfg(not(span_locations))]
615    pub(crate) fn join(&self, _other: Span) -> Option<Span> {
616        Some(Span {})
617    }
618
619    #[cfg(span_locations)]
620    pub(crate) fn join(&self, other: Span) -> Option<Span> {
621        #[cfg(fuzzing)]
622        return {
623            let _ = other;
624            None
625        };
626
627        #[cfg(not(fuzzing))]
628        SOURCE_MAP.with(|sm| {
629            let sm = sm.borrow();
630            // If `other` is not within the same FileInfo as us, return None.
631            if !sm.fileinfo(*self).span_within(other) {
632                return None;
633            }
634            Some(Span {
635                lo: cmp::min(self.lo, other.lo),
636                hi: cmp::max(self.hi, other.hi),
637            })
638        })
639    }
640
641    #[cfg(not(span_locations))]
642    pub(crate) fn source_text(&self) -> Option<String> {
643        None
644    }
645
646    #[cfg(span_locations)]
647    pub(crate) fn source_text(&self) -> Option<String> {
648        #[cfg(fuzzing)]
649        return None;
650
651        #[cfg(not(fuzzing))]
652        {
653            if self.is_call_site() {
654                None
655            } else {
656                Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
657            }
658        }
659    }
660
661    #[cfg(not(span_locations))]
662    pub(crate) fn first_byte(self) -> Self {
663        self
664    }
665
666    #[cfg(span_locations)]
667    pub(crate) fn first_byte(self) -> Self {
668        Span {
669            lo: self.lo,
670            hi: cmp::min(self.lo.saturating_add(1), self.hi),
671        }
672    }
673
674    #[cfg(not(span_locations))]
675    pub(crate) fn last_byte(self) -> Self {
676        self
677    }
678
679    #[cfg(span_locations)]
680    pub(crate) fn last_byte(self) -> Self {
681        Span {
682            lo: cmp::max(self.hi.saturating_sub(1), self.lo),
683            hi: self.hi,
684        }
685    }
686
687    #[cfg(span_locations)]
688    fn is_call_site(&self) -> bool {
689        self.lo == 0 && self.hi == 0
690    }
691}
692
693impl Debug for Span {
694    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
695        #[cfg(span_locations)]
696        return write!(f, "bytes({}..{})", self.lo, self.hi);
697
698        #[cfg(not(span_locations))]
699        write!(f, "Span")
700    }
701}
702
703pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
704    #[cfg(span_locations)]
705    {
706        if span.is_call_site() {
707            return;
708        }
709    }
710
711    if cfg!(span_locations) {
712        debug.field("span", &span);
713    }
714}
715
716#[derive(Clone)]
717pub(crate) struct Group {
718    delimiter: Delimiter,
719    stream: TokenStream,
720    span: Span,
721}
722
723impl Group {
724    pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
725        Group {
726            delimiter,
727            stream,
728            span: Span::call_site(),
729        }
730    }
731
732    pub(crate) fn delimiter(&self) -> Delimiter {
733        self.delimiter
734    }
735
736    pub(crate) fn stream(&self) -> TokenStream {
737        self.stream.clone()
738    }
739
740    pub(crate) fn span(&self) -> Span {
741        self.span
742    }
743
744    pub(crate) fn span_open(&self) -> Span {
745        self.span.first_byte()
746    }
747
748    pub(crate) fn span_close(&self) -> Span {
749        self.span.last_byte()
750    }
751
752    pub(crate) fn set_span(&mut self, span: Span) {
753        self.span = span;
754    }
755}
756
757impl Display for Group {
758    // We attempt to match libproc_macro's formatting.
759    // Empty parens: ()
760    // Nonempty parens: (...)
761    // Empty brackets: []
762    // Nonempty brackets: [...]
763    // Empty braces: { }
764    // Nonempty braces: { ... }
765    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
766        let (open, close) = match self.delimiter {
767            Delimiter::Parenthesis => ("(", ")"),
768            Delimiter::Brace => ("{ ", "}"),
769            Delimiter::Bracket => ("[", "]"),
770            Delimiter::None => ("", ""),
771        };
772
773        f.write_str(open)?;
774        Display::fmt(&self.stream, f)?;
775        if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
776            f.write_str(" ")?;
777        }
778        f.write_str(close)?;
779
780        Ok(())
781    }
782}
783
784impl Debug for Group {
785    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
786        let mut debug = fmt.debug_struct("Group");
787        debug.field("delimiter", &self.delimiter);
788        debug.field("stream", &self.stream);
789        debug_span_field_if_nontrivial(&mut debug, self.span);
790        debug.finish()
791    }
792}
793
794#[derive(Clone)]
795pub(crate) struct Ident {
796    sym: Box<str>,
797    span: Span,
798    raw: bool,
799}
800
801impl Ident {
802    #[track_caller]
803    pub(crate) fn new_checked(string: &str, span: Span) -> Self {
804        validate_ident(string);
805        Ident::new_unchecked(string, span)
806    }
807
808    pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
809        Ident {
810            sym: Box::from(string),
811            span,
812            raw: false,
813        }
814    }
815
816    #[track_caller]
817    pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
818        validate_ident_raw(string);
819        Ident::new_raw_unchecked(string, span)
820    }
821
822    pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
823        Ident {
824            sym: Box::from(string),
825            span,
826            raw: true,
827        }
828    }
829
830    pub(crate) fn span(&self) -> Span {
831        self.span
832    }
833
834    pub(crate) fn set_span(&mut self, span: Span) {
835        self.span = span;
836    }
837}
838
839pub(crate) fn is_ident_start(c: char) -> bool {
840    c == '_' || unicode_ident::is_xid_start(c)
841}
842
843pub(crate) fn is_ident_continue(c: char) -> bool {
844    unicode_ident::is_xid_continue(c)
845}
846
847#[track_caller]
848fn validate_ident(string: &str) {
849    if string.is_empty() {
850        panic!("Ident is not allowed to be empty; use Option<Ident>");
851    }
852
853    if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
854        panic!("Ident cannot be a number; use Literal instead");
855    }
856
857    fn ident_ok(string: &str) -> bool {
858        let mut chars = string.chars();
859        let first = chars.next().unwrap();
860        if !is_ident_start(first) {
861            return false;
862        }
863        for ch in chars {
864            if !is_ident_continue(ch) {
865                return false;
866            }
867        }
868        true
869    }
870
871    if !ident_ok(string) {
872        panic!("{:?} is not a valid Ident", string);
873    }
874}
875
876#[track_caller]
877fn validate_ident_raw(string: &str) {
878    validate_ident(string);
879
880    match string {
881        "_" | "super" | "self" | "Self" | "crate" => {
882            panic!("`r#{}` cannot be a raw identifier", string);
883        }
884        _ => {}
885    }
886}
887
888impl PartialEq for Ident {
889    fn eq(&self, other: &Ident) -> bool {
890        self.sym == other.sym && self.raw == other.raw
891    }
892}
893
894impl<T> PartialEq<T> for Ident
895where
896    T: ?Sized + AsRef<str>,
897{
898    fn eq(&self, other: &T) -> bool {
899        let other = other.as_ref();
900        if self.raw {
901            other.starts_with("r#") && *self.sym == other[2..]
902        } else {
903            *self.sym == *other
904        }
905    }
906}
907
908impl Display for Ident {
909    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
910        if self.raw {
911            f.write_str("r#")?;
912        }
913        Display::fmt(&self.sym, f)
914    }
915}
916
917#[allow(clippy::missing_fields_in_debug)]
918impl Debug for Ident {
919    // Ident(proc_macro), Ident(r#union)
920    #[cfg(not(span_locations))]
921    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
922        let mut debug = f.debug_tuple("Ident");
923        debug.field(&format_args!("{}", self));
924        debug.finish()
925    }
926
927    // Ident {
928    //     sym: proc_macro,
929    //     span: bytes(128..138)
930    // }
931    #[cfg(span_locations)]
932    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
933        let mut debug = f.debug_struct("Ident");
934        debug.field("sym", &format_args!("{}", self));
935        debug_span_field_if_nontrivial(&mut debug, self.span);
936        debug.finish()
937    }
938}
939
940#[derive(Clone)]
941pub(crate) struct Literal {
942    pub(crate) repr: String,
943    span: Span,
944}
945
946macro_rules! suffixed_numbers {
947    ($($name:ident => $kind:ident,)*) => ($(
948        pub(crate) fn $name(n: $kind) -> Literal {
949            Literal::_new(format!(concat!("{}", stringify!($kind)), n))
950        }
951    )*)
952}
953
954macro_rules! unsuffixed_numbers {
955    ($($name:ident => $kind:ident,)*) => ($(
956        pub(crate) fn $name(n: $kind) -> Literal {
957            Literal::_new(n.to_string())
958        }
959    )*)
960}
961
962impl Literal {
963    pub(crate) fn _new(repr: String) -> Self {
964        Literal {
965            repr,
966            span: Span::call_site(),
967        }
968    }
969
970    pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
971        let mut cursor = get_cursor(repr);
972        #[cfg(span_locations)]
973        let lo = cursor.off;
974
975        let negative = cursor.starts_with_char('-');
976        if negative {
977            cursor = cursor.advance(1);
978            if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
979                return Err(LexError::call_site());
980            }
981        }
982
983        if let Ok((rest, mut literal)) = parse::literal(cursor) {
984            if rest.is_empty() {
985                if negative {
986                    literal.repr.insert(0, '-');
987                }
988                literal.span = Span {
989                    #[cfg(span_locations)]
990                    lo,
991                    #[cfg(span_locations)]
992                    hi: rest.off,
993                };
994                return Ok(literal);
995            }
996        }
997        Err(LexError::call_site())
998    }
999
1000    pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
1001        Literal::_new(repr.to_owned())
1002    }
1003
1004    suffixed_numbers! {
1005        u8_suffixed => u8,
1006        u16_suffixed => u16,
1007        u32_suffixed => u32,
1008        u64_suffixed => u64,
1009        u128_suffixed => u128,
1010        usize_suffixed => usize,
1011        i8_suffixed => i8,
1012        i16_suffixed => i16,
1013        i32_suffixed => i32,
1014        i64_suffixed => i64,
1015        i128_suffixed => i128,
1016        isize_suffixed => isize,
1017
1018        f32_suffixed => f32,
1019        f64_suffixed => f64,
1020    }
1021
1022    unsuffixed_numbers! {
1023        u8_unsuffixed => u8,
1024        u16_unsuffixed => u16,
1025        u32_unsuffixed => u32,
1026        u64_unsuffixed => u64,
1027        u128_unsuffixed => u128,
1028        usize_unsuffixed => usize,
1029        i8_unsuffixed => i8,
1030        i16_unsuffixed => i16,
1031        i32_unsuffixed => i32,
1032        i64_unsuffixed => i64,
1033        i128_unsuffixed => i128,
1034        isize_unsuffixed => isize,
1035    }
1036
1037    pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1038        let mut s = f.to_string();
1039        if !s.contains('.') {
1040            s.push_str(".0");
1041        }
1042        Literal::_new(s)
1043    }
1044
1045    pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1046        let mut s = f.to_string();
1047        if !s.contains('.') {
1048            s.push_str(".0");
1049        }
1050        Literal::_new(s)
1051    }
1052
1053    pub(crate) fn string(string: &str) -> Literal {
1054        let mut repr = String::with_capacity(string.len() + 2);
1055        repr.push('"');
1056        escape_utf8(string, &mut repr);
1057        repr.push('"');
1058        Literal::_new(repr)
1059    }
1060
1061    pub(crate) fn character(ch: char) -> Literal {
1062        let mut repr = String::new();
1063        repr.push('\'');
1064        if ch == '"' {
1065            // escape_debug turns this into '\"' which is unnecessary.
1066            repr.push(ch);
1067        } else {
1068            repr.extend(ch.escape_debug());
1069        }
1070        repr.push('\'');
1071        Literal::_new(repr)
1072    }
1073
1074    pub(crate) fn byte_character(byte: u8) -> Literal {
1075        let mut repr = "b'".to_string();
1076        #[allow(clippy::match_overlapping_arm)]
1077        match byte {
1078            b'\0' => repr.push_str(r"\0"),
1079            b'\t' => repr.push_str(r"\t"),
1080            b'\n' => repr.push_str(r"\n"),
1081            b'\r' => repr.push_str(r"\r"),
1082            b'\'' => repr.push_str(r"\'"),
1083            b'\\' => repr.push_str(r"\\"),
1084            b'\x20'..=b'\x7E' => repr.push(byte as char),
1085            _ => {
1086                let _ = write!(repr, r"\x{:02X}", byte);
1087            }
1088        }
1089        repr.push('\'');
1090        Literal::_new(repr)
1091    }
1092
1093    pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1094        let mut repr = "b\"".to_string();
1095        let mut bytes = bytes.iter();
1096        while let Some(&b) = bytes.next() {
1097            #[allow(clippy::match_overlapping_arm)]
1098            match b {
1099                b'\0' => repr.push_str(match bytes.as_slice().first() {
1100                    // circumvent clippy::octal_escapes lint
1101                    Some(b'0'..=b'7') => r"\x00",
1102                    _ => r"\0",
1103                }),
1104                b'\t' => repr.push_str(r"\t"),
1105                b'\n' => repr.push_str(r"\n"),
1106                b'\r' => repr.push_str(r"\r"),
1107                b'"' => repr.push_str("\\\""),
1108                b'\\' => repr.push_str(r"\\"),
1109                b'\x20'..=b'\x7E' => repr.push(b as char),
1110                _ => {
1111                    let _ = write!(repr, r"\x{:02X}", b);
1112                }
1113            }
1114        }
1115        repr.push('"');
1116        Literal::_new(repr)
1117    }
1118
1119    pub(crate) fn c_string(string: &CStr) -> Literal {
1120        let mut repr = "c\"".to_string();
1121        let mut bytes = string.to_bytes();
1122        while !bytes.is_empty() {
1123            let (valid, invalid) = match str::from_utf8(bytes) {
1124                Ok(all_valid) => {
1125                    bytes = b"";
1126                    (all_valid, bytes)
1127                }
1128                Err(utf8_error) => {
1129                    let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1130                    let valid = str::from_utf8(valid).unwrap();
1131                    let invalid = utf8_error
1132                        .error_len()
1133                        .map_or(rest, |error_len| &rest[..error_len]);
1134                    bytes = &bytes[valid.len() + invalid.len()..];
1135                    (valid, invalid)
1136                }
1137            };
1138            escape_utf8(valid, &mut repr);
1139            for &byte in invalid {
1140                let _ = write!(repr, r"\x{:02X}", byte);
1141            }
1142        }
1143        repr.push('"');
1144        Literal::_new(repr)
1145    }
1146
1147    pub(crate) fn span(&self) -> Span {
1148        self.span
1149    }
1150
1151    pub(crate) fn set_span(&mut self, span: Span) {
1152        self.span = span;
1153    }
1154
1155    pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1156        #[cfg(not(span_locations))]
1157        {
1158            let _ = range;
1159            None
1160        }
1161
1162        #[cfg(span_locations)]
1163        {
1164            use core::ops::Bound;
1165
1166            let lo = match range.start_bound() {
1167                Bound::Included(start) => {
1168                    let start = u32::try_from(*start).ok()?;
1169                    self.span.lo.checked_add(start)?
1170                }
1171                Bound::Excluded(start) => {
1172                    let start = u32::try_from(*start).ok()?;
1173                    self.span.lo.checked_add(start)?.checked_add(1)?
1174                }
1175                Bound::Unbounded => self.span.lo,
1176            };
1177            let hi = match range.end_bound() {
1178                Bound::Included(end) => {
1179                    let end = u32::try_from(*end).ok()?;
1180                    self.span.lo.checked_add(end)?.checked_add(1)?
1181                }
1182                Bound::Excluded(end) => {
1183                    let end = u32::try_from(*end).ok()?;
1184                    self.span.lo.checked_add(end)?
1185                }
1186                Bound::Unbounded => self.span.hi,
1187            };
1188            if lo <= hi && hi <= self.span.hi {
1189                Some(Span { lo, hi })
1190            } else {
1191                None
1192            }
1193        }
1194    }
1195}
1196
1197impl Display for Literal {
1198    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1199        Display::fmt(&self.repr, f)
1200    }
1201}
1202
1203impl Debug for Literal {
1204    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1205        let mut debug = fmt.debug_struct("Literal");
1206        debug.field("lit", &format_args!("{}", self.repr));
1207        debug_span_field_if_nontrivial(&mut debug, self.span);
1208        debug.finish()
1209    }
1210}
1211
1212fn escape_utf8(string: &str, repr: &mut String) {
1213    let mut chars = string.chars();
1214    while let Some(ch) = chars.next() {
1215        if ch == '\0' {
1216            repr.push_str(
1217                if chars
1218                    .as_str()
1219                    .starts_with(|next| '0' <= next && next <= '7')
1220                {
1221                    // circumvent clippy::octal_escapes lint
1222                    r"\x00"
1223                } else {
1224                    r"\0"
1225                },
1226            );
1227        } else if ch == '\'' {
1228            // escape_debug turns this into "\'" which is unnecessary.
1229            repr.push(ch);
1230        } else {
1231            repr.extend(ch.escape_debug());
1232        }
1233    }
1234}
1235
1236#[cfg(feature = "proc-macro")]
1237pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1238    #[cfg(wrap_proc_macro)]
1239    fn valid(src: &str) -> bool;
1240
1241    #[cfg(wrap_proc_macro)]
1242    fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1243        // Validate using fallback parser, because rustc is incapable of
1244        // returning a recoverable Err for certain invalid token streams, and
1245        // will instead permanently poison the compilation.
1246        if !Self::valid(src) {
1247            return Err(imp::LexError::CompilerPanic);
1248        }
1249
1250        // Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
1251        match panic::catch_unwind(|| Self::from_str(src)) {
1252            Ok(Ok(ok)) => Ok(ok),
1253            Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1254            Err(_panic) => Err(imp::LexError::CompilerPanic),
1255        }
1256    }
1257
1258    fn from_str_unchecked(src: &str) -> Self {
1259        Self::from_str(src).unwrap()
1260    }
1261}
1262
1263#[cfg(feature = "proc-macro")]
1264impl FromStr2 for proc_macro::TokenStream {
1265    #[cfg(wrap_proc_macro)]
1266    fn valid(src: &str) -> bool {
1267        TokenStream::from_str_checked(src).is_ok()
1268    }
1269}
1270
1271#[cfg(feature = "proc-macro")]
1272impl FromStr2 for proc_macro::Literal {
1273    #[cfg(wrap_proc_macro)]
1274    fn valid(src: &str) -> bool {
1275        Literal::from_str_checked(src).is_ok()
1276    }
1277}