Cleanup split between lexer and sliding-text-window

src/Compilers/CSharp/Portable/Parser/AbstractLexer.cs

@@ -4,8 +4,6 @@
 
 using System;
 using System.Collections.Generic;
-using Microsoft.CodeAnalysis.CSharp.Symbols;
-using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Microsoft.CodeAnalysis.Text;
 
 namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax
@@ -15,6 +13,7 @@ internal class AbstractLexer : IDisposable
     {
         internal readonly SlidingTextWindow TextWindow;
         private List<SyntaxDiagnosticInfo>? _errors;
+        protected int LexemeStartPosition;
 
         protected AbstractLexer(SourceText text)
         {
@@ -28,7 +27,7 @@ public virtual void Dispose()
 
         protected void Start()
         {
-            TextWindow.Start();
+            LexemeStartPosition = this.TextWindow.Position;
             _errors = null;
         }
 
@@ -131,9 +130,18 @@ protected XmlSyntaxDiagnosticInfo MakeError(int position, int width, XmlParseErr
 
         private int GetLexemeOffsetFromPosition(int position)
         {
-            return position >= TextWindow.LexemeStartPosition ? position - TextWindow.LexemeStartPosition : position;
+            return position >= LexemeStartPosition ? position - LexemeStartPosition : position;
         }
 
+        protected string GetNonInternedLexemeText()
+            => TextWindow.GetText(LexemeStartPosition, intern: false);
+
+        protected string GetInternedLexemeText()
+            => TextWindow.GetText(LexemeStartPosition, intern: true);
+
+        protected int CurrentLexemeWidth
+            => this.TextWindow.Position - LexemeStartPosition;
+
         protected static SyntaxDiagnosticInfo MakeError(ErrorCode code)
         {
             return new SyntaxDiagnosticInfo(code);

src/Compilers/CSharp/Portable/Parser/LexerCache.cs

@@ -5,9 +5,11 @@
 // #define COLLECT_STATS
 
 using System;
+using System.Linq;
 using Microsoft.CodeAnalysis.PooledObjects;
 using Microsoft.CodeAnalysis.Syntax.InternalSyntax;
 using Roslyn.Utilities;
+using static Microsoft.CodeAnalysis.CSharp.NullableWalker;
 
 namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax
 {
@@ -200,6 +202,35 @@ internal SyntaxTrivia LookupTrivia<TArg>(
             return value;
         }
 
+
+        internal SyntaxTrivia LookupWhitespaceTrivia(
+            SlidingTextWindow textWindow,
+            int lexemeStartPosition,
+            int hashCode)
+        {
+            var lexemeWidth = textWindow.Position - lexemeStartPosition;
+            var textBuffer = textWindow.CharacterWindow;
+
+            // If the whitespace is entirely within the character window, grab from that and cache.
+            var lexemeEndPosition = lexemeStartPosition + lexemeWidth;
+            if (lexemeStartPosition >= textWindow.CharacterWindowStartPositionInText && lexemeEndPosition <= textWindow.CharacterWindowEndPositionInText)
+            {
+                var keyStart = lexemeStartPosition - textWindow.CharacterWindowStartPositionInText;
+                var value = TriviaMap.FindItem(textBuffer, keyStart, lexemeWidth, hashCode);
+
+                if (value == null)
+                {
+                    value = SyntaxFactory.Whitespace(textWindow.GetText(lexemeStartPosition, intern: true));
+                    TriviaMap.AddItem(textBuffer, keyStart, lexemeWidth, hashCode, value);
+                }
+
+                return value;
+            }
+
+            // Otherwise, if it's outside of the window, just grab from the underlying text.
+            return SyntaxFactory.Whitespace(textWindow.GetText(lexemeStartPosition, intern: true));
+        }
+
         // TODO: remove this when done tweaking this cache.
 #if COLLECT_STATS
         private static int hits = 0;

src/Compilers/CSharp/Portable/Parser/Lexer_RawStringLiteral.cs

@@ -87,7 +87,7 @@ private void ScanRawStringLiteral(ref TokenInfo info, bool inDirective)
             // trusting the contents.
             if (this.HasErrors)
             {
-                var afterStartDelimiter = TextWindow.LexemeStartPosition + startingQuoteCount;
+                var afterStartDelimiter = this.LexemeStartPosition + startingQuoteCount;
                 var valueLength = TextWindow.Position - afterStartDelimiter;
 
                 info.StringValue = TextWindow.GetText(
@@ -120,7 +120,7 @@ private void ScanRawStringLiteral(ref TokenInfo info, bool inDirective)
                 };
             }
 
-            info.Text = TextWindow.GetText(intern: true);
+            info.Text = this.GetInternedLexemeText();
         }
 
         private void ScanSingleLineRawStringLiteral(ref TokenInfo info, int startingQuoteCount)
@@ -169,7 +169,7 @@ private void ScanSingleLineRawStringLiteral(ref TokenInfo info, int startingQuot
                 }
 
                 // We have enough quotes to finish this string at this point.
-                var afterStartDelimiter = TextWindow.LexemeStartPosition + startingQuoteCount;
+                var afterStartDelimiter = this.LexemeStartPosition + startingQuoteCount;
                 var valueLength = beforeEndDelimiter - afterStartDelimiter;
 
                 info.StringValue = TextWindow.GetText(

src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs

@@ -60,7 +60,7 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective)
                     //String and character literals can contain any Unicode character. They are not limited
                     //to valid UTF-16 characters. So if we get the SlidingTextWindow's sentinel value,
                     //double check that it was not real user-code contents. This will be rare.
-                    Debug.Assert(TextWindow.Width > 0);
+                    Debug.Assert(this.CurrentLexemeWidth > 0);
                     this.AddError(ErrorCode.ERR_NewlineInConst);
                     break;
                 }
@@ -73,7 +73,7 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective)
 
             if (quoteCharacter == '\'')
             {
-                info.Text = TextWindow.GetText(intern: true);
+                info.Text = this.GetInternedLexemeText();
                 info.Kind = SyntaxKind.CharacterLiteralToken;
                 if (_builder.Length != 1)
                 {
@@ -102,7 +102,7 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective)
                     info.Kind = SyntaxKind.StringLiteralToken;
                 }
 
-                info.Text = TextWindow.GetText(intern: true);
+                info.Text = this.GetInternedLexemeText();
 
                 if (_builder.Length > 0)
                 {
@@ -247,7 +247,7 @@ private void ScanVerbatimStringLiteral(ref TokenInfo info)
                 info.Kind = SyntaxKind.StringLiteralToken;
             }
 
-            info.Text = TextWindow.GetText(intern: false);
+            info.Text = this.GetNonInternedLexemeText();
             info.StringValue = _builder.ToString();
         }
 
@@ -290,7 +290,7 @@ internal void ScanInterpolatedStringLiteralTop(
             subScanner.ScanInterpolatedStringLiteralTop(out kind, out openQuoteRange, interpolations, out closeQuoteRange);
             error = subScanner.Error;
             info.Kind = SyntaxKind.InterpolatedStringToken;
-            info.Text = TextWindow.GetText(intern: false);
+            info.Text = this.GetNonInternedLexemeText();
         }
 
         /// <summary>

src/Compilers/CSharp/Portable/Parser/QuickScanner.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Linq;
 using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.CSharp.Syntax.InternalSyntax
@@ -193,19 +194,22 @@ private enum CharFlags : byte
         {
             this.Start();
             var state = QuickScanState.Initial;
-            int i = TextWindow.Offset;
-            int n = TextWindow.CharacterWindowCount;
-            n = Math.Min(n, i + MaxCachedTokenSize);
+
+            var charWindow = TextWindow.CharacterWindow;
+            var startIndex = TextWindow.Position - TextWindow.CharacterWindowStartPositionInText;
+            var currentIndex = startIndex;
+            var n = charWindow.Length;
+
+            n = Math.Min(n, currentIndex + MaxCachedTokenSize);
 
             int hashCode = Hash.FnvOffsetBias;
 
             //localize frequently accessed fields
-            var charWindow = TextWindow.CharacterWindow;
             var charPropLength = CharProperties.Length;
 
-            for (; i < n; i++)
+            for (; currentIndex < n; currentIndex++)
             {
-                char c = charWindow[i];
+                char c = charWindow[currentIndex];
                 int uc = unchecked((int)c);
 
                 var flags = uc < charPropLength ? (CharFlags)CharProperties[uc] : CharFlags.Complex;
@@ -228,34 +232,36 @@ private enum CharFlags : byte
             state = QuickScanState.Bad; // ran out of characters in window
 exitWhile:
 
-            TextWindow.AdvanceChar(i - TextWindow.Offset);
+// Note: it is fine to change this position here and then read from charWindow below.  AdvanceChar guarantees
+// that it will not actually mutate that state.
+            TextWindow.AdvanceChar(currentIndex - TextWindow.Offset);
             Debug.Assert(state == QuickScanState.Bad || state == QuickScanState.Done, "can only exit with Bad or Done");
 
             if (state == QuickScanState.Done)
             {
                 // this is a good token!
                 var token = _cache.LookupToken(
-                    TextWindow.CharacterWindow,
-                    TextWindow.LexemeRelativeStart,
-                    i - TextWindow.LexemeRelativeStart,
+                    charWindow,
+                    keyStart: startIndex,
+                    keyLength: currentIndex - startIndex,
                     hashCode,
                     CreateQuickToken,
                     this);
                 return token;
             }
             else
             {
-                TextWindow.Reset(TextWindow.LexemeStartPosition);
+                TextWindow.Reset(this.LexemeStartPosition);
                 return null;
             }
         }
 
         private static SyntaxToken CreateQuickToken(Lexer lexer)
         {
 #if DEBUG
-            var quickWidth = lexer.TextWindow.Width;
+            var quickWidth = lexer.CurrentLexemeWidth;
 #endif
-            lexer.TextWindow.Reset(lexer.TextWindow.LexemeStartPosition);
+            lexer.TextWindow.Reset(lexer.LexemeStartPosition);
             var token = lexer.LexSyntaxToken();
 #if DEBUG
             Debug.Assert(quickWidth == token.FullWidth);

src/Compilers/CSharp/Portable/Parser/SlidingTextWindow.cs

@@ -43,8 +43,6 @@ internal sealed class SlidingTextWindow : IDisposable
         private char[] _characterWindow;                   // Moveable window of chars from source text
         private int _characterWindowCount;                 // # of valid characters in chars buffer
 
-        private int _lexemeStart;                          // Start of current lexeme relative to the window start.
-
         // Example for the above variables:
         // The text starts at 0.
         // The window onto the text starts at basis.
@@ -64,7 +62,6 @@ public SlidingTextWindow(SourceText text)
             _textEnd = text.Length;
             _strings = StringTable.GetInstance();
             _characterWindow = s_windowPool.Allocate();
-            _lexemeStart = 0;
         }
 
         public void Dispose()
@@ -112,17 +109,6 @@ public char[] CharacterWindow
             }
         }
 
-        /// <summary>
-        /// Returns the start of the current lexeme relative to the window start.
-        /// </summary>
-        public int LexemeRelativeStart
-        {
-            get
-            {
-                return _lexemeStart;
-            }
-        }
-
         /// <summary>
         /// Number of characters in the character window.
         /// </summary>
@@ -135,35 +121,19 @@ public int CharacterWindowCount
         }
 
         /// <summary>
-        /// The absolute position of the start of the current lexeme in the given
-        /// SourceText.
-        /// </summary>
-        public int LexemeStartPosition
-        {
-            get
-            {
-                return _basis + _lexemeStart;
-            }
-        }
-
-        /// <summary>
-        /// The number of characters in the current lexeme.
+        /// Offset of the <see cref="_characterWindow"/> within <see cref="_text"/>.  In other words, if this is 2048, then that means
+        /// it represents the chunk of characters starting at position 2048 in the source text.  <see cref="CharacterWindowCount"/> represents
+        /// how large the chunk is.  Characters <c>[0, CharacterWindowCount)</c> are valid characters within the window, and represents
+        /// the chunk <c>[CharacterWindowStartPositionInText, CharacterWindowEndPositionInText)</c> in <see cref="_text"/>.
         /// </summary>
-        public int Width
-        {
-            get
-            {
-                return _offset - _lexemeStart;
-            }
-        }
+        public int CharacterWindowStartPositionInText => _basis;
 
         /// <summary>
-        /// Start parsing a new lexeme.
+        /// Similar to <see cref="CharacterWindowStartPositionInText"/>, except this represents the index (exclusive) of the first character
+        /// that <see cref="_characterWindow"/> encompases in <see cref="_text"/>.  This is equal to <see cref="CharacterWindowStartPositionInText"/>
+        /// + <see cref="CharacterWindowCount"/>.
         /// </summary>
-        public void Start()
-        {
-            _lexemeStart = _offset;
-        }
+        public int CharacterWindowEndPositionInText => this.CharacterWindowStartPositionInText + this.CharacterWindowCount;
 
         public void Reset(int position)
         {
@@ -183,7 +153,6 @@ public void Reset(int position)
                     _text.CopyTo(position, _characterWindow, 0, amountToRead);
                 }
 
-                _lexemeStart = 0;
                 _offset = 0;
                 _basis = position;
                 _characterWindowCount = amountToRead;
@@ -199,21 +168,6 @@ private bool MoreChars()
                     return false;
                 }
 
-                // if lexeme scanning is sufficiently into the char buffer, 
-                // then refocus the window onto the lexeme
-                if (_lexemeStart > (_characterWindowCount / 4))
-                {
-                    Array.Copy(_characterWindow,
-                        _lexemeStart,
-                        _characterWindow,
-                        0,
-                        _characterWindowCount - _lexemeStart);
-                    _characterWindowCount -= _lexemeStart;
-                    _offset -= _lexemeStart;
-                    _basis += _lexemeStart;
-                    _lexemeStart = 0;
-                }
-
                 if (_characterWindowCount >= _characterWindow.Length)
                 {
                     // grow char array, since we need more contiguous space
@@ -272,8 +226,8 @@ public bool TryAdvance(char c)
         }
 
         /// <summary>
-        /// Advance the current position by n. No guarantee that this position
-        /// is valid.
+        /// Advance the current position by n. No guarantee that this position is valid.  This will <em>not</em> change the character window
+        /// in any way.  Specifically, it will not create a new character window, nor will it change the contents of the current window.
         /// </summary>
         public void AdvanceChar(int n)
         {
@@ -419,14 +373,9 @@ public string Intern(char[] array, int start, int length)
             return _strings.Add(array, start, length);
         }
 
-        public string GetInternedText()
-        {
-            return this.Intern(_characterWindow, _lexemeStart, this.Width);
-        }
-
-        public string GetText(bool intern)
+        public string GetText(int startPosition, bool intern)
         {
-            return this.GetText(this.LexemeStartPosition, this.Width, intern);
+            return this.GetText(startPosition, this.Position - startPosition, intern);
         }
 
         public string GetText(int position, int length, bool intern)