diff --git a/src/FsAutoComplete.Core/Commands.fs b/src/FsAutoComplete.Core/Commands.fs index c4af5c608..4c49e1fe7 100644 --- a/src/FsAutoComplete.Core/Commands.fs +++ b/src/FsAutoComplete.Core/Commands.fs @@ -1114,6 +1114,10 @@ module Commands = highlights |> Array.sortBy startToken + // LSP semantic tokens require single-line ranges; multiline tokens cause integer underflow + // when computing token length (End.Character - Start.Character wraps for uint32). + // Filter them out here since they cannot be represented correctly. + |> Array.filter (fun m -> m.Range.StartLine = m.Range.EndLine) |> Array.groupBy (fun m -> m.Range.StartLine) |> Array.collect (fun (_, highlights) -> diff --git a/src/FsAutoComplete/LspHelpers.fs b/src/FsAutoComplete/LspHelpers.fs index 2f03a98b1..4050cc077 100644 --- a/src/FsAutoComplete/LspHelpers.fs +++ b/src/FsAutoComplete/LspHelpers.fs @@ -1212,7 +1212,13 @@ let encodeSemanticHighlightRanges else uint32 range.Start.Character - let tokenLen = uint32 (range.End.Character - range.Start.Character) + // Guard against multiline ranges: End.Character < Start.Character would underflow uint32 + let tokenLen = + if range.End.Line > range.Start.Line || range.End.Character < range.Start.Character then + 0u + else + uint32 (range.End.Character - range.Start.Character) + let tokenTy = uint32 ty let tokenMods = diff --git a/test/FsAutoComplete.Tests.Lsp/HighlightingTests.fs b/test/FsAutoComplete.Tests.Lsp/HighlightingTests.fs index 5c19948a0..1cf2b0f9e 100644 --- a/test/FsAutoComplete.Tests.Lsp/HighlightingTests.fs +++ b/test/FsAutoComplete.Tests.Lsp/HighlightingTests.fs @@ -151,4 +151,26 @@ let tests state = tokenIsOfType (3u, 52u) ClassificationUtils.SemanticTokenTypes.Class fullHighlights // the `string` type annotation in the PooPoo srtp member tokenIsOfType (6u, 21u) ClassificationUtils.SemanticTokenTypes.EnumMember fullHighlights // the `PeePee` AP application in the `yeet` function definition tokenIsOfType (9u, 10u) ClassificationUtils.SemanticTokenTypes.Class fullHighlights //the `SomeJson` type alias should be a type - tokenIsOfType (15u, 2u) ClassificationUtils.SemanticTokenTypes.Module fullHighlights ] ] // tests that module coloration isn't overwritten by function coloration when a module function is used, so Foo in Foo.x should be module-colored + tokenIsOfType (15u, 2u) ClassificationUtils.SemanticTokenTypes.Module fullHighlights // tests that module coloration isn't overwritten by function coloration when a module function is used, so Foo in Foo.x should be module-colored + + // Regression test for https://github.com/ionide/FsAutoComplete/issues/1407: + // A file containing a multiline string literal must not produce any decoded token + // whose end column is unreasonably far from its start column (which would indicate + // a uint32 underflow from the old tokenLen = uint32(End.Character - Start.Character) + // on a multiline range). If the fix is reverted, the decoded range for a multiline + // string would have endCol = startCol + ~4294967290, and the editor would freeze. + testCaseAsync + "no uint32 underflow in decoded token ranges when file contains a multiline string" + (async { + let! highlights = fullHighlights + + let maxReasonableTokenLen = 1_000u + + for (r, _tokenType, _tokenMods) in highlights do + let tokenLen = r.End.Character - r.Start.Character + + Expect.isLessThan + tokenLen + maxReasonableTokenLen + $"Token length {tokenLen} is unreasonably large (possible uint32 underflow from multiline range {r})" + }) ] ] diff --git a/test/FsAutoComplete.Tests.Lsp/LspHelpersTests.fs b/test/FsAutoComplete.Tests.Lsp/LspHelpersTests.fs new file mode 100644 index 000000000..4678ec64c --- /dev/null +++ b/test/FsAutoComplete.Tests.Lsp/LspHelpersTests.fs @@ -0,0 +1,142 @@ +module FsAutoComplete.Tests.LspHelpersTests + +open Expecto +open FsAutoComplete.LspHelpers +open Ionide.LanguageServerProtocol.Types + +/// Helpers to construct test data for encodeSemanticHighlightRanges +module private Helpers = + + let makeRange startLine startChar endLine endChar : Range = + { Start = + { Line = uint32 startLine + Character = uint32 startChar } + End = + { Line = uint32 endLine + Character = uint32 endChar } } + + let makeEntry + (range: Range) + ty + : struct (Range * ClassificationUtils.SemanticTokenTypes * ClassificationUtils.SemanticTokenModifier list) = + struct (range, ty, []) + +/// Unit tests for encodeSemanticHighlightRanges, focused on the uint32 underflow +/// regression described in https://github.com/ionide/FsAutoComplete/issues/1407. +/// +/// The LSP semantic-token encoding stores each token as five uint32 values: +/// [lineDelta, charDelta, tokenLen, tokenType, tokenModifiers] +/// Before the fix, a multiline range (End.Character < Start.Character) would +/// cause tokenLen to underflow (e.g. uint32(4 - 10) = 4294967290u), which +/// could freeze or crash editors such as Neovim. +let encodeSemanticHighlightRangesTests = + testList + "encodeSemanticHighlightRanges" + [ testCase "empty input returns None" + <| fun _ -> + let result = encodeSemanticHighlightRanges [||] + Expect.isNone result "Empty input should return None" + + testCase "single normal single-line range is encoded with correct length" + <| fun _ -> + // Range on line 3, columns 5–10 → length 5 + let range = Helpers.makeRange 3 5 3 10 + let entry = Helpers.makeEntry range ClassificationUtils.SemanticTokenTypes.Variable + + match encodeSemanticHighlightRanges [| entry |] with + | None -> failtest "Expected Some but got None" + | Some data -> + Expect.equal data.Length 5 "Single token should produce 5 uint32 values" + // lineDelta relative to file start (line 0) = 3 + Expect.equal data.[0] 3u "lineDelta should be 3" + // charDelta = absolute column when line changes = 5 + Expect.equal data.[1] 5u "charDelta should be 5" + // tokenLen = 10 - 5 = 5 + Expect.equal data.[2] 5u "tokenLen should be 5" + + testCase "multiline range (End.Line > Start.Line) produces tokenLen = 0, not a uint32 underflow" + <| fun _ -> + // Simulates a multiline string: starts at col 10 on line 2, ends at col 4 on line 3. + // Before the fix: uint32(4u - 10u) would underflow to 4294967290u (~4 GB). + let range = Helpers.makeRange 2 10 3 4 + let entry = Helpers.makeEntry range ClassificationUtils.SemanticTokenTypes.String + + match encodeSemanticHighlightRanges [| entry |] with + | None -> failtest "Expected Some but got None" + | Some data -> + Expect.equal data.Length 5 "Single token should produce 5 uint32 values" + Expect.equal data.[2] 0u "Multiline range must clamp tokenLen to 0, not underflow" + // Belt-and-suspenders: confirm it is not the old underflowed value + Expect.isLessThan data.[2] 1_000_000u "tokenLen must not be a huge underflowed value" + + testCase "single-line inverted range (End.Character < Start.Character) produces tokenLen = 0" + <| fun _ -> + // Degenerate range on the same line where end column < start column. + let range = Helpers.makeRange 5 15 5 3 + let entry = Helpers.makeEntry range ClassificationUtils.SemanticTokenTypes.Variable + + match encodeSemanticHighlightRanges [| entry |] with + | None -> failtest "Expected Some but got None" + | Some data -> Expect.equal data.[2] 0u "Inverted single-line range must clamp tokenLen to 0" + + testCase "multiline range followed by valid single-line range: subsequent token is encoded correctly" + <| fun _ -> + // A multiline token (lines 2–3) should not corrupt the delta encoding of the + // token that follows it on line 5. + let multilineRange = Helpers.makeRange 2 10 3 4 + // Valid token: line 5, col 2–8, length 6 + let validRange = Helpers.makeRange 5 2 5 8 + + let entries = + [| Helpers.makeEntry multilineRange ClassificationUtils.SemanticTokenTypes.String + Helpers.makeEntry validRange ClassificationUtils.SemanticTokenTypes.Variable |] + + match encodeSemanticHighlightRanges entries with + | None -> failtest "Expected Some but got None" + | Some data -> + Expect.equal data.Length 10 "Two tokens should produce 10 uint32 values" + + // First token (multiline): tokenLen must be 0 + Expect.equal data.[2] 0u "Multiline token must have tokenLen = 0" + + // Second token (valid, line 5): lineDelta from line 2 = 3, char = 2 (absolute, new line), len = 6 + Expect.equal data.[5] 3u "lineDelta of second token should be 5 - 2 = 3" + Expect.equal data.[6] 2u "charDelta of second token should be 2 (absolute on new line)" + Expect.equal data.[7] 6u "tokenLen of second token should be 6" + + testCase "sequence of valid single-line tokens uses correct relative deltas" + <| fun _ -> + // Three tokens on different lines; verifies standard delta encoding is unaffected. + // Token A: line 1, col 0–4 (len 4) + // Token B: line 1, col 8–12 (len 4, same line → charDelta = 8-0 = 8) + // Token C: line 3, col 2–7 (len 5, new line → charDelta = 2 absolute) + let rangeA = Helpers.makeRange 1 0 1 4 + let rangeB = Helpers.makeRange 1 8 1 12 + let rangeC = Helpers.makeRange 3 2 3 7 + + let entries = + [| Helpers.makeEntry rangeA ClassificationUtils.SemanticTokenTypes.Variable + Helpers.makeEntry rangeB ClassificationUtils.SemanticTokenTypes.Function + Helpers.makeEntry rangeC ClassificationUtils.SemanticTokenTypes.Class |] + + match encodeSemanticHighlightRanges entries with + | None -> failtest "Expected Some but got None" + | Some data -> + Expect.equal data.Length 15 "Three tokens should produce 15 uint32 values" + + // Token A: lineDelta=1 (from line 0), charDelta=0 (absolute on new line), len=4 + Expect.equal data.[0] 1u "Token A lineDelta = 1" + Expect.equal data.[1] 0u "Token A charDelta = 0" + Expect.equal data.[2] 4u "Token A tokenLen = 4" + + // Token B: same line as A → lineDelta=0, charDelta = 8-0 = 8, len=4 + Expect.equal data.[5] 0u "Token B lineDelta = 0 (same line)" + Expect.equal data.[6] 8u "Token B charDelta = 8 (relative to prev col 0)" + Expect.equal data.[7] 4u "Token B tokenLen = 4" + + // Token C: new line → lineDelta = 3-1 = 2, charDelta = 2 (absolute), len=5 + Expect.equal data.[10] 2u "Token C lineDelta = 2" + Expect.equal data.[11] 2u "Token C charDelta = 2 (absolute on new line)" + Expect.equal data.[12] 5u "Token C tokenLen = 5" ] + +let allTests = testList "LspHelpers" [ encodeSemanticHighlightRangesTests ] diff --git a/test/FsAutoComplete.Tests.Lsp/Program.fs b/test/FsAutoComplete.Tests.Lsp/Program.fs index 255a22711..1ec37b451 100644 --- a/test/FsAutoComplete.Tests.Lsp/Program.fs +++ b/test/FsAutoComplete.Tests.Lsp/Program.fs @@ -147,7 +147,9 @@ let generalTests = "general" [ testList (nameof (Utils)) [ Utils.Tests.Utils.tests; Utils.Tests.TextEdit.tests ] InlayHintTests.explicitTypeInfoTests sourceTextFactory - FindReferences.tryFixupRangeTests sourceTextFactory ] + FindReferences.tryFixupRangeTests sourceTextFactory + UtilsTests.allTests + LspHelpersTests.allTests ] [] let tests = diff --git a/test/FsAutoComplete.Tests.Lsp/TestCases/HighlightingTest/Script.fsx b/test/FsAutoComplete.Tests.Lsp/TestCases/HighlightingTest/Script.fsx index fa7e3bcac..0f4999190 100644 --- a/test/FsAutoComplete.Tests.Lsp/TestCases/HighlightingTest/Script.fsx +++ b/test/FsAutoComplete.Tests.Lsp/TestCases/HighlightingTest/Script.fsx @@ -14,3 +14,12 @@ module Foo = let myFoo x y = Foo.x x y + +// Multiline string: exercises the fix for uint32 underflow in semantic token encoding. +// Before the fix, FCS would return a multiline SemanticClassificationItem for this token, +// causing tokenLen = uint32(End.Character - Start.Character) to underflow for multiline ranges. +let multilineString = + """this is a +multiline string literal""" + +let afterMultiline = 42