diff options
Diffstat (limited to 'src/PositionCache.cxx')
| -rw-r--r-- | src/PositionCache.cxx | 15 |
1 files changed, 13 insertions, 2 deletions
diff --git a/src/PositionCache.cxx b/src/PositionCache.cxx index d9fb9a783..cd5becf1a 100644 --- a/src/PositionCache.cxx +++ b/src/PositionCache.cxx @@ -440,6 +440,10 @@ int BreakFinder::First() { return nextBreak; } +static bool IsTrailByte(int ch) { + return (ch >= 0x80) && (ch < (0x80 + 0x40)); +} + int BreakFinder::Next() { if (subBreak == -1) { int prev = nextBreak; @@ -472,15 +476,20 @@ int BreakFinder::Next() { } else { int lastGoodBreak = -1; int lastOKBreak = -1; + int lastUTF8Break = -1; int j; for (j = subBreak + 1; j <= nextBreak; j++) { if (IsSpaceOrTab(ll->chars[j - 1]) && !IsSpaceOrTab(ll->chars[j])) { lastGoodBreak = j; } - if (ll->chars[j] < 'A') { + if (static_cast<unsigned char>(ll->chars[j]) < 'A') { lastOKBreak = j; } - if (((j - subBreak) >= lengthEachSubdivision) && ((lastGoodBreak >= 0) || (lastOKBreak >= 0))) { + if (utf8 && !IsTrailByte(static_cast<unsigned char>(ll->chars[j]))) { + lastUTF8Break = j; + } + if (((j - subBreak) >= lengthEachSubdivision) && + ((lastGoodBreak >= 0) || (lastOKBreak >= 0) || (lastUTF8Break >= 0))) { break; } } @@ -488,6 +497,8 @@ int BreakFinder::Next() { subBreak = lastGoodBreak; } else if (lastOKBreak >= 0) { subBreak = lastOKBreak; + } else if (lastUTF8Break >= 0) { + subBreak = lastUTF8Break; } else { subBreak = nextBreak; } |