Improve support for isolated Hangul Jamo

Korean Hangul can be represented in Unicode either as precomposed Hangul
syllables, or as sequences of alphabetic components called Jamo.

Jamo at U+1100..U+11FF, U+A960..U+A97F, U+D7B0..U+D7FF have conjoining
behavior, a sequence of Jamo for Leading Consonants, Vowels, and
(optionally) Trailing Consonants, in that order, conjoins into a
syllable block, (i.e. a sequence L+V+T*, where + and * have their
typical regex meaning).

To represent a Jamo in isolation, there are several options:

 - Use non-conjoining Jamo from U+3130..U+318F (or the halfwidth forms
   at U+FFA0..U+FFDF.  These blocks covers all Jamo used in modern
   Korean, but not all Jamo used in Old Korean.
 - Use the fillers at U+115F and U+1160 to substitute for non-existing
   Leading Consonants, and Vowels.  Fonts may stretch Jamo into a
   corner.
 - Use Jamo from U+1100..U+11FF, U+A960..U+A97F, U+D7B0..U+D7FF, and
   separate them with non-Korean characters.  Unicode recommends U+200B
   ZERO WIDTH SPACE and U+2060 WORD JOINER for this purpose.

Previously, Konsole treated conjoining Jamo as combining marks which
could apply to any previous character.  This commit makes conjoining
Jamo only join when previous Jamo when they form (the beginning) of a
proper L+V+T* syllable.  It also fixes the width reported when they
cannot join previous characters, which should now be 2, instead of 0.

src/Screen.cpp

@@ -925,6 +925,7 @@ void Screen::displayCharacter(uint c)
         // Find previous "real character" to try to combine with
         int charToCombineWithX = qMin(_cuX, _screenLines.at(_cuY).length());
         int charToCombineWithY = _cuY;
+        bool previousChar = true;
         do {
             if (charToCombineWithX > 0) {
                 --charToCombineWithX;
@@ -933,16 +934,33 @@ void Screen::displayCharacter(uint c)
                 charToCombineWithX = _screenLines.at(charToCombineWithY).length() - 1;
             } else {
                 // Give up
-                return;
+                previousChar = false;
+                break;
             }
 
             // Failsafe
             if (charToCombineWithX < 0) {
-                return;
+                previousChar = false;
+                break;
             }
         } while (_screenLines.at(charToCombineWithY).at(charToCombineWithX).isRightHalfOfDoubleWide());
 
+        if (!previousChar) {
+            if (!Hangul::isHangul(c)) {
+                return;
+            } else {
+                w = 2;
+                goto notcombine;
+            }
+        }
+
         Character &currentChar = _screenLines[charToCombineWithY][charToCombineWithX];
+
+        if (Hangul::isHangul(c) && !Hangul::combinesWith(currentChar, c)) {
+            w = 2;
+            goto notcombine;
+        }
+
         if ((currentChar.rendition & RE_EXTENDED_CHAR) == 0) {
             const uint chars[2] = {currentChar.character, c};
             currentChar.rendition |= RE_EXTENDED_CHAR;
@@ -954,7 +972,7 @@ void Screen::displayCharacter(uint c)
             ushort extendedCharLength;
             const uint *oldChars = ExtendedCharTable::instance.lookupExtendedChar(currentChar.character, extendedCharLength);
             Q_ASSERT(oldChars);
-            if (((oldChars) != nullptr) && extendedCharLength < 3) {
+            if (((oldChars) != nullptr) && extendedCharLength < 8) {
                 Q_ASSERT(extendedCharLength > 1);
                 Q_ASSERT(extendedCharLength < 65535); // redundant due to above check
                 auto chars = std::make_unique<uint[]>(extendedCharLength + 1);
@@ -969,6 +987,7 @@ void Screen::displayCharacter(uint c)
         return;
     }
 
+notcombine:
     if (_cuX + w > getScreenLineColumns(_cuY)) {
         if (getMode(MODE_Wrap)) {
             _lineProperties[_cuY] = static_cast<LineProperty>(_lineProperties.at(_cuY) | LINE_WRAPPED);

src/characters/CMakeLists.txt

@@ -2,6 +2,7 @@
 set(konsolecharacters_SRCS
     CharacterFormat.cpp
     CharacterWidth.cpp
+    Hangul.cpp
     LineBlockCharacters.cpp
     ExtendedCharTable.cpp
 )

src/characters/Character.h

@@ -12,6 +12,7 @@
 #include "CharacterColor.h"
 #include "CharacterWidth.h"
 #include "ExtendedCharTable.h"
+#include "Hangul.h"
 #include "LineBlockCharacters.h"
 
 // Qt
@@ -160,8 +161,17 @@ public:
     static int stringWidth(const uint *ucs4Str, int len)
     {
         int w = 0;
+        Hangul::SyllablePos hangulSyllablePos = Hangul::NotInSyllable;
+
         for (int i = 0; i < len; ++i) {
-            w += width(ucs4Str[i]);
+            const uint c = ucs4Str[i];
+
+            if (!Hangul::isHangul(c)) {
+                w += width(c);
+                hangulSyllablePos = Hangul::NotInSyllable;
+            } else {
+                w += Hangul::width(c, width(c), hangulSyllablePos);
+            }
         }
         return w;
     }

src/characters/Hangul.cpp

+/*
+    SPDX-FileCopyrightText: 2022 Luis Javier Merino Morán <ninjalj@gmail.com>
+
+    SPDX-License-Identifier: GPL-2.0-or-later
+*/
+
+// Own
+#include "Hangul.h"
+
+// Qt
+#include <QtGlobal>
+
+// Konsole
+#include "Character.h"
+
+namespace Konsole
+{
+
+int Hangul::width(uint c, int widthFromTable, enum Hangul::SyllablePos &syllablePos)
+{
+    int w = 0;
+    switch (syllablePos) {
+    case NotInSyllable:
+        if (isLeadingJamo(c)) {
+            // new Hangul syllable
+            syllablePos = AtLeadingJamo;
+            w += widthFromTable;
+        } else if (isLvSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtVowelJamo;
+            w += widthFromTable;
+        } else if (isLvtSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtTrailingJamo;
+            w += widthFromTable;
+        } else if (isVowelJamo(c)) {
+            // invalid here, isolated Jamo
+            w += 2;
+        } else if (isTrailingJamo(c)) {
+            // invalid here, isolated Jamo
+            w += 2;
+        }
+        break;
+    case AtLeadingJamo:
+        if (isLeadingJamo(c)) {
+            // conjoin
+        } else if (isLvSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtVowelJamo;
+            w += widthFromTable;
+        } else if (isLvtSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtTrailingJamo;
+            w += widthFromTable;
+        } else if (isVowelJamo(c)) {
+            syllablePos = AtVowelJamo;
+            // conjoin
+        } else if (isTrailingJamo(c)) {
+            // invalid here, isolated Jamo
+            syllablePos = NotInSyllable;
+            w += 2;
+        }
+        break;
+    case AtVowelJamo:
+        if (isLeadingJamo(c)) {
+            // new Hangul syllable
+            syllablePos = AtLeadingJamo;
+            w += widthFromTable;
+        } else if (isLvSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtVowelJamo;
+            w += widthFromTable;
+        } else if (isLvtSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtTrailingJamo;
+            w += widthFromTable;
+        } else if (isVowelJamo(c)) {
+            // conjoin
+        } else if (isTrailingJamo(c)) {
+            syllablePos = AtTrailingJamo;
+            // conjoin
+        }
+        break;
+    case AtTrailingJamo:
+        if (isLeadingJamo(c)) {
+            // new Hangul syllable
+            syllablePos = AtLeadingJamo;
+            w += widthFromTable;
+        } else if (isLvSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtVowelJamo;
+            w += widthFromTable;
+        } else if (isLvtSyllable(c)) {
+            // new Hangul syllable
+            syllablePos = AtTrailingJamo;
+            w += widthFromTable;
+        } else if (isVowelJamo(c)) {
+            // invalid here, isolated Jamo
+            syllablePos = NotInSyllable;
+            w += 2;
+        } else if (isTrailingJamo(c)) {
+            // conjoin
+        }
+    }
+
+    return w;
+}
+
+void Hangul::updateHangulSyllablePos(Hangul::SyllablePos &syllablePos, uint c)
+{
+    if (!isHangul(c)) {
+        syllablePos = NotInSyllable;
+    }
+    (void)Hangul::width(c, 0, syllablePos);
+}
+
+bool Hangul::validSyllableContinuation(Hangul::SyllablePos syllablePos, uint c)
+{
+    SyllableType type = jamoType(c);
+
+    switch (syllablePos) {
+    case AtLeadingJamo:
+        return type != Trailing_Jamo && type != Not_Applicable;
+    case AtVowelJamo:
+        return type == Trailing_Jamo || type == Vowel_Jamo;
+    case AtTrailingJamo:
+        return type == Trailing_Jamo;
+    default:
+        return false;
+    }
+}
+
+bool Hangul::combinesWith(Character prevChar, uint c)
+{
+    Hangul::SyllablePos syllablePos = Hangul::NotInSyllable;
+
+    if ((prevChar.rendition & RE_EXTENDED_CHAR) == 0) {
+        const uint prev = prevChar.character;
+        updateHangulSyllablePos(syllablePos, prev);
+    } else {
+        ushort extendedCharLength;
+        const uint *oldChars = ExtendedCharTable::instance.lookupExtendedChar(prevChar.character, extendedCharLength);
+        if (oldChars == nullptr) {
+            return false;
+        } else {
+            for (int i = 0; i < extendedCharLength; i++) {
+                updateHangulSyllablePos(syllablePos, oldChars[i]);
+            }
+        }
+    }
+
+    return validSyllableContinuation(syllablePos, c);
+}
+
+}

src/characters/Hangul.h

+/*
+    SPDX-FileCopyrightText: 2022 Luis Javier Merino Morán <ninjalj@gmail.com>
+
+    SPDX-License-Identifier: GPL-2.0-or-later
+*/
+
+#ifndef HANGUL_H
+#define HANGUL_H
+
+#include <QtGlobal>
+
+namespace Konsole
+{
+
+class Character;
+
+class Hangul
+{
+public:
+    // See HangulSyllableType.txt from the Unicode data distribution
+    enum SyllableType {
+        Not_Applicable,
+        Leading_Jamo,
+        Vowel_Jamo,
+        Trailing_Jamo,
+
+        LV_Syllable,
+        LVT_Syllable,
+    };
+
+    enum SyllablePos {
+        NotInSyllable,
+        AtLeadingJamo,
+        AtVowelJamo,
+        AtTrailingJamo,
+    };
+
+    static int width(uint c, int widthFromTable, enum SyllablePos &syllablePos);
+    static bool combinesWith(Character prev, uint c);
+
+    static bool isHangul(const uint c)
+    {
+        return (c >= 0x1100 && c <= 0x11ff) || (c >= 0xa960 && c <= 0xa97f) || (c >= 0xd7b0 && c <= 0xd7ff) || (c >= 0xac00 && c <= 0xd7a3);
+    }
+
+private:
+    static bool isLeadingJamo(const uint c)
+    {
+        return (c >= 0x1100 && c <= 0x115f) || (c >= 0xa960 && c <= 0xa97f);
+    }
+
+    static bool isVowelJamo(const uint c)
+    {
+        return (c >= 0x1160 && c <= 0x11a7) || (c >= 0xd7b0 && c <= 0xd7c6);
+    }
+
+    static bool isTrailingJamo(const uint c)
+    {
+        return (c >= 0x11a8 && c <= 0x11ff) || (c >= 0xd7cb && c <= 0xd7ff);
+    }
+
+    static bool isLvSyllable(const uint c)
+    {
+        return (c >= 0xac00 && c <= 0xd7a3) && (c % 0x1c == 0);
+    }
+
+    static bool isLvtSyllable(const uint c)
+    {
+        return (c >= 0xac00 && c <= 0xd7a3) && (c % 0x1c != 0);
+    }
+
+    static SyllableType jamoType(const uint c)
+    {
+        // clang-format off
+        if (isLeadingJamo(c))  return Leading_Jamo;
+        if (isVowelJamo(c))    return Vowel_Jamo;
+        if (isTrailingJamo(c)) return Trailing_Jamo;
+        if (isLvSyllable(c))   return LV_Syllable;
+        if (isLvtSyllable(c))  return LVT_Syllable;
+        return Not_Applicable;
+        // clang-format on
+    }
+
+    static void updateHangulSyllablePos(Hangul::SyllablePos &syllablePos, uint c);
+    static bool validSyllableContinuation(Hangul::SyllablePos syllablePos, uint c);
+};
+
+}
+
+#endif // HANGUL_H

src/decoders/PlainTextDecoder.cpp

@@ -131,7 +131,7 @@ void PlainTextDecoder::decodeLine(const Character *const characters, int count,
             // of `dialog --infobox "qwe" 10 10` .
             if (characters[i].isRealCharacter || i <= realCharacterGuard) {
                 characterBuffer.append(characters[i].character);
-                i += qMax(1, characters[i].width());
+                i += qMax(1, Character::stringWidth(&characters[i].character, 1));
             } else {
                 ++i; // should we 'break' directly here?
             }