A step out from Harfbuzz (reduce dependency)

Introduce QCharAttributes and use it instead of HB_CharAttributes everywhere in Qt
(in Harfbuzz, the HB_CharAttributes is only used in the text segmentation algorithm
which has been moved from HB to Qt (well, most of it)).
Rename some members to better reflect their meaning,
remember to keep HB_CharAttributes in sync with QCharAttributes.
Also replace HB_ScriptItem with a (temporary) QUnicodeTools::ScriptItem struct
that will be replaced with a more efficient/friendly solution a bit later.

The soft hyphen and the mandatory break detection has been factored out
of the default text breaking algorithm to a higher level in order to refactor
the QCharAttributes bitfields and to optimize the implementation for the common case.

Change-Id: Ieb365623ae954430f1c8b2dfcd65c82973143eec
Reviewed-by: Lars Knoll <lars.knoll@digia.com>

4 files changed, 101 insertions, 64 deletions

diff --git a/src/corelib/tools/qharfbuzz_p.h b/src/corelib/tools/qharfbuzz_p.h
index 23de3efa16..72d5bda77f 100644
--- a/src/corelib/tools/qharfbuzz_p.h
+++ b/src/corelib/tools/qharfbuzz_p.h
@@ -65,9 +65,6 @@ Q_CORE_EXPORT HB_Face qHBNewFace(void *font, HB_GetFontTableFunc tableFunc);
 Q_CORE_EXPORT void qHBFreeFace(HB_Face);
 Q_CORE_EXPORT HB_Face qHBLoadFace(HB_Face face);
 
-Q_DECLARE_TYPEINFO(HB_CharAttributes, Q_PRIMITIVE_TYPE);
-Q_DECLARE_TYPEINFO(HB_ScriptItem, Q_PRIMITIVE_TYPE);
-
 Q_DECLARE_TYPEINFO(HB_GlyphAttributes, Q_PRIMITIVE_TYPE);
 Q_DECLARE_TYPEINFO(HB_FixedPoint, Q_PRIMITIVE_TYPE);
 
diff --git a/src/corelib/tools/qtextboundaryfinder.cpp b/src/corelib/tools/qtextboundaryfinder.cpp
index c3d44da042..f928932548 100644
--- a/src/corelib/tools/qtextboundaryfinder.cpp
+++ b/src/corelib/tools/qtextboundaryfinder.cpp
@@ -49,12 +49,12 @@ QT_BEGIN_NAMESPACE
 class QTextBoundaryFinderPrivate
 {
 public:
-    HB_CharAttributes attributes[1];
+    QCharAttributes attributes[1];
 };
 
-static void init(QTextBoundaryFinder::BoundaryType type, const QChar *chars, int length, HB_CharAttributes *attributes)
+static void init(QTextBoundaryFinder::BoundaryType type, const QChar *chars, int length, QCharAttributes *attributes)
 {
-    QVarLengthArray<HB_ScriptItem> scriptItems;
+    QVarLengthArray<QUnicodeTools::ScriptItem> scriptItems;
 
     const ushort *string = reinterpret_cast<const ushort *>(chars);
     const ushort *unicode = string;
@@ -72,11 +72,9 @@ static void init(QTextBoundaryFinder::BoundaryType type, const QChar *chars, int
             script = QUnicodeTables::Common;
         if (script != lastScript) {
             if (uc != start) {
-                HB_ScriptItem item;
-                item.pos = start - string;
-                item.length = uc - start;
-                item.script = (HB_Script)lastScript;
-                item.bidiLevel = 0; // ### what's the proper value?
+                QUnicodeTools::ScriptItem item;
+                item.position = start - string;
+                item.script = lastScript;
                 scriptItems.append(item);
                 start = uc;
             }
@@ -85,11 +83,9 @@ static void init(QTextBoundaryFinder::BoundaryType type, const QChar *chars, int
         ++uc;
     }
     if (uc != start) {
-        HB_ScriptItem item;
-        item.pos = start - string;
-        item.length = uc - start;
-        item.script = (HB_Script)lastScript;
-        item.bidiLevel = 0; // ### what's the proper value?
+        QUnicodeTools::ScriptItem item;
+        item.position = start - string;
+        item.script = lastScript;
         scriptItems.append(item);
     }
 
@@ -193,9 +189,9 @@ QTextBoundaryFinder::QTextBoundaryFinder(const QTextBoundaryFinder &other)
     , pos(other.pos)
     , freePrivate(true)
 {
-    d = (QTextBoundaryFinderPrivate *) malloc(length*sizeof(HB_CharAttributes));
+    d = (QTextBoundaryFinderPrivate *) malloc(length*sizeof(QCharAttributes));
     Q_CHECK_PTR(d);
-    memcpy(d, other.d, length*sizeof(HB_CharAttributes));
+    memcpy(d, other.d, length*sizeof(QCharAttributes));
 }
 
 /*!
@@ -213,11 +209,11 @@ QTextBoundaryFinder &QTextBoundaryFinder::operator=(const QTextBoundaryFinder &o
     pos = other.pos;
 
     QTextBoundaryFinderPrivate *newD = (QTextBoundaryFinderPrivate *)
-        realloc(freePrivate ? d : 0, length*sizeof(HB_CharAttributes));
+        realloc(freePrivate ? d : 0, length*sizeof(QCharAttributes));
     Q_CHECK_PTR(newD);
     freePrivate = true;
     d = newD;
-    memcpy(d, other.d, length*sizeof(HB_CharAttributes));
+    memcpy(d, other.d, length*sizeof(QCharAttributes));
 
     return *this;
 }
@@ -242,7 +238,7 @@ QTextBoundaryFinder::QTextBoundaryFinder(BoundaryType type, const QString &strin
     , pos(0)
     , freePrivate(true)
 {
-    d = (QTextBoundaryFinderPrivate *) malloc(length*sizeof(HB_CharAttributes));
+    d = (QTextBoundaryFinderPrivate *) malloc(length*sizeof(QCharAttributes));
     Q_CHECK_PTR(d);
     init(t, chars, length, d->attributes);
 }
@@ -266,11 +262,11 @@ QTextBoundaryFinder::QTextBoundaryFinder(BoundaryType type, const QChar *chars,
     , length(length)
     , pos(0)
 {
-    if (buffer && (uint)bufferSize >= length*sizeof(HB_CharAttributes)) {
+    if (buffer && (uint)bufferSize >= length*sizeof(QCharAttributes)) {
         d = (QTextBoundaryFinderPrivate *)buffer;
         freePrivate = false;
     } else {
-        d = (QTextBoundaryFinderPrivate *) malloc(length*sizeof(HB_CharAttributes));
+        d = (QTextBoundaryFinderPrivate *) malloc(length*sizeof(QCharAttributes));
         Q_CHECK_PTR(d);
         freePrivate = true;
     }
@@ -368,11 +364,11 @@ int QTextBoundaryFinder::toNextBoundary()
     
     switch(t) {
     case Grapheme:
-        while (pos < length && !d->attributes[pos].charStop)
+        while (pos < length && !d->attributes[pos].graphemeBoundary)
             ++pos;
         break;
     case Word:
-        while (pos < length && !d->attributes[pos].wordBoundary)
+        while (pos < length && !d->attributes[pos].wordBreak)
             ++pos;
         break;
     case Sentence:
@@ -380,7 +376,7 @@ int QTextBoundaryFinder::toNextBoundary()
             ++pos;
         break;
     case Line:
-        while (pos < length && d->attributes[pos].lineBreakType == HB_NoBreak)
+        while (pos < length && !d->attributes[pos].lineBreak)
             ++pos;
         break;
     }
@@ -410,11 +406,11 @@ int QTextBoundaryFinder::toPreviousBoundary()
 
     switch(t) {
     case Grapheme:
-        while (pos > 0 && !d->attributes[pos].charStop)
+        while (pos > 0 && !d->attributes[pos].graphemeBoundary)
             --pos;
         break;
     case Word:
-        while (pos > 0 && !d->attributes[pos].wordBoundary)
+        while (pos > 0 && !d->attributes[pos].wordBreak)
             --pos;
         break;
     case Sentence:
@@ -422,7 +418,7 @@ int QTextBoundaryFinder::toPreviousBoundary()
             --pos;
         break;
     case Line:
-        while (pos > 0 && d->attributes[pos].lineBreakType == HB_NoBreak)
+        while (pos > 0 && !d->attributes[pos].lineBreak)
             --pos;
         break;
     }
@@ -443,11 +439,11 @@ bool QTextBoundaryFinder::isAtBoundary() const
 
     switch(t) {
     case Grapheme:
-        return d->attributes[pos].charStop;
+        return d->attributes[pos].graphemeBoundary;
     case Word:
-        return d->attributes[pos].wordBoundary;
+        return d->attributes[pos].wordBreak;
     case Line:
-        return pos == 0 || d->attributes[pos].lineBreakType != HB_NoBreak;
+        return pos == 0 || d->attributes[pos].lineBreak;
     case Sentence:
         return d->attributes[pos].sentenceBoundary;
     }
@@ -463,8 +459,6 @@ QTextBoundaryFinder::BoundaryReasons QTextBoundaryFinder::boundaryReasons() cons
         return NotAtBoundary;
     if (! isAtBoundary())
         return NotAtBoundary;
-    if (t == Line && pos < length && d->attributes[pos].lineBreakType == HB_SoftHyphen)
-        return SoftHyphen;
     if (pos == 0) {
         if (d->attributes[pos].whiteSpace)
             return NotAtBoundary;
@@ -476,6 +470,12 @@ QTextBoundaryFinder::BoundaryReasons QTextBoundaryFinder::boundaryReasons() cons
         return EndWord;
     }
 
+    if (t == Line && chars[pos - 1].unicode() == QChar::SoftHyphen)
+        return SoftHyphen;
+
+    if (t != Word)
+        return BoundaryReasons(StartWord | EndWord);
+
     const bool nextIsSpace = d->attributes[pos].whiteSpace;
     const bool prevIsSpace = d->attributes[pos - 1].whiteSpace;
 
diff --git a/src/corelib/tools/qunicodetools.cpp b/src/corelib/tools/qunicodetools.cpp
index f8db6cbac3..ffb9512229 100644
--- a/src/corelib/tools/qunicodetools.cpp
+++ b/src/corelib/tools/qunicodetools.cpp
@@ -42,6 +42,9 @@
 #include "qunicodetools_p.h"
 
 #include "qunicodetables_p.h"
+#include "qvarlengtharray.h"
+
+#include <harfbuzz-shaper.h>
 
 #define FLAG(x) (1 << (x))
 
@@ -77,7 +80,7 @@ static const uchar breakTable[QUnicodeTables::GraphemeBreak_LVT + 1][QUnicodeTab
 
 } // namespace GB
 
-static void getGraphemeBreaks(const ushort *string, quint32 len, HB_CharAttributes *attributes)
+static void getGraphemeBreaks(const ushort *string, quint32 len, QCharAttributes *attributes)
 {
     QUnicodeTables::GraphemeBreakClass lcls = QUnicodeTables::GraphemeBreak_LF; // to meet GB1
     for (quint32 i = 0; i != len; ++i) {
@@ -94,7 +97,8 @@ static void getGraphemeBreaks(const ushort *string, quint32 len, HB_CharAttribut
         const QUnicodeTables::Properties *prop = QUnicodeTables::properties(ucs4);
         QUnicodeTables::GraphemeBreakClass cls = (QUnicodeTables::GraphemeBreakClass) prop->graphemeBreakClass;
 
-        attributes[pos].charStop = GB::breakTable[lcls][cls];
+        if (Q_LIKELY(GB::breakTable[lcls][cls]))
+            attributes[pos].graphemeBoundary = true;
 
         lcls = cls;
     }
@@ -127,7 +131,7 @@ static const uchar breakTable[QUnicodeTables::WordBreak_ExtendNumLet + 1][QUnico
 
 } // namespace WB
 
-static void getWordBreaks(const ushort *string, quint32 len, HB_CharAttributes *attributes)
+static void getWordBreaks(const ushort *string, quint32 len, QCharAttributes *attributes)
 {
     QUnicodeTables::WordBreakClass cls = QUnicodeTables::WordBreak_LF; // to meet WB1
     for (quint32 i = 0; i != len; ++i) {
@@ -175,7 +179,7 @@ static void getWordBreaks(const ushort *string, quint32 len, HB_CharAttributes *
         }
         cls = ncls;
         if (action == WB::Break)
-            attributes[pos].wordBoundary = true;
+            attributes[pos].wordBreak = true;
     }
 }
 
@@ -217,7 +221,7 @@ static const uchar breakTable[BAfter + 1][QUnicodeTables::SentenceBreak_Close +
 
 } // namespace SB
 
-static void getSentenceBreaks(const ushort *string, quint32 len, HB_CharAttributes *attributes)
+static void getSentenceBreaks(const ushort *string, quint32 len, QCharAttributes *attributes)
 {
     uchar state = SB::BAfter; // to meet SB1
     for (quint32 i = 0; i != len; ++i) {
@@ -402,12 +406,11 @@ static const uchar breakTable[QUnicodeTables::LineBreak_CB + 1][QUnicodeTables::
 
 } // namespace LB
 
-static void getLineBreaks(const ushort *string, quint32 len, HB_CharAttributes *attributes)
+static void getLineBreaks(const ushort *string, quint32 len, QCharAttributes *attributes)
 {
     quint32 nestart = 0;
     LB::NS::Class nelast = LB::NS::XX;
 
-    uint lucs4 = 0;
     QUnicodeTables::LineBreakClass lcls = QUnicodeTables::LineBreak_LF; // to meet LB10
     QUnicodeTables::LineBreakClass cls = lcls;
     for (quint32 i = 0; i != len; ++i) {
@@ -443,7 +446,7 @@ static void getLineBreaks(const ushort *string, quint32 len, HB_CharAttributes *
             case LB::NS::Break:
                 // do not change breaks before and after the expression
                 for (quint32 j = nestart + 1; j < pos; ++j)
-                    attributes[j].lineBreakType = HB_NoBreak;
+                    attributes[j].lineBreak = false;
                 // fall through
             case LB::NS::None:
                 nelast = LB::NS::XX; // reset state
@@ -457,12 +460,10 @@ static void getLineBreaks(const ushort *string, quint32 len, HB_CharAttributes *
             }
         }
 
-        HB_LineBreakType lineBreakType = HB_NoBreak;
-
         if (Q_UNLIKELY(lcls >= QUnicodeTables::LineBreak_CR)) {
             // LB4: BK!, LB5: (CRxLF|CR|LF|NL)!
             if (lcls > QUnicodeTables::LineBreak_CR || ncls != QUnicodeTables::LineBreak_LF)
-                lineBreakType = HB_ForcedBreak;
+                attributes[pos].lineBreak = true;
             goto next;
         }
 
@@ -479,18 +480,16 @@ static void getLineBreaks(const ushort *string, quint32 len, HB_CharAttributes *
 
         switch (LB::breakTable[cls][ncls < QUnicodeTables::LineBreak_SA ? ncls : QUnicodeTables::LineBreak_AL]) {
         case LB::DirectBreak:
-            lineBreakType = HB_Break;
-            if (lucs4 == QChar::SoftHyphen)
-                lineBreakType = HB_SoftHyphen;
+            attributes[pos].lineBreak = true;
             break;
         case LB::IndirectBreak:
             if (lcls == QUnicodeTables::LineBreak_SP)
-                lineBreakType = HB_Break;
+                attributes[pos].lineBreak = true;
             break;
         case LB::CombiningIndirectBreak:
             if (lcls != QUnicodeTables::LineBreak_SP)
                 goto next_no_cls_update;
-            lineBreakType = HB_Break;
+            attributes[pos].lineBreak = true;
             break;
         case LB::CombiningProhibitedBreak:
             if (lcls != QUnicodeTables::LineBreak_SP)
@@ -504,24 +503,21 @@ static void getLineBreaks(const ushort *string, quint32 len, HB_CharAttributes *
 
     next:
         cls = ncls;
-        lucs4 = ucs4;
     next_no_cls_update:
         lcls = ncls;
-        if (Q_LIKELY(lineBreakType != HB_NoBreak))
-            attributes[pos].lineBreakType = lineBreakType;
     }
 
     if (Q_UNLIKELY(LB::NS::actionTable[nelast][LB::NS::XX] == LB::NS::Break)) {
         // LB25: do not break lines inside numbers
         for (quint32 j = nestart + 1; j < len; ++j)
-            attributes[j].lineBreakType = HB_NoBreak;
+            attributes[j].lineBreak = false;
     }
 
-    attributes[0].lineBreakType = HB_NoBreak; // LB2
+    attributes[0].lineBreak = false; // LB2
 }
 
 
-static void getWhiteSpaces(const ushort *string, quint32 len, HB_CharAttributes *attributes)
+static void getWhiteSpaces(const ushort *string, quint32 len, QCharAttributes *attributes)
 {
     for (quint32 i = 0; i != len; ++i) {
         uint ucs4 = string[i];
@@ -540,14 +536,14 @@ static void getWhiteSpaces(const ushort *string, quint32 len, HB_CharAttributes
 
 
 Q_CORE_EXPORT void initCharAttributes(const ushort *string, int length,
-                                      const HB_ScriptItem *items, int numItems,
-                                      HB_CharAttributes *attributes, CharAttributeOptions options)
+                                      const ScriptItem *items, int numItems,
+                                      QCharAttributes *attributes, CharAttributeOptions options)
 {
     if (length <= 0)
         return;
 
     if (!(options & DontClearAttributes))
-        ::memset(attributes, 0, length * sizeof(HB_CharAttributes));
+        ::memset(attributes, 0, length * sizeof(QCharAttributes));
 
     if (options & GraphemeBreaks)
         getGraphemeBreaks(string, length, attributes);
@@ -562,8 +558,34 @@ Q_CORE_EXPORT void initCharAttributes(const ushort *string, int length,
 
     if (!items || numItems <= 0)
         return;
-    if (!qt_initcharattributes_default_algorithm_only)
-        HB_GetTailoredCharAttributes(string, length, items, numItems, attributes);
+    if (!qt_initcharattributes_default_algorithm_only) {
+        QVarLengthArray<HB_ScriptItem, 64> scriptItems;
+        scriptItems.reserve(numItems);
+        int start = 0;
+        for (int i = start + 1; i < numItems; ++i) {
+            if (items[i].script == items[start].script)
+                continue;
+            HB_ScriptItem item;
+            item.pos = items[start].position;
+            item.length = items[i].position - items[start].position;
+            item.script = (HB_Script)items[start].script;
+            item.bidiLevel = 0; // unused
+            scriptItems.append(item);
+            start = i;
+        }
+        if (items[start].position + 1 < length) {
+            HB_ScriptItem item;
+            item.pos = items[start].position;
+            item.length = length - items[start].position;
+            item.script = (HB_Script)items[start].script;
+            item.bidiLevel = 0; // unused
+            scriptItems.append(item);
+        }
+        Q_STATIC_ASSERT(sizeof(QCharAttributes) == sizeof(HB_CharAttributes));
+        HB_GetTailoredCharAttributes(string, length,
+                                     scriptItems.constData(), scriptItems.size(),
+                                     reinterpret_cast<HB_CharAttributes *>(attributes));
+    }
 }
 
 } // namespace QUnicodeTools
diff --git a/src/corelib/tools/qunicodetools_p.h b/src/corelib/tools/qunicodetools_p.h
index ea407e7ddc..3396c33230 100644
--- a/src/corelib/tools/qunicodetools_p.h
+++ b/src/corelib/tools/qunicodetools_p.h
@@ -53,12 +53,30 @@
 // We mean it.
 //
 
-#include <private/qharfbuzz_p.h>
+#include <QtCore/qchar.h>
 
 QT_BEGIN_NAMESPACE
 
+struct Q_PACKED QCharAttributes
+{
+    uchar graphemeBoundary : 1;
+    uchar wordBreak        : 1;
+    uchar sentenceBoundary : 1;
+    uchar lineBreak        : 1;
+    uchar whiteSpace       : 1;
+    uchar unused           : 3;
+};
+Q_DECLARE_TYPEINFO(QCharAttributes, Q_PRIMITIVE_TYPE);
+
 namespace QUnicodeTools {
 
+// ### temporary
+struct ScriptItem
+{
+    int position;
+    int script;
+};
+
 enum CharAttributeOption {
     GraphemeBreaks = 0x01,
     WordBreaks = 0x02,
@@ -72,8 +90,8 @@ enum CharAttributeOption {
 Q_DECLARE_FLAGS(CharAttributeOptions, CharAttributeOption)
 
 Q_CORE_EXPORT void initCharAttributes(const ushort *string, int length,
-                                      const HB_ScriptItem *items, int numItems,
-                                      HB_CharAttributes *attributes, CharAttributeOptions options = DefaultOptionsCompat);
+                                      const ScriptItem *items, int numItems,
+                                      QCharAttributes *attributes, CharAttributeOptions options = DefaultOptionsCompat);
 
 } // namespace QUnicodeTools