Add a Latin1String variant and use it for values

String based values will now be stored in 8 bit
if they are short enough and latin1 only. This should
significantly reduce memory usage.

Change-Id: Ib0c7f3bec4aa669572f742752c13dcfca7083625
Reviewed-by: Hicks James <jamey.hicks@nokia.com>

6 files changed, 139 insertions, 27 deletions

diff --git a/src/qjson_p.h b/src/qjson_p.h
index 2ef94c7..47da9e4 100644
--- a/src/qjson_p.h
+++ b/src/qjson_p.h
@@ -71,9 +71,11 @@ static inline int qStringSize(const QString &string)
     return (sizeof(ushort)*string.length() + 3) & ~3;
 }
 
-struct QStackString
+struct Latin1String;
+
+struct String
 {
-    QStackString(const ushort *utf16, int length)
+    String(const ushort *utf16, int length)
         : length(length), utf16(utf16) {}
     int length;
     const ushort *utf16;
@@ -90,20 +92,91 @@ struct QStackString
         return (l == -1);
     }
 
+    inline bool equals(const uchar *s, int slen) const
+    {
+        if (slen != length)
+            return false;
+        int l = length;
+        const ushort *a = utf16;
+        const uchar *b = s;
+        while (l-- && *a == *b)
+            a++,b++;
+        return (l == -1);
+    }
+
+    bool operator ==(const QString &str) {
+        return equals((const ushort *)str.constData(), str.length());
+    }
+    bool operator !=(const QString &str) {
+        return !equals((const ushort *)str.constData(), str.length());
+    }
+    bool operator ==(const String &str) {
+        return equals(str.utf16, str.length);
+    }
+    bool operator !=(const String &str) {
+        return !equals(str.utf16, str.length);
+    }
+    inline bool operator ==(const Latin1String &str);
+    inline bool operator !=(const Latin1String &str);
+};
+
+struct Latin1String
+{
+    Latin1String(const uchar *latin1, int length)
+        : length(length), latin1(latin1) {}
+    short length;
+    const uchar *latin1;
+
+    inline bool equals(const ushort *s, int slen) const
+    {
+        if (slen != length)
+            return false;
+        int l = length;
+        const uchar *a = latin1;
+        const ushort *b = s;
+        while (l-- && *a == *b)
+            a++,b++;
+        return (l == -1);
+    }
+
+    inline bool equals(const uchar *s, int slen) const
+    {
+        if (slen != length)
+            return false;
+        int l = length;
+        const uchar *a = latin1;
+        const uchar *b = s;
+        while (l-- && *a == *b)
+            a++,b++;
+        return (l == -1);
+    }
+
     bool operator ==(const QString &str) {
         return equals((const ushort *)str.constData(), str.length());
     }
     bool operator !=(const QString &str) {
         return !equals((const ushort *)str.constData(), str.length());
     }
-    bool operator ==(const QStackString &str) {
+    bool operator ==(const String &str) {
         return equals(str.utf16, str.length);
     }
-    bool operator !=(const QStackString &str) {
+    bool operator !=(const String &str) {
         return !equals(str.utf16, str.length);
     }
+    bool operator ==(const Latin1String &str) {
+        return equals(str.latin1, str.length);
+    }
+    bool operator !=(const Latin1String &str) {
+        return !equals(str.latin1, str.length);
+    }
 };
 
+inline bool String::operator ==(const Latin1String &str) {
+    return equals(str.latin1, str.length);
+}
+inline bool String::operator !=(const Latin1String &str) {
+    return !equals(str.latin1, str.length);
+}
 
 struct Base
 {
@@ -169,7 +242,8 @@ struct Value
     double toNumber(const Base *b) const;
 //    int toInt() const;
     QString toString(const Base *b) const;
-    QStackString shallowString(const Base *b) const;
+    String asString(const Base *b) const;
+    Latin1String asLatin1String(const Base *b) const;
     Array *array(const Base *b) const;
     Object *object(const Base *b) const;
 
@@ -225,7 +299,7 @@ struct Entry {
     // keydata
     // value data follows key data
 
-    QStackString shallowKey() const { return QStackString(keyData(), keyLength); }
+    String shallowKey() const { return String(keyData(), keyLength); }
     QString key() const
     {
         return QString((const QChar *)keyData(), keyLength);
@@ -273,17 +347,31 @@ inline double Value::toNumber(const Base *b) const
 inline QString Value::toString(const Base *b) const
 {
     char *d = data(b);
+    if (compressed) {
+        int l = *(ushort *)d;
+        return QString::fromLatin1(d + sizeof(ushort), l);
+    }
     int l = *(int *)d;
     const QChar *c = (const QChar *)(d + sizeof(int));
     return QString(c, l);
 }
 
-inline QStackString Value::shallowString(const Base *b) const
+inline String Value::asString(const Base *b) const
 {
+    Q_ASSERT(type == StringValue && !compressed);
     char *d = data(b);
     int l = *(int *)d;
     const ushort *c = (const ushort *)(d + sizeof(int));
-    return QStackString(c, l);
+    return String(c, l);
+}
+
+inline Latin1String Value::asLatin1String(const Base *b) const
+{
+    Q_ASSERT(type == StringValue && compressed);
+    char *d = data(b);
+    ushort l = *(ushort *)d;
+    const uchar *c = (const uchar *)(d + sizeof(ushort));
+    return Latin1String(c, l);
 }
 
 inline Array *Value::array(const Base *b) const
diff --git a/src/qjsonarray.cpp b/src/qjsonarray.cpp
index 5266b30..6c54cb6 100644
--- a/src/qjsonarray.cpp
+++ b/src/qjsonarray.cpp
@@ -143,7 +143,8 @@ void JsonArray::insert(int i, const JsonValue &value)
 {
     Q_ASSERT (i >= 0 && i <= (int)(a ? a->length : 0));
 
-    int valueSize = value.requiredStorage();
+    bool compressed;
+    int valueSize = value.requiredStorage(&compressed);
 
     detach(valueSize + sizeof(offset) + sizeof(Value)); // offset for the new index entry
 
@@ -153,10 +154,10 @@ void JsonArray::insert(int i, const JsonValue &value)
     int valueOffset = a->reserveSpace(valueSize, i, 1);
     Value &v = (*a)[i];
     v.type = value.t;
-    v.compressed = false;
+    v.compressed = compressed;
     v.val = value.valueToStore(valueOffset);
     if (valueSize)
-        value.copyData((char *)a + valueOffset);
+        value.copyData((char *)a + valueOffset, compressed);
 }
 
 bool JsonArray::contains(const JsonValue &element) const
diff --git a/src/qjsonobject.cpp b/src/qjsonobject.cpp
index 1dbdaa7..c975dd9 100644
--- a/src/qjsonobject.cpp
+++ b/src/qjsonobject.cpp
@@ -136,7 +136,8 @@ JsonValue JsonObject::value(const QString &key) const
 
 void JsonObject::insert(const QString &key, const JsonValue &value)
 {
-    int valueSize = value.requiredStorage();
+    bool compressed;
+    int valueSize = value.requiredStorage(&compressed);
 
     int valueOffset = sizeof(Entry) + qStringSize(key);
     int requiredSize = valueOffset + valueSize;
@@ -156,10 +157,10 @@ void JsonObject::insert(const QString &key, const JsonValue &value)
     e->keyLength = key.length();
     memcpy(e->keyData(), key.constData(), sizeof(ushort)*key.length());
     e->value.type = value.t;
-    e->value.compressed = false;
+    e->value.compressed = compressed;
     e->value.val = value.valueToStore(d->offsetOf(e) + valueOffset);
     if (valueSize)
-        value.copyData((char *)e + valueOffset);
+        value.copyData((char *)e + valueOffset, compressed);
 }
 
 void JsonObject::remove(const QString &key)
diff --git a/src/qjsonvalue.cpp b/src/qjsonvalue.cpp
index b5da375..08c3d08 100644
--- a/src/qjsonvalue.cpp
+++ b/src/qjsonvalue.cpp
@@ -322,13 +322,27 @@ void JsonValue::detach()
     object = d->_object;
 }
 
-int JsonValue::requiredStorage() const
+int JsonValue::requiredStorage(bool *compressed) const
 {
+    *compressed = false;
     switch (t) {
     case NumberValue:
         return sizeof(double);
     case StringValue:
-        return sizeof(uint) + sizeof(ushort)*stringData->size;
+        if (stringData->size < 0x8000) {
+            QString s = toString();
+            const ushort *uc = (const ushort *)s.constData();
+            const ushort *e = uc + s.length();
+            *compressed = true;
+            while (uc < e) {
+                if (*uc > 0xff) {
+                    *compressed = false;
+                    break;
+                }
+                ++uc;
+            }
+        }
+        return *compressed ? sizeof(ushort) + stringData->size : sizeof(uint) + sizeof(ushort)*stringData->size;
     case ArrayValue:
         return array ? array->size : sizeof(Array);
     case ObjectValue:
@@ -356,7 +370,7 @@ uint JsonValue::valueToStore(uint offset) const
     return 0;
 }
 
-void JsonValue::copyData(char *dest) const
+void JsonValue::copyData(char *dest, bool compressed) const
 {
     switch (t) {
     case NumberValue:
@@ -364,8 +378,16 @@ void JsonValue::copyData(char *dest) const
         break;
     case StringValue: {
         QString str = toString();
-        *((int *)dest) = str.length();
-        memcpy(dest + sizeof(int), str.constData(), str.length()*sizeof(ushort));
+        if (compressed) {
+            *((ushort *)dest) = str.length();
+            uchar *d = (uchar *)dest + sizeof(ushort);
+            const ushort *uc = (const ushort *)str.unicode();
+            for (int i = 0; i < str.length(); ++i)
+                *d++ = uc[i];
+        } else {
+            *((int *)dest) = str.length();
+            memcpy(dest + sizeof(int), str.constData(), str.length()*sizeof(ushort));
+        }
         break;
     }
     case ArrayValue:
diff --git a/src/qjsonvalue.h b/src/qjsonvalue.h
index 2c90f04..74f2702 100644
--- a/src/qjsonvalue.h
+++ b/src/qjsonvalue.h
@@ -56,9 +56,9 @@ private:
     friend class JsonArray;
     friend class JsonObject;
     JsonValue(Data *d, Base *b, const Value& v);
-    int requiredStorage() const;
+    int requiredStorage(bool *compressed) const;
     uint valueToStore(uint offset) const;
-    void copyData(char *dest) const;
+    void copyData(char *dest, bool compressed) const;
 
     ValueType t;
     Data *d; // needed for Object and Array
diff --git a/src/qjsonwriter.cpp b/src/qjsonwriter.cpp
index 394f66b..ce162ba 100644
--- a/src/qjsonwriter.cpp
+++ b/src/qjsonwriter.cpp
@@ -26,16 +26,16 @@ static inline uchar hexdig(uint u)
     return (u < 0xa ? '0' + u : 'a' + u - 0xa);
 }
 
-static QByteArray escapedString(const QStackString &s)
+static QByteArray escapedString(const QString &s)
 {
     const uchar replacement = '?';
-    QByteArray ba(s.length, Qt::Uninitialized);
+    QByteArray ba(s.length(), Qt::Uninitialized);
 
     uchar *cursor = (uchar *)ba.data();
     const uchar *ba_end = cursor + ba.length();
 
-    const QChar *ch = (const QChar *)s.utf16;
-    const QChar *end = ch + s.length;
+    const QChar *ch = (const QChar *)s.constData();
+    const QChar *end = ch + s.length();
 
     int surrogate_high = -1;
 
@@ -146,7 +146,7 @@ static void valueToJson(const Base *b, const Value &v, QByteArray &json, int ind
         break;
     case StringValue:
         json += '"';
-        json += escapedString(v.shallowString(b));
+        json += escapedString(v.toString(b));
         json += '"';
         break;
     case ArrayValue:
@@ -201,7 +201,7 @@ static void objectContentToJson(const Object *o, QByteArray &json, int indent)
         Entry *e = o->entryAt(i);
         json += indentString;
         json += '"';
-        json += escapedString(e->shallowKey());
+        json += escapedString(e->key());
         json += "\": ";
         valueToJson(o, e->value, json, indent);