Re-add some inline property storage

It turns out that not using any inline property storage
comes at a relatively high price in terms of memory
consumption, as we always need to also create a
memberData for any object.

This avoids the memberData creation in quite a few
cases, as we use the additional padding we have up
to the 32 byte boundary given by the memory manager
to store some property data.

This complicates property access somewhat. To avoid
performance regressions because of this, add specialized
QV4::Lookup functions that optimize for properties that
are inline or in the memberData struct.

Change seems to be performance neutral on v8-bench on
x86_64, but reduces peak memory usage when running the
benchmark by around 20%.

Change-Id: I0127d31a2d6038aaa540c4c4a1156f45ca3b7464
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
Reviewed-by: Robin Burchell <robin.burchell@crimson.no>

8 files changed, 196 insertions, 32 deletions

diff --git a/src/qml/jsruntime/qv4arraydata.cpp b/src/qml/jsruntime/qv4arraydata.cpp
index c29cedaa9b..6d1fb62e0a 100644
--- a/src/qml/jsruntime/qv4arraydata.cpp
+++ b/src/qml/jsruntime/qv4arraydata.cpp
@@ -50,6 +50,8 @@ QT_WARNING_SUPPRESS_GCC_TAUTOLOGICAL_COMPARE_ON
 
 const QV4::VTable QV4::ArrayData::static_vtbl = {
     0,
+    0,
+    0,
     QV4::ArrayData::IsExecutionContext,
     QV4::ArrayData::IsString,
     QV4::ArrayData::IsObject,
diff --git a/src/qml/jsruntime/qv4internalclass.cpp b/src/qml/jsruntime/qv4internalclass.cpp
index bac71b4537..7e3fd7dc12 100644
--- a/src/qml/jsruntime/qv4internalclass.cpp
+++ b/src/qml/jsruntime/qv4internalclass.cpp
@@ -43,6 +43,7 @@
 #include <qv4identifier_p.h>
 #include "qv4object_p.h"
 #include "qv4identifiertable_p.h"
+#include "qv4value_p.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -128,22 +129,48 @@ InternalClass::InternalClass(const QV4::InternalClass &other)
 
 static void insertHoleIntoPropertyData(Object *object, int idx)
 {
+    int inlineSize = object->d()->vt->nInlineProperties;
     int icSize = object->internalClass()->size;
-    int from = idx;
+    int from = qMax(idx, inlineSize);
     int to = from + 1;
-    if (from < icSize)
+    if (from < icSize) {
         memmove(object->propertyData(to), object->propertyData(from),
                 (icSize - from - 1) * sizeof(Value));
+    }
+    if (from == idx)
+        return;
+    if (inlineSize < icSize)
+        *object->propertyData(inlineSize) = *object->propertyData(inlineSize - 1);
+    from = idx;
+    to = from + 1;
+    if (from < inlineSize - 1) {
+        memmove(object->propertyData(to), object->propertyData(from),
+                (inlineSize - from - 1) * sizeof(Value));
+    }
 }
 
 static void removeFromPropertyData(Object *object, int idx, bool accessor = false)
 {
+    int inlineSize = object->d()->vt->nInlineProperties;
     int delta = (accessor ? 2 : 1);
     int oldSize = object->internalClass()->size + delta;
     int to = idx;
     int from = to + delta;
-    if (from < oldSize)
+    if (from < inlineSize) {
+        memmove(object->propertyData(to), object->d()->propertyData(from), (inlineSize - from)*sizeof(Value));
+        to = inlineSize - delta;
+        from = inlineSize;
+    }
+    if (to < inlineSize && from < oldSize) {
+        Q_ASSERT(from >= inlineSize);
+        memcpy(object->propertyData(to), object->d()->propertyData(from), (inlineSize - to)*sizeof(Value));
+        to = inlineSize;
+        from = inlineSize + delta;
+    }
+    if (from < oldSize) {
+        Q_ASSERT(to >= inlineSize && from > to);
         memmove(object->propertyData(to), object->d()->propertyData(from), (oldSize - to)*sizeof(Value));
+    }
 }
 
 void InternalClass::changeMember(Object *object, String *string, PropertyAttributes data, uint *index)
diff --git a/src/qml/jsruntime/qv4lookup.cpp b/src/qml/jsruntime/qv4lookup.cpp
index 52ed449664..80a24cdde7 100644
--- a/src/qml/jsruntime/qv4lookup.cpp
+++ b/src/qml/jsruntime/qv4lookup.cpp
@@ -307,15 +307,18 @@ ReturnedValue Lookup::getterTwoClasses(Lookup *l, ExecutionEngine *engine, const
 {
     Lookup l1 = *l;
 
-    if (l1.getter == Lookup::getter0 || l1.getter == Lookup::getter1) {
+    if (l1.getter == Lookup::getter0MemberData || l1.getter == Lookup::getter0Inline || l1.getter == Lookup::getter1) {
         if (const Object *o = object.as<Object>()) {
             ReturnedValue v = o->getLookup(l);
             Lookup l2 = *l;
 
-            if (l->index != UINT_MAX && (l2.getter == Lookup::getter0 || l2.getter == Lookup::getter1)) {
-                // if we have a getter0, make sure it comes first
-                if (l2.getter == Lookup::getter0)
-                    qSwap(l1, l2);
+            if (l2.index != UINT_MAX) {
+                if (l1.getter != Lookup::getter0Inline) {
+                    if (l2.getter == Lookup::getter0Inline ||
+                        (l1.getter != Lookup::getter0MemberData && l2.getter == Lookup::getter0MemberData))
+                        // sort the better getter first
+                        qSwap(l1, l2);
+                }
 
                 l->classList[0] = l1.classList[0];
                 l->classList[1] = l1.classList[1];
@@ -324,8 +327,22 @@ ReturnedValue Lookup::getterTwoClasses(Lookup *l, ExecutionEngine *engine, const
                 l->index = l1.index;
                 l->index2 = l2.index;
 
-                if (l1.getter == Lookup::getter0) {
-                    l->getter = (l2.getter == Lookup::getter0) ? Lookup::getter0getter0 : Lookup::getter0getter1;
+                if (l1.getter == Lookup::getter0Inline) {
+                    if (l2.getter == Lookup::getter0Inline)
+                        l->getter = Lookup::getter0Inlinegetter0Inline;
+                    else if (l2.getter == Lookup::getter0MemberData)
+                        l->getter = Lookup::getter0Inlinegetter0MemberData;
+                    else if (l2.getter == Lookup::getter1)
+                        l->getter = Lookup::getter0Inlinegetter1;
+                    else
+                        Q_UNREACHABLE();
+                } else if (l1.getter == Lookup::getter0MemberData) {
+                    if (l2.getter == Lookup::getter0MemberData)
+                        l->getter = Lookup::getter0MemberDatagetter0MemberData;
+                    else if (l2.getter == Lookup::getter1)
+                        l->getter = Lookup::getter0MemberDatagetter1;
+                    else
+                        Q_UNREACHABLE();
                 } else {
                     Q_ASSERT(l1.getter == Lookup::getter1 && l2.getter == Lookup::getter1);
                     l->getter = Lookup::getter1getter1;
@@ -349,14 +366,26 @@ ReturnedValue Lookup::getterFallback(Lookup *l, ExecutionEngine *engine, const V
     return o->get(name);
 }
 
-ReturnedValue Lookup::getter0(Lookup *l, ExecutionEngine *engine, const Value &object)
+ReturnedValue Lookup::getter0MemberData(Lookup *l, ExecutionEngine *engine, const Value &object)
 {
     // we can safely cast to a QV4::Object here. If object is actually a string,
     // the internal class won't match
     Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
     if (o) {
         if (l->classList[0] == o->internalClass)
-            return o->propertyData(l->index)->asReturnedValue();
+            return o->memberData->data[l->index].asReturnedValue();
+    }
+    return getterTwoClasses(l, engine, object);
+}
+
+ReturnedValue Lookup::getter0Inline(Lookup *l, ExecutionEngine *engine, const Value &object)
+{
+    // we can safely cast to a QV4::Object here. If object is actually a string,
+    // the internal class won't match
+    Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
+    if (o) {
+        if (l->classList[0] == o->internalClass)
+            return o->inlinePropertyData(l->index)->asReturnedValue();
     }
     return getterTwoClasses(l, engine, object);
 }
@@ -392,29 +421,74 @@ ReturnedValue Lookup::getter2(Lookup *l, ExecutionEngine *engine, const Value &o
     return getterFallback(l, engine, object);
 }
 
-ReturnedValue Lookup::getter0getter0(Lookup *l, ExecutionEngine *engine, const Value &object)
+ReturnedValue Lookup::getter0Inlinegetter0Inline(Lookup *l, ExecutionEngine *engine, const Value &object)
 {
     // we can safely cast to a QV4::Object here. If object is actually a string,
     // the internal class won't match
     Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
     if (o) {
         if (l->classList[0] == o->internalClass)
-            return o->propertyData(l->index)->asReturnedValue();
+            return o->inlinePropertyData(l->index)->asReturnedValue();
         if (l->classList[2] == o->internalClass)
-            return o->propertyData(l->index2)->asReturnedValue();
+            return o->inlinePropertyData(l->index2)->asReturnedValue();
     }
     l->getter = getterFallback;
     return getterFallback(l, engine, object);
 }
 
-ReturnedValue Lookup::getter0getter1(Lookup *l, ExecutionEngine *engine, const Value &object)
+ReturnedValue Lookup::getter0Inlinegetter0MemberData(Lookup *l, ExecutionEngine *engine, const Value &object)
 {
     // we can safely cast to a QV4::Object here. If object is actually a string,
     // the internal class won't match
     Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
     if (o) {
         if (l->classList[0] == o->internalClass)
-            return o->propertyData(l->index)->asReturnedValue();
+            return o->inlinePropertyData(l->index)->asReturnedValue();
+        if (l->classList[2] == o->internalClass)
+            return o->memberData->data[l->index2].asReturnedValue();
+    }
+    l->getter = getterFallback;
+    return getterFallback(l, engine, object);
+}
+
+ReturnedValue Lookup::getter0MemberDatagetter0MemberData(Lookup *l, ExecutionEngine *engine, const Value &object)
+{
+    // we can safely cast to a QV4::Object here. If object is actually a string,
+    // the internal class won't match
+    Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
+    if (o) {
+        if (l->classList[0] == o->internalClass)
+            return o->memberData->data[l->index].asReturnedValue();
+        if (l->classList[2] == o->internalClass)
+            return o->memberData->data[l->index2].asReturnedValue();
+    }
+    l->getter = getterFallback;
+    return getterFallback(l, engine, object);
+}
+
+ReturnedValue Lookup::getter0Inlinegetter1(Lookup *l, ExecutionEngine *engine, const Value &object)
+{
+    // we can safely cast to a QV4::Object here. If object is actually a string,
+    // the internal class won't match
+    Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
+    if (o) {
+        if (l->classList[0] == o->internalClass)
+            return o->inlinePropertyData(l->index)->asReturnedValue();
+        if (l->classList[2] == o->internalClass && l->classList[3] == o->prototype->internalClass)
+            return o->prototype->propertyData(l->index2)->asReturnedValue();
+    }
+    l->getter = getterFallback;
+    return getterFallback(l, engine, object);
+}
+
+ReturnedValue Lookup::getter0MemberDatagetter1(Lookup *l, ExecutionEngine *engine, const Value &object)
+{
+    // we can safely cast to a QV4::Object here. If object is actually a string,
+    // the internal class won't match
+    Heap::Object *o = static_cast<Heap::Object *>(object.heapObject());
+    if (o) {
+        if (l->classList[0] == o->internalClass)
+            return o->memberData->data[l->index].asReturnedValue();
         if (l->classList[2] == o->internalClass && l->classList[3] == o->prototype->internalClass)
             return o->prototype->propertyData(l->index2)->asReturnedValue();
     }
@@ -604,9 +678,15 @@ ReturnedValue Lookup::globalGetterGeneric(Lookup *l, ExecutionEngine *engine)
     ReturnedValue v = l->lookup(o, &attrs);
     if (v != Primitive::emptyValue().asReturnedValue()) {
         if (attrs.isData()) {
-            if (l->level == 0)
-                l->globalGetter = globalGetter0;
-            else if (l->level == 1)
+            if (l->level == 0) {
+                uint nInline = o->d()->vt->nInlineProperties;
+                if (l->index < nInline)
+                    l->globalGetter = globalGetter0Inline;
+                else {
+                    l->index -= nInline;
+                    l->globalGetter = globalGetter0MemberData;
+                }
+            } else if (l->level == 1)
                 l->globalGetter = globalGetter1;
             else if (l->level == 2)
                 l->globalGetter = globalGetter2;
@@ -626,11 +706,21 @@ ReturnedValue Lookup::globalGetterGeneric(Lookup *l, ExecutionEngine *engine)
     return engine->throwReferenceError(n);
 }
 
-ReturnedValue Lookup::globalGetter0(Lookup *l, ExecutionEngine *engine)
+ReturnedValue Lookup::globalGetter0Inline(Lookup *l, ExecutionEngine *engine)
+{
+    Object *o = engine->globalObject;
+    if (l->classList[0] == o->internalClass())
+        return o->d()->inlinePropertyData(l->index)->asReturnedValue();
+
+    l->globalGetter = globalGetterGeneric;
+    return globalGetterGeneric(l, engine);
+}
+
+ReturnedValue Lookup::globalGetter0MemberData(Lookup *l, ExecutionEngine *engine)
 {
     Object *o = engine->globalObject;
     if (l->classList[0] == o->internalClass())
-        return o->propertyData(l->index)->asReturnedValue();
+        return o->d()->memberData->data[l->index].asReturnedValue();
 
     l->globalGetter = globalGetterGeneric;
     return globalGetterGeneric(l, engine);
diff --git a/src/qml/jsruntime/qv4lookup_p.h b/src/qml/jsruntime/qv4lookup_p.h
index 2ffb43cce9..5a67c203ce 100644
--- a/src/qml/jsruntime/qv4lookup_p.h
+++ b/src/qml/jsruntime/qv4lookup_p.h
@@ -102,11 +102,15 @@ struct Lookup {
     static ReturnedValue getterTwoClasses(Lookup *l, ExecutionEngine *engine, const Value &object);
     static ReturnedValue getterFallback(Lookup *l, ExecutionEngine *engine, const Value &object);
 
-    static ReturnedValue getter0(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0MemberData(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0Inline(Lookup *l, ExecutionEngine *engine, const Value &object);
     static ReturnedValue getter1(Lookup *l, ExecutionEngine *engine, const Value &object);
     static ReturnedValue getter2(Lookup *l, ExecutionEngine *engine, const Value &object);
-    static ReturnedValue getter0getter0(Lookup *l, ExecutionEngine *engine, const Value &object);
-    static ReturnedValue getter0getter1(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0Inlinegetter0Inline(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0Inlinegetter0MemberData(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0MemberDatagetter0MemberData(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0Inlinegetter1(Lookup *l, ExecutionEngine *engine, const Value &object);
+    static ReturnedValue getter0MemberDatagetter1(Lookup *l, ExecutionEngine *engine, const Value &object);
     static ReturnedValue getter1getter1(Lookup *l, ExecutionEngine *engine, const Value &object);
     static ReturnedValue getterAccessor0(Lookup *l, ExecutionEngine *engine, const Value &object);
     static ReturnedValue getterAccessor1(Lookup *l, ExecutionEngine *engine, const Value &object);
@@ -120,7 +124,8 @@ struct Lookup {
     static ReturnedValue arrayLengthGetter(Lookup *l, ExecutionEngine *engine, const Value &object);
 
     static ReturnedValue globalGetterGeneric(Lookup *l, ExecutionEngine *engine);
-    static ReturnedValue globalGetter0(Lookup *l, ExecutionEngine *engine);
+    static ReturnedValue globalGetter0Inline(Lookup *l, ExecutionEngine *engine);
+    static ReturnedValue globalGetter0MemberData(Lookup *l, ExecutionEngine *engine);
     static ReturnedValue globalGetter1(Lookup *l, ExecutionEngine *engine);
     static ReturnedValue globalGetter2(Lookup *l, ExecutionEngine *engine);
     static ReturnedValue globalGetterAccessor0(Lookup *l, ExecutionEngine *engine);
diff --git a/src/qml/jsruntime/qv4managed.cpp b/src/qml/jsruntime/qv4managed.cpp
index 3a84a83b9c..200380eda0 100644
--- a/src/qml/jsruntime/qv4managed.cpp
+++ b/src/qml/jsruntime/qv4managed.cpp
@@ -47,6 +47,8 @@ using namespace QV4;
 const VTable Managed::static_vtbl =
 {
     0,
+    0,
+    0,
     Managed::IsExecutionContext,
     Managed::IsString,
     Managed::IsObject,
diff --git a/src/qml/jsruntime/qv4managed_p.h b/src/qml/jsruntime/qv4managed_p.h
index 5c764e7ff0..7e674c6ec7 100644
--- a/src/qml/jsruntime/qv4managed_p.h
+++ b/src/qml/jsruntime/qv4managed_p.h
@@ -129,6 +129,9 @@ inline void qYouForgotTheQ_MANAGED_Macro(T1, T2) {}
 #define DEFINE_MANAGED_VTABLE_INT(classname, parentVTable) \
 {     \
     parentVTable, \
+    (sizeof(classname::Data) + sizeof(QV4::Value) - 1)/sizeof(QV4::Value), \
+    (sizeof(classname::Data) + QV4::Chunk::SlotSize - 1)/QV4::Chunk::SlotSize*QV4::Chunk::SlotSize/sizeof(QV4::Value) \
+        - (sizeof(classname::Data) + sizeof(QV4::Value) - 1)/sizeof(QV4::Value), \
     classname::IsExecutionContext,   \
     classname::IsString,   \
     classname::IsObject,   \
diff --git a/src/qml/jsruntime/qv4object.cpp b/src/qml/jsruntime/qv4object.cpp
index 12157af728..9595158ce9 100644
--- a/src/qml/jsruntime/qv4object.cpp
+++ b/src/qml/jsruntime/qv4object.cpp
@@ -61,9 +61,13 @@ DEFINE_OBJECT_VTABLE(Object);
 void Object::setInternalClass(InternalClass *ic)
 {
     d()->internalClass = ic;
+    uint nInline = d()->vtable()->nInlineProperties;
+    if (ic->size <= nInline)
+        return;
     bool hasMD = d()->memberData != nullptr;
-    if ((!hasMD && ic->size) || (hasMD && d()->memberData->size < ic->size))
-        d()->memberData = MemberData::allocate(ic->engine, ic->size, d()->memberData);
+    uint requiredSize = ic->size - nInline;
+    if (!hasMD || (hasMD && d()->memberData->size < requiredSize))
+        d()->memberData = MemberData::allocate(ic->engine, requiredSize, d()->memberData);
 }
 
 void Object::getProperty(uint index, Property *p, PropertyAttributes *attrs) const
@@ -261,6 +265,13 @@ void Object::markObjects(Heap::Base *that, ExecutionEngine *e)
         o->arrayData->mark(e);
     if (o->prototype)
         o->prototype->mark(e);
+    uint nInline = o->vtable()->nInlineProperties;
+    Value *v = reinterpret_cast<Value *>(o) + o->vt->inlinePropertyOffset;
+    const Value *end = v + nInline;
+    while (v < end) {
+        v->mark(e);
+        ++v;
+    }
 }
 
 void Object::insertMember(String *s, const Property *p, PropertyAttributes attributes)
@@ -495,8 +506,15 @@ ReturnedValue Object::getLookup(const Managed *m, Lookup *l)
     ReturnedValue v = l->lookup(o, &attrs);
     if (v != Primitive::emptyValue().asReturnedValue()) {
         if (attrs.isData()) {
-            if (l->level == 0)
-                l->getter = Lookup::getter0;
+            if (l->level == 0) {
+                uint nInline = o->d()->vt->nInlineProperties;
+                if (l->index < nInline)
+                    l->getter = Lookup::getter0Inline;
+                else {
+                    l->index -= nInline;
+                    l->getter = Lookup::getter0MemberData;
+                }
+                }
             else if (l->level == 1)
                 l->getter = Lookup::getter1;
             else if (l->level == 2)
diff --git a/src/qml/jsruntime/qv4object_p.h b/src/qml/jsruntime/qv4object_p.h
index 6a543ae1a8..c1d5fd66b2 100644
--- a/src/qml/jsruntime/qv4object_p.h
+++ b/src/qml/jsruntime/qv4object_p.h
@@ -71,8 +71,25 @@ struct Object : Base {
     void init() { Base::init(); }
     void destroy() { Base::destroy(); }
 
-    const Value *propertyData(uint index) const { return memberData->data + index; }
-    Value *propertyData(uint index) { return memberData->data + index; }
+    const Value *inlinePropertyData(uint index) const {
+        Q_ASSERT(index < vt->nInlineProperties);
+        return reinterpret_cast<const Value *>(this) + vt->inlinePropertyOffset + index;
+    }
+
+    const Value *propertyData(uint index) const {
+        uint nInline = vt->nInlineProperties;
+        if (index < nInline)
+            return reinterpret_cast<const Value *>(this) + vt->inlinePropertyOffset + index;
+        index -= nInline;
+        return memberData->data + index;
+    }
+    Value *propertyData(uint index) {
+        uint nInline = vt->nInlineProperties;
+        if (index < nInline)
+            return reinterpret_cast<Value *>(this) + vt->inlinePropertyOffset + index;
+        index -= nInline;
+        return memberData->data + index;
+    }
 
     InternalClass *internalClass;
     Pointer<Object> prototype;