Debugger: always hex encode the value of assignments in cdbext

Fixes assigning negative values to locals

Change-Id: Ief6e7f47e8e6f0a5d38458396164dfcd24e408a5
Fixes: QTCREATORBUG-17269
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

9 files changed, 53 insertions, 171 deletions

diff --git a/src/libs/qtcreatorcdbext/qtcreatorcdbextension.cpp b/src/libs/qtcreatorcdbext/qtcreatorcdbextension.cpp
index fad25230a3..6bbd71367c 100644
--- a/src/libs/qtcreatorcdbext/qtcreatorcdbextension.cpp
+++ b/src/libs/qtcreatorcdbext/qtcreatorcdbextension.cpp
@@ -820,7 +820,7 @@ extern "C" HRESULT CALLBACK assign(CIDebugClient *client, PCSTR argsIn)
 
     std::string errorMessage;
     bool success = false;
-    AssignEncoding enc = AssignPlainValue;
+    bool encoded = false;
     int token = 0;
     do {
         StringList tokens = commandTokens<StringList>(argsIn, &token);
@@ -830,10 +830,7 @@ extern "C" HRESULT CALLBACK assign(CIDebugClient *client, PCSTR argsIn)
         }
 
         if (tokens.front() == "-h") {
-            enc = AssignHexEncoded;
-            tokens.pop_front();
-        } else if (tokens.front() == "-u") {
-            enc = AssignHexEncodedUtf16;
+            encoded = true;
             tokens.pop_front();
         }
 
@@ -864,7 +861,7 @@ extern "C" HRESULT CALLBACK assign(CIDebugClient *client, PCSTR argsIn)
         SymbolGroup *symGroup = ExtensionContext::instance().symbolGroup(exc.symbols(), exc.threadId(), currentFrame, &errorMessage);
         if (!symGroup)
             break;
-        success = symGroup->assign(iname, enc, value,
+        success = symGroup->assign(iname, encoded ? stringFromHex(value) : value,
                                    SymbolGroupValueContext(exc.dataSpaces(), exc.symbols()),
                                    &errorMessage);
     } while (false);
diff --git a/src/libs/qtcreatorcdbext/stringutils.cpp b/src/libs/qtcreatorcdbext/stringutils.cpp
index cd4eb683b0..7a8554f91c 100644
--- a/src/libs/qtcreatorcdbext/stringutils.cpp
+++ b/src/libs/qtcreatorcdbext/stringutils.cpp
@@ -33,6 +33,7 @@
 #include <cstring>
 #include <iostream>
 #include <sstream>
+#include <codecvt>
 #include <iomanip>
 
 static const char whiteSpace[] = " \t\r\n";
@@ -149,6 +150,11 @@ std::string wStringToString(const std::wstring &w)
     return rc;
 }
 
+std::wstring utf8ToUtf16(const std::string &s)
+{
+    return std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(s.data());
+}
+
 // Convert an ASCII hex digit to its value 'A'->10
 static inline unsigned hexDigit(char c)
 {
@@ -210,6 +216,11 @@ std::string stringFromHex(const char *p, const char *end)
     return rc;
 }
 
+std::string stringFromHex(const std::string &hexEncoded)
+{
+    return stringFromHex(hexEncoded.c_str(), hexEncoded.c_str() + hexEncoded.size());
+}
+
 // Helper for dumping memory
 std::string dumpMemory(const unsigned char *p, size_t size,
                        bool wantQuotes)
diff --git a/src/libs/qtcreatorcdbext/stringutils.h b/src/libs/qtcreatorcdbext/stringutils.h
index e8297d44a4..f891cf3ae7 100644
--- a/src/libs/qtcreatorcdbext/stringutils.h
+++ b/src/libs/qtcreatorcdbext/stringutils.h
@@ -167,6 +167,7 @@ inline std::ostream &operator<<(std::ostream &str, const gdbmiWStringFormat &wsf
 }
 
 std::string wStringToString(const std::wstring &w);
+std::wstring utf8ToUtf16(const std::string &s);
 
 // Strings from raw data.
 std::wstring quotedWStringFromCharData(const unsigned char *data, size_t size, bool truncated = false);
@@ -177,6 +178,7 @@ std::string dumpMemory(const unsigned char *data, size_t size, bool wantQuotes =
 
 // String from hex "414A" -> "AJ".
 std::string stringFromHex(const char *begin, const char *end);
+std::string stringFromHex(const std::string &hexEncoded);
 // Decode hex to a memory area.
 void decodeHex(const char *begin, const char *end, unsigned char *target);
 
diff --git a/src/libs/qtcreatorcdbext/symbolgroup.cpp b/src/libs/qtcreatorcdbext/symbolgroup.cpp
index 8404f0d77c..0d55fb6a39 100644
--- a/src/libs/qtcreatorcdbext/symbolgroup.cpp
+++ b/src/libs/qtcreatorcdbext/symbolgroup.cpp
@@ -449,7 +449,6 @@ void SymbolGroup::markUninitialized(const std::vector<std::string> &uniniNodes)
 }
 
 bool SymbolGroup::assign(const std::string &nodeName,
-                         int valueEncoding,
                          const std::string &value,
                          const SymbolGroupValueContext &ctx,
                          std::string *errorMessage)
@@ -468,9 +467,9 @@ bool SymbolGroup::assign(const std::string &nodeName,
     int kt = node->dumperType();
     if (kt < 0)
         kt = knownType(node->type(), KnownTypeAutoStripPointer | KnownTypeHasClassPrefix);
-    return (kt & KT_Editable) ? // Edit complex types
-        assignType(node, kt, valueEncoding, value, ctx, errorMessage) :
-        node->assign(value, errorMessage);
+    if (kt & KT_Editable)
+        return assignType(node, kt, value, ctx, errorMessage);
+    return node->assign(value, errorMessage);
 }
 
 bool SymbolGroup::accept(SymbolGroupNodeVisitor &visitor) const
diff --git a/src/libs/qtcreatorcdbext/symbolgroup.h b/src/libs/qtcreatorcdbext/symbolgroup.h
index 20281a500b..92ac15238f 100644
--- a/src/libs/qtcreatorcdbext/symbolgroup.h
+++ b/src/libs/qtcreatorcdbext/symbolgroup.h
@@ -96,7 +96,6 @@ public:
 
     // Assign a value by iname
     bool assign(const std::string &node,
-                int valueEncoding,
                 const std::string &value,
                 const SymbolGroupValueContext &ctx,
                 std::string *errorMessage);
diff --git a/src/libs/qtcreatorcdbext/symbolgroupnode.cpp b/src/libs/qtcreatorcdbext/symbolgroupnode.cpp
index d595a5925b..9ac8b821bf 100644
--- a/src/libs/qtcreatorcdbext/symbolgroupnode.cpp
+++ b/src/libs/qtcreatorcdbext/symbolgroupnode.cpp
@@ -1432,10 +1432,11 @@ std::string SymbolGroupNode::msgAssignError(const std::string &nodeName,
 }
 
 // Simple type
-bool SymbolGroupNode::assign(const std::string &value, std::string *errorMessage /* = 0 */)
+bool SymbolGroupNode::assign(const std::string &value,
+                             std::string *errorMessage /* = 0 */)
 {
     const HRESULT hr =
-        m_symbolGroup->debugSymbolGroup()->WriteSymbol(m_index, const_cast<char *>(value.c_str()));
+        m_symbolGroup->debugSymbolGroup()->WriteSymbol(m_index, value.c_str());
     if (FAILED(hr)) {
         if (errorMessage)
             *errorMessage = SymbolGroupNode::msgAssignError(name(), value, msgDebugEngineComFailed("WriteSymbol", hr));
diff --git a/src/libs/qtcreatorcdbext/symbolgroupvalue.cpp b/src/libs/qtcreatorcdbext/symbolgroupvalue.cpp
index 14c3732678..e06cf91fe0 100644
--- a/src/libs/qtcreatorcdbext/symbolgroupvalue.cpp
+++ b/src/libs/qtcreatorcdbext/symbolgroupvalue.cpp
@@ -3309,101 +3309,6 @@ static inline std::vector<AbstractSymbolGroupNode *>
     return rc;
 }
 
-/* AssignmentStringData: Helper struct used for assigning values
- * to string classes. Contains an (allocated) data array with size for use
- * with IDebugDataSpaced::FillVirtual() + string length information and
- * provides a conversion function decodeString() to create the array
- * depending on the argument format (blow up ASCII to UTF16 or vice versa). */
-
-struct AssignmentStringData
-{
-    explicit AssignmentStringData(size_t dataLengthIn, size_t stringLengthIn);
-    static AssignmentStringData decodeString(const char *begin, const char *end,
-                                   int valueEncoding, bool toUtf16);
-
-    static inline AssignmentStringData decodeString(const std::string &value,
-                                        int valueEncoding, bool toUtf16)
-        { return decodeString(value.c_str(), value.c_str() + value.size(),
-                              valueEncoding, toUtf16); }
-
-    unsigned char *data;
-    size_t dataLength;
-    size_t stringLength;
-};
-
-AssignmentStringData::AssignmentStringData(size_t dataLengthIn, size_t stringLengthIn) :
-    data(new unsigned char[dataLengthIn]), dataLength(dataLengthIn),
-    stringLength(stringLengthIn)
-{
-    if (dataLength)
-        memset(data, 0, dataLength);
-}
-
-AssignmentStringData AssignmentStringData::decodeString(const char *begin, const char *end,
-                                    int valueEncoding, bool toUtf16)
-{
-    if (toUtf16) { // Target is UTF16 consisting of unsigned short characters.
-        switch (valueEncoding) {
-        // Hex encoded ASCII/2 digits per char: Decode to plain characters and
-        // recurse to expand them.
-        case AssignHexEncoded: {
-            const AssignmentStringData decoded = decodeString(begin, end, AssignHexEncoded, false);
-            const char *source = reinterpret_cast<const char*>(decoded.data);
-            const AssignmentStringData utf16 = decodeString(source, source + decoded.stringLength,
-                                                  AssignPlainValue, true);
-            delete [] decoded.data;
-            return utf16;
-        }
-        // Hex encoded UTF16: 4 hex digits per character: Decode sequence.
-        case AssignHexEncodedUtf16: {
-            const size_t stringLength = (end - begin) / 4;
-            AssignmentStringData result(sizeof(unsigned short) *(stringLength + 1), stringLength);
-            decodeHex(begin, end, result.data);
-            return result;
-        }
-        default:
-            break;
-        }
-        // Convert plain ASCII data to UTF16 by expanding.
-        const size_t stringLength = end - begin;
-        AssignmentStringData result(sizeof(unsigned short) *(stringLength + 1), stringLength);
-        const unsigned char *source = reinterpret_cast<const unsigned char *>(begin);
-        unsigned short *target = reinterpret_cast<unsigned short *>(result.data);
-        std::copy(source, source + stringLength, target);
-
-        return result;
-    } // toUtf16
-    switch (valueEncoding) {
-    case AssignHexEncoded: { // '0A5A'..2 digits per character
-        const size_t stringLength = (end - begin) / 2;
-        AssignmentStringData result(stringLength + 1, stringLength);
-        decodeHex(begin, end, result.data);
-        return result;
-    }
-    // Condense UTF16 characters to ASCII: Decode and use only every 2nd character
-    // (little endian, first byte)
-    case AssignHexEncodedUtf16: {
-        const AssignmentStringData decoded = decodeString(begin, end, AssignHexEncoded, false);
-        const size_t stringLength = decoded.stringLength / 2;
-        const AssignmentStringData result(stringLength + 1, stringLength);
-        const unsigned char *sourceEnd = decoded.data + decoded.stringLength;
-        unsigned char *target = result.data;
-        for (const unsigned char *source = decoded.data; source < sourceEnd; source += 2)
-            *target++ = *source;
-        delete [] decoded.data;
-        return result;
-    }
-        break;
-    default:
-        break;
-    }
-    // Plain 0-terminated copy
-    const size_t stringLength = end - begin;
-    AssignmentStringData result(stringLength + 1, stringLength);
-    memcpy(result.data, begin, stringLength);
-    return result;
-}
-
 // Assignment helpers
 static inline std::string msgAssignStringFailed(const std::string &value, int errorCode)
 {
@@ -3417,8 +3322,9 @@ static inline std::string msgAssignStringFailed(const std::string &value, int er
  * recurse (since 'd' might become invalid). This works for QString with UTF16
  * data and for QByteArray with ASCII data due to the similar member
  * names and both using a terminating '\0' w_char/byte. */
+template <typename string>
 static int assignQStringI(SymbolGroupNode  *n, const char *className,
-                          const AssignmentStringData &data,
+                          const string &data,
                           const SymbolGroupValueContext &ctx,
                           bool doAlloc = true)
 {
@@ -3431,7 +3337,7 @@ static int assignQStringI(SymbolGroupNode  *n, const char *className,
     const QtStringAddressData addressData = readQtStringAddressData(d, qtInfo);
     if (!addressData.address)
         return 9;
-    const bool needRealloc = addressData.allocated < data.stringLength;
+    const bool needRealloc = addressData.allocated < data.size();
     if (needRealloc) {
         if (!doAlloc) // Calling re-alloc failed somehow.
             return 3;
@@ -3439,7 +3345,7 @@ static int assignQStringI(SymbolGroupNode  *n, const char *className,
         const std::string funcName
             = qtInfo.prependQtCoreModule(std::string(className) + "::resize");
         callStr << funcName << '(' << std::hex << std::showbase
-                << v.address() << ',' << data.stringLength << ')';
+                << v.address() << ',' << data.size() << ')';
         std::wstring wOutput;
         std::string errorMessage;
         return ExtensionContext::instance().call(callStr.str(), 0, &wOutput, &errorMessage) ?
@@ -3447,8 +3353,8 @@ static int assignQStringI(SymbolGroupNode  *n, const char *className,
     }
     // Write data.
     if (!SymbolGroupValue::writeMemory(v.context().dataspaces,
-                                       addressData.address, data.data,
-                                       ULONG(data.dataLength)))
+                                       addressData.address, (const unsigned char *)(data.c_str()),
+                                       ULONG(data.empty() ? 0 : sizeof(data.front()) * data.size())))
         return 11;
     // Correct size unless we re-allocated
     if (!needRealloc) {
@@ -3460,7 +3366,7 @@ static int assignQStringI(SymbolGroupNode  *n, const char *className,
         const SymbolGroupValue size = dV["size"];
         if (!size)
             return 16;
-        if (!size.node()->assign(toString(data.stringLength)))
+        if (!size.node()->assign(toString(data.size())))
             return 17;
     }
     return 0;
@@ -3468,13 +3374,11 @@ static int assignQStringI(SymbolGroupNode  *n, const char *className,
 
 // QString assignment
 static inline bool assignQString(SymbolGroupNode  *n,
-                                 int valueEncoding, const std::string &value,
+                                 const std::string &value,
                                  const SymbolGroupValueContext &ctx,
                                  std::string *errorMessage)
 {
-    const AssignmentStringData utf16 = AssignmentStringData::decodeString(value, valueEncoding, true);
-    const int errorCode = assignQStringI(n, "QString", utf16, ctx);
-    delete [] utf16.data;
+    const int errorCode = assignQStringI(n, "QString", utf8ToUtf16(value), ctx);
     if (errorCode) {
         *errorMessage = msgAssignStringFailed(value, errorCode);
         return false;
@@ -3484,13 +3388,11 @@ static inline bool assignQString(SymbolGroupNode  *n,
 
 // QByteArray assignment
 static inline bool assignQByteArray(SymbolGroupNode  *n,
-                                    int valueEncoding, const std::string &value,
+                                    const std::string &value,
                                     const SymbolGroupValueContext &ctx,
                                     std::string *errorMessage)
 {
-    const AssignmentStringData data = AssignmentStringData::decodeString(value, valueEncoding, false);
-    const int errorCode = assignQStringI(n, "QByteArray", data, ctx);
-    delete [] data.data;
+    const int errorCode = assignQStringI(n, "QByteArray", value, ctx);
     if (errorCode) {
         *errorMessage = msgAssignStringFailed(value, errorCode);
         return false;
@@ -3499,8 +3401,9 @@ static inline bool assignQByteArray(SymbolGroupNode  *n,
 }
 
 // Helper to assign character data to std::string or std::wstring.
-static inline int assignStdStringI(SymbolGroupNode  *n, int type,
-                                   const AssignmentStringData &data,
+template <typename string>
+static inline int assignStdStringI(SymbolGroupNode *n, int type,
+                                   const string &data,
                                    const SymbolGroupValueContext &ctx)
 {
     /* We do not reallocate and just write to the allocated buffer
@@ -3527,7 +3430,7 @@ static inline int assignStdStringI(SymbolGroupNode  *n, int type,
     int reserved = base["_Myres"].intValue();
     if (reserved < 0 || !size || !bx)
         return 42;
-    if (reserved <= (int)data.stringLength)
+    if (reserved <= (int)data.size())
         return 1; // Insufficient memory.
     // Copy data: 'Buf' array for small strings, else pointer 'Ptr'.
     const int bufSize = type == KT_StdString ? 16 : 8; // see basic_string.
@@ -3536,25 +3439,25 @@ static inline int assignStdStringI(SymbolGroupNode  *n, int type,
     if (!address)
         return 3;
     if (!SymbolGroupValue::writeMemory(v.context().dataspaces,
-                                       address, data.data, ULONG(data.dataLength)))
+                                       address,
+                                       (const unsigned char *)(data.c_str()),
+                                       ULONG(data.empty() ? 0 : sizeof(data.front()) * data.size())))
         return 7;
     // Correct size
-    if (!size.node()->assign(toString(data.stringLength)))
+    if (!size.node()->assign(toString(data.size())))
         return 13;
     return 0;
 }
 
 // assignment of std::string assign via ASCII, std::wstring via UTF16
 static inline bool assignStdString(SymbolGroupNode  *n,
-                                   int type, int valueEncoding, const std::string &value,
+                                   int type, const std::string &value,
                                    const SymbolGroupValueContext &ctx,
                                    std::string *errorMessage)
 {
-    const bool toUtf16 = type == KT_StdWString;
-    const AssignmentStringData data = AssignmentStringData::decodeString(value, valueEncoding,
-                                                                         toUtf16);
-    const int errorCode = assignStdStringI(n, type, data, ctx);
-    delete [] data.data;
+    const int errorCode = type == KT_StdString
+                              ? assignStdStringI(n, type, value, ctx)
+                              : assignStdStringI(n, type, utf8ToUtf16(value), ctx);
     if (errorCode) {
         *errorMessage = msgAssignStringFailed(value, errorCode);
         return false;
@@ -3562,17 +3465,17 @@ static inline bool assignStdString(SymbolGroupNode  *n,
     return true;
 }
 
-bool assignType(SymbolGroupNode *n, int knownType, int valueEncoding, const std::string &value,
+bool assignType(SymbolGroupNode *n, int knownType, const std::string &value,
                 const SymbolGroupValueContext &ctx, std::string *errorMessage)
 {
     switch (knownType) {
     case KT_QString:
-        return assignQString(n, valueEncoding, value, ctx, errorMessage);
+        return assignQString(n, value, ctx, errorMessage);
     case KT_QByteArray:
-        return assignQByteArray(n, valueEncoding, value, ctx, errorMessage);
+        return assignQByteArray(n,value, ctx, errorMessage);
     case KT_StdString:
     case KT_StdWString:
-        return assignStdString(n, knownType, valueEncoding, value, ctx, errorMessage);
+        return assignStdString(n, knownType, value, ctx, errorMessage);
     default:
         break;
     }
diff --git a/src/libs/qtcreatorcdbext/symbolgroupvalue.h b/src/libs/qtcreatorcdbext/symbolgroupvalue.h
index b2c89cba47..bb69794cc0 100644
--- a/src/libs/qtcreatorcdbext/symbolgroupvalue.h
+++ b/src/libs/qtcreatorcdbext/symbolgroupvalue.h
@@ -286,9 +286,8 @@ enum AssignEncoding
     AssignHexEncodedUtf16
 };
 
-bool assignType(SymbolGroupNode  *n, int knownType, int valueEncoding, const std::string &value,
-                const SymbolGroupValueContext &ctx,
-                std::string *errorMessage);
+bool assignType(SymbolGroupNode *n, int knownType, const std::string &value,
+                const SymbolGroupValueContext &ctx, std::string *errorMessage);
 
 // Non-container complex dumpers (QObjects/QVariants).
 std::vector<AbstractSymbolGroupNode *>
diff --git a/src/plugins/debugger/cdb/cdbengine.cpp b/src/plugins/debugger/cdb/cdbengine.cpp
index a5554a3aa8..4e04a02fc4 100644
--- a/src/plugins/debugger/cdb/cdbengine.cpp
+++ b/src/plugins/debugger/cdb/cdbengine.cpp
@@ -917,15 +917,6 @@ void CdbEngine::handleJumpToLineAddressResolution(const DebuggerResponse &respon
     }
 }
 
-static inline bool isAsciiWord(const QString &s)
-{
-    for (const QChar &c : s) {
-        if (!c.isLetterOrNumber() || c.toLatin1() == 0)
-            return false;
-    }
-    return true;
-}
-
 void CdbEngine::assignValueInDebugger(WatchItem *w, const QString &expr, const QVariant &value)
 {
     if (debug)
@@ -935,28 +926,8 @@ void CdbEngine::assignValueInDebugger(WatchItem *w, const QString &expr, const Q
         qWarning("Internal error: assignValueInDebugger: Invalid state or no stack frame.");
         return;
     }
-    QString cmd;
-    StringInputStream str(cmd);
-    switch (value.type()) {
-    case QVariant::String: {
-        // Convert qstring to Utf16 data not considering endianness for Windows.
-        const QString s = value.toString();
-        if (isAsciiWord(s)) {
-            str << m_extensionCommandPrefix << "assign \"" << w->iname << '=' << s << '"';
-        } else {
-            const QByteArray utf16(reinterpret_cast<const char *>(s.utf16()), 2 * s.size());
-            str << m_extensionCommandPrefix << "assign -u " << w->iname << '='
-                << QString::fromLatin1(utf16.toHex());
-        }
-    }
-        break;
-    default:
-        str << m_extensionCommandPrefix << "assign " << w->iname << '='
-            << value.toString();
-        break;
-    }
-
-    runCommand({cmd, NoFlags});
+    runCommand({m_extensionCommandPrefix + "assign -h " + w->iname + '=' + toHex(value.toString()),
+                NoFlags});
     // Update all locals in case we change a union or something pointed to
     // that affects other variables, too.
     updateLocals();