geniviextras: Add better handling for long messages

DLT silently drops too long messages and produces
empty lines instead.
This changes introduces a behavior enum which helps
to configure geniviextras to truncate or split the
long messages.

Fixes: AUTOSUITE-1187
Change-Id: Ieece74a984460fb6c2fe603789361b59ada9df12
Reviewed-by: Robert Griebl <robert.griebl@pelagicore.com>

4 files changed, 233 insertions, 17 deletions

diff --git a/src/geniviextras/qdltregistration.cpp b/src/geniviextras/qdltregistration.cpp
index 67e7c15..188b01f 100644
--- a/src/geniviextras/qdltregistration.cpp
+++ b/src/geniviextras/qdltregistration.cpp
@@ -67,10 +67,47 @@ QDltRegistration *globalDltRegistration()
 
 QT_END_NAMESPACE
 
+// helper functions to split or truncate a utf8 correctly (not between a composed character)
+namespace QtGeniviExtrasPrivate {
+
+    static constexpr int MAX_MSG_LEN = DLT_USER_BUF_MAX_SIZE - 10;
+
+    QByteArray utf8_mid(const QByteArray &buffer, int position, int len)
+    {
+        int max = qMin(position + len, buffer.size());
+        int end = max;
+        // move from the maximum length backwards so we don't cut in-between a multi-byte char
+        for (; end > position && end < buffer.size(); --end) {
+            if ((buffer[end] & 0xC0) != 0x80)
+                break;
+        }
+        // if we can't find a good position for a split (e.g. a invalid utf-8 string) take the max
+        // len for the split
+        if (end == position)
+            end = max;
+        return buffer.mid(position, end - position);
+    }
+
+    QVector<QByteArray> utf8_split(const QByteArray &buffer, int len)
+    {
+        QVector<QByteArray> split;
+        int position = 0;
+        do {
+            const QByteArray mid = utf8_mid(buffer, position, len);
+            split.append(mid);
+            position += mid.size();
+        } while (position < buffer.size());
+
+        return split;
+    }
+}
+
+
 QDltRegistrationPrivate::QDltRegistrationPrivate(QDltRegistration *parent)
     : q_ptr(parent)
     , m_dltAppRegistered(false)
     , m_registerOnFirstUse(false)
+    , m_longMessageBehavior(QDltRegistration::LongMessageBehavior::Truncate)
 {
 }
 
@@ -184,6 +221,7 @@ void QDltRegistrationPrivate::dltLogLevelChanged(char context_id[], uint8_t log_
                 emit q->logLevelChanged(category);
             }
         }
+        break;
     }
 }
 
@@ -231,6 +269,31 @@ DltLogLevelType QDltRegistrationPrivate::severity2dltLevel(QtMsgType type)
     types of a QLoggingCategory whenever the log level of a dlt context changes.
 */
 
+/*!
+    \enum QDltRegistration::LongMessageBehavior
+
+    This enum type describes the available options for long messages.
+
+    \value Truncate Truncate the message to the maximum size.
+    \value Split Split the message into several smaller ones.
+    \value Pass Pass the message as is to DLT. This option has the least performance impact, but
+                DLT might ignore the message if it is too long and produce just an empty line instead.
+
+    \since 5.12.4
+*/
+
+/*!
+    \property QDltRegistration::longMessageBehavior
+    \brief Defines the handling of messages that are too long for DLT to handle correctly
+
+    DLT defines that a message has a maximum size of about 1k (The real value depends on the DLT
+    version used).
+
+    The default behavior is to truncate the message to the maximum supported length.
+
+    \since 5.12.4
+*/
+
 QDltRegistration::QDltRegistration(QObject *parent)
     : QObject(parent)
     , d_ptr(new QDltRegistrationPrivate(this))
@@ -346,6 +409,18 @@ void QDltRegistration::registerUnregisteredContexts()
     }
 }
 
+void QDltRegistration::setLongMessageBehavior(QDltRegistration::LongMessageBehavior config)
+{
+    Q_D(QDltRegistration);
+    d->m_longMessageBehavior = config;
+}
+
+QDltRegistration::LongMessageBehavior QDltRegistration::longMessageBehavior() const
+{
+    Q_D(const QDltRegistration);
+    return d->m_longMessageBehavior;
+}
+
 /*!
     Unregisters the application with the dlt-daemon.
     The registered application as well as all registered dlt context will be deleted.
@@ -381,7 +456,30 @@ void QDltRegistration::messageHandler(QtMsgType msgType, const QMessageLogContex
 
     DltLogLevelType logLevel = globalDltRegistration()->d_ptr->severity2dltLevel(msgType);
 
-    DLT_LOG(*dltCtx, logLevel, DLT_STRING(qPrintable(qFormatLogMessage(msgType, msgCtx, msg))));
+    const QByteArray fullMessage = qFormatLogMessage(msgType, msgCtx, msg).toUtf8();
+
+    if (Q_UNLIKELY(fullMessage.size() > QtGeniviExtrasPrivate::MAX_MSG_LEN)) {
+#if GENIVIEXTRAS_DEBUG
+        std::cout << "LOG MESSAGE TOO LONG" << std::endl;
+#endif
+        if (globalDltRegistration()->longMessageBehavior() == LongMessageBehavior::Truncate) {
+#if GENIVIEXTRAS_DEBUG
+            std::cout << "TRUNCATE" << std::endl;
+#endif
+            DLT_LOG(*dltCtx, logLevel, DLT_UTF8(QtGeniviExtrasPrivate::utf8_mid(fullMessage, 0, QtGeniviExtrasPrivate::MAX_MSG_LEN)));
+            return;
+        } else if (globalDltRegistration()->longMessageBehavior() == LongMessageBehavior::Split) {
+#if GENIVIEXTRAS_DEBUG
+            std::cout << "SPLIT" << std::endl;
+#endif
+            const QVector<QByteArray> split = QtGeniviExtrasPrivate::utf8_split(fullMessage, QtGeniviExtrasPrivate::MAX_MSG_LEN);
+            for (const QByteArray &m : split)
+                DLT_LOG(*dltCtx, logLevel, DLT_UTF8(m));
+            return;
+        }
+    }
+
+    DLT_LOG(*dltCtx, logLevel, DLT_UTF8(fullMessage));
 }
 
 /*!
diff --git a/src/geniviextras/qdltregistration.h b/src/geniviextras/qdltregistration.h
index 84480d3..931d164 100644
--- a/src/geniviextras/qdltregistration.h
+++ b/src/geniviextras/qdltregistration.h
@@ -59,6 +59,15 @@ class Q_GENIVIEXTRAS_EXPORT QDltRegistration : public QObject
     Q_DISABLE_COPY(QDltRegistration)
 
 public:
+    enum class LongMessageBehavior {
+        Truncate,
+        Split,
+        Pass
+    };
+    Q_ENUM(LongMessageBehavior)
+
+    Q_PROPERTY(LongMessageBehavior longMessageBehavior READ longMessageBehavior WRITE setLongMessageBehavior)
+
     QDltRegistration(QObject *parent = nullptr);
     ~QDltRegistration() override;
 
@@ -71,6 +80,9 @@ public:
     void setRegisterContextOnFirstUseEnabled(bool enabled);
     void registerUnregisteredContexts();
 
+    void setLongMessageBehavior(LongMessageBehavior config);
+    LongMessageBehavior longMessageBehavior() const;
+
     static void messageHandler(QtMsgType msgTypes, const QMessageLogContext &msgCtx, const QString &msg);
 
 Q_SIGNALS:
diff --git a/src/geniviextras/qdltregistration_p.h b/src/geniviextras/qdltregistration_p.h
index 93f178a..396e3a0 100644
--- a/src/geniviextras/qdltregistration_p.h
+++ b/src/geniviextras/qdltregistration_p.h
@@ -41,6 +41,7 @@
 ****************************************************************************/
 
 #include <private/qgeniviextrasglobal_p.h>
+#include <qdltregistration.h>
 #include <QString>
 #include <QHash>
 
@@ -61,8 +62,6 @@ QT_BEGIN_NAMESPACE
 
 void qtGeniviLogLevelChangedHandler(char context_id[], uint8_t log_level, uint8_t trace_status);
 
-class QDltRegistration;
-
 class QDltRegistrationPrivate
 {
 private:
@@ -99,6 +98,7 @@ private:
     QString m_defaultCategory;
     QHash<QString, CategoryInfo> m_categoryInfoHash;
     bool m_registerOnFirstUse;
+    QDltRegistration::LongMessageBehavior m_longMessageBehavior;
 };
 
 QT_END_NAMESPACE
diff --git a/tests/auto/dlt/tst_dlt.cpp b/tests/auto/dlt/tst_dlt.cpp
index 3fb7136..12185af 100644
--- a/tests/auto/dlt/tst_dlt.cpp
+++ b/tests/auto/dlt/tst_dlt.cpp
@@ -40,6 +40,25 @@
 
 Q_LOGGING_CATEGORY(TEST1, "qtgeniviextras.test1")
 
+static QString longMsg = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy "
+"eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et "
+"accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est "
+"Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam "
+"nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero "
+"eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata anctu "
+"sest Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr,sed diam "
+"nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero "
+"eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimta sanctu "
+"sest Lorem ipsum dolor sit amet. Duis autem vel eum iriure dolor in hendrerit in vulputate velit "
+"esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et "
+"iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugit null "
+"afacilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nih euismod "
+"tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veiam, quis nostru "
+"dexerci dolor";
+
+// Copied from dlt-daemon_cfg.h
+// Defines how big the buffer for a msg payload needs to be to copy long msg
+#define DLT_DAEMON_TEXTSIZE 10024
 
 struct QDltMessage {
     QString appId;
@@ -62,28 +81,31 @@ public:
     {
         QVERIFY(dlt_file_close(dltFile, false) >= 0);
         QVERIFY(dlt_file_free(dltFile, false) >= 0);
+        delete dltFile;
     }
 
     QString idToString(char *id)
     {
         QByteArray appData(DLT_ID_SIZE, '\0');
-        dlt_print_id(appData.data(),id);
+        dlt_print_id(appData.data(), id);
         return QString(appData);
     }
 
     QString readPayload(DltMessage *msg)
     {
-        QByteArray payload(msg->datasize, '\0');
-        dlt_message_payload(msg,payload.data(),msg->datasize,DLT_OUTPUT_ASCII,false);
+        QByteArray payload(DLT_DAEMON_TEXTSIZE, '\0');
+        dlt_message_payload(msg, payload.data(), DLT_DAEMON_TEXTSIZE, DLT_OUTPUT_ASCII, false);
         return QString::fromLocal8Bit(payload);
     }
 
     QList<QDltMessage> readNextMessages()
     {
         QList<QDltMessage> messageList;
-        while (dlt_file_read(dltFile, false) == 1)
-        {
-            dlt_file_message(dltFile, 0, false);
+        //Read the complete file
+        while (dlt_file_read(dltFile, false) >= 0){};
+        //Parse the messages
+        for (int i = 0; i < dltFile->counter; i++) {
+            dlt_file_message(dltFile, i, false);
 
             QDltMessage msg;
             msg.appId = idToString(dltFile->msg.extendedheader->apid);
@@ -111,13 +133,18 @@ public:
         m_dltParser = new QDltParser(&m_tempFile);
     }
 
+    ~QDltTest()
+    {
+        delete m_dltParser;
+    }
+
 private Q_SLOTS:
 
     void initTestCase();
-//    void registerApplication();
-//    void registerTwoApplications();
-//    void registerContexts();
+    void init();
+    void testLongMessages_data();
     void testLogging();
+    void testLongMessages();
 
 private:
     QTemporaryFile m_tempFile;
@@ -126,21 +153,30 @@ private:
 
 void QDltTest::initTestCase()
 {
-    //Initialize the dlt system before the QDltRegistration can do it to log to a file.
-    dlt_init_file(m_tempFile.fileName().toLocal8Bit());
-
+    qRegisterMetaType<QDltRegistration::LongMessageBehavior>();
     //Install the DLT message handler
     qInstallMessageHandler(QDltRegistration::messageHandler);
 }
 
+void QDltTest::init()
+{
+    //Reset the log file after each test
+    m_tempFile.resize(0);
+    //Initialize the DLT log file and write the header to it.
+    dlt_init_file(m_tempFile.fileName().toLocal8Bit());
+    //Create a new Parser (to start reading the file from the beginning)
+    delete m_dltParser;
+    m_dltParser = new QDltParser(&m_tempFile);
+}
+
 void QDltTest::testLogging()
 {
     QDltRegistration *registration = globalDltRegistration();
     registration->registerApplication("APP1", "Description for APP");
     registration->registerCategory(&TEST1(), "TES1", "Test Category One");
 
-    QString msg = QLatin1String("TEST");
-    QString expectedMsg = QString("%1: \"%2\"").arg(TEST1().categoryName(), msg);
+    QString msg = QStringLiteral("TEST");
+    QString expectedMsg = QString(QStringLiteral("%1: \"%2\"")).arg(TEST1().categoryName(), msg);
 
     qWarning(TEST1) << msg;
 
@@ -151,6 +187,76 @@ void QDltTest::testLogging()
     QCOMPARE(dltMessages.at(0).payload, expectedMsg);
 }
 
+void QDltTest::testLongMessages_data()
+{
+    QTest::addColumn<QDltRegistration::LongMessageBehavior>("behavior");
+    QTest::addColumn<QString>("msg");
+    QTest::addColumn<QStringList>("expected_msgs");
+
+    QTest::addRow("Truncate ASCII") << QDltRegistration::LongMessageBehavior::Truncate
+                                    << longMsg + QStringLiteral("ipsum")
+                                    << QStringList(longMsg + QStringLiteral("ipsu"));
+    QTest::addRow("Truncate UTF-8 2 bytes") << QDltRegistration::LongMessageBehavior::Truncate
+                                            << longMsg + QStringLiteral("©®¥¶¼")
+                                            << QStringList(longMsg + QStringLiteral("©®"));
+    QTest::addRow("Truncate UTF-8 3 bytes") << QDltRegistration::LongMessageBehavior::Truncate
+                                            << longMsg + QStringLiteral("你好世界")
+                                            << QStringList(longMsg +  QStringLiteral("你"));
+    QTest::addRow("Split ASCII") << QDltRegistration::LongMessageBehavior::Split
+                                 << longMsg + QStringLiteral("ipsum")
+                                 << QStringList({longMsg + QStringLiteral("ipsu"), QStringLiteral("m\"")});
+    QTest::addRow("Split UTF-8 2 bytes") << QDltRegistration::LongMessageBehavior::Split
+                                         << longMsg + QStringLiteral("©®¥¶¼")
+                                         << QStringList({longMsg + QStringLiteral("©®"), QStringLiteral("¥¶¼\"")});
+    QTest::addRow("Split UTF-8 3 bytes") << QDltRegistration::LongMessageBehavior::Split
+                                         << longMsg +  QStringLiteral("你好世界")
+                                         << QStringList({longMsg +  QStringLiteral("你"),  QStringLiteral("好世界\"")});
+    QTest::addRow("Pass ASCII") << QDltRegistration::LongMessageBehavior::Pass
+                                << longMsg + QStringLiteral("ipsum123456789a")
+                                << QStringList(QString());
+    QTest::addRow("Pass UTF-8 2 bytes") << QDltRegistration::LongMessageBehavior::Pass
+                                        << longMsg + QStringLiteral("©®¥¶¼")
+                                        << QStringList(QString());
+    QTest::addRow("Pass UTF-8 3 bytes") << QDltRegistration::LongMessageBehavior::Pass
+                                        << longMsg +  QStringLiteral("你好世界")
+                                        << QStringList(QString());
+}
+
+void QDltTest::testLongMessages()
+{
+    QFETCH(QDltRegistration::LongMessageBehavior, behavior);
+    QFETCH(QString, msg);
+    QFETCH(QStringList, expected_msgs);
+
+    QDltRegistration *registration = globalDltRegistration();
+    registration->registerApplication("APP1", "Description for APP");
+    registration->registerCategory(&TEST1(), "TES1", "Test Category One");
+    registration->setLongMessageBehavior(behavior);
+
+    qWarning(TEST1) << msg;
+
+    QList<QDltMessage> dltMessages = m_dltParser->readNextMessages();
+    QCOMPARE(dltMessages.count(), expected_msgs.count());
+
+    int i = 0;
+    for (const QString &expected_msg : expected_msgs) {
+        QString expectedMsg;
+        //The logging category will be added before the splitting, it's only part of the first msg
+        if (i == 0 && behavior != QDltRegistration::LongMessageBehavior::Pass) {
+            //The closing quotes are part of the message pattern and will be cut as well.
+            expectedMsg = QString(QStringLiteral("%1: \"%2")).arg(TEST1().categoryName(), expected_msg);
+        } else {
+            expectedMsg = expected_msg;
+        }
+
+        QCOMPARE(dltMessages.at(i).appId, QLatin1String("APP1"));
+        QCOMPARE(dltMessages.at(i).ctxId, QLatin1String("TES1"));
+        // Enable me to compare the full content on the console
+        // std::cout << dltMessages.at(i).payload.toStdString() << "\n" << expectedMsg.toStdString() << std::endl;
+        QCOMPARE(dltMessages.at(i).payload, expectedMsg);
+        i++;
+    }
+}
 
 QTEST_APPLESS_MAIN(QDltTest)