[qmf] IMAP: encode folder name with non-US characters

Part 1: Encode folder name on create/rename according to RFC 3501, section 5.1.3.
Part 2: Move decode/encode implementation in separate files,
because decodeFolderName also needs to be used in ImapRenameFolderStrategy::folderRenamed

Testing (Gmail):
-------
- Create folder with non-English chars (e.g. "Папка") using onlineCreateFolder API
- In web interface - verify that folder name is correct
- Rename folder to some another non-English name using onlineRenameFolder API
- Verify that folder is renamed correctly (Web interface and local device)

Change-Id: Ifb93cacb8832992d796037eb3fb47c4253e08d14
Reviewed-by: Matthew Vogt <matthew.vogt@qinetic.com.au>
Reviewed-by: Christopher Adams <chris.adams@jollamobile.com>

6 files changed, 266 insertions, 125 deletions

diff --git a/src/libraries/qmfclient/qmailcodec.cpp b/src/libraries/qmfclient/qmailcodec.cpp
index 0672827b..883651f6 100644
--- a/src/libraries/qmfclient/qmailcodec.cpp
+++ b/src/libraries/qmfclient/qmailcodec.cpp
@@ -1190,3 +1190,224 @@ void QMailLineEndingCodec::decodeChunk(QDataStream& out, const char* it, int len
     Q_UNUSED(finalChunk)
 }
 
+
+/*! \internal */
+static QString encodeModifiedBase64(const QString &in)
+{
+    // Modified Base64 chars pattern
+    const QString encodingSchema = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,";
+
+    QString result = "&";
+    QList<ushort> buf;
+    unsigned short tmp;
+    int i;
+
+    // chars to numeric
+    for (i = 0; i < in.length(); i++) {
+        buf.push_back(in[i].unicode());
+    }
+
+    i = 0;
+    // encode every 6 bits separately in pattern by 3 symbols
+    while (i < buf.length()) {
+
+        result += encodingSchema[(buf[i] & 0xfc00) >> 10];
+        result += encodingSchema[(buf[i] & 0x3f0) >> 4];
+
+        tmp = 0;
+        tmp |= ((buf[i] & 0xf) << 12);
+        if (i + 1 < buf.length()) {
+            i++;
+            tmp |= ((buf[i] & 0xc000) >> 4);
+            result += encodingSchema[tmp >> 10];
+        } else {
+            result += encodingSchema[tmp >> 10];
+            break;
+        }
+
+        result += encodingSchema[(buf[i] & 0x3f00) >> 8];
+        result += encodingSchema[(buf[i] & 0xfc) >> 2];
+
+        tmp = 0;
+        tmp |= ((buf[i] & 0x3) << 14);
+        if (i + 1 < buf.length()) {
+            i++;
+            tmp |= ((buf[i] & 0xf000) >> 2);
+            result += encodingSchema[tmp >> 10];
+        } else {
+            result += encodingSchema[tmp >> 10];
+            break;
+        }
+
+        result += encodingSchema[(buf[i] & 0xfc0) >> 6];
+        result += encodingSchema[buf[i] & 0x3f];
+        i++;
+    }
+
+    result += '-';
+
+    return result;
+}
+
+/*! \internal */
+static QString decodeModifiedBase64(QString in)
+{
+    //remove  & -
+    in.remove(0,1);
+    in.remove(in.length()-1,1);
+
+    if (in.isEmpty())
+        return "&";
+
+    QByteArray buf(in.length(),static_cast<char>(0));
+    QByteArray out(in.length() * 3 / 4 + 2,static_cast<char>(0));
+
+    //chars to numeric
+    QByteArray latinChars = in.toLatin1();
+    for (int x = 0; x < in.length(); x++) {
+        int c = latinChars[x];
+        if ( c >= 'A' && c <= 'Z')
+            buf[x] = c - 'A';
+        if ( c >= 'a' && c <= 'z')
+            buf[x] = c - 'a' + 26;
+        if ( c >= '0' && c <= '9')
+            buf[x] = c - '0' + 52;
+        if ( c == '+')
+            buf[x] = 62;
+        if ( c == ',')
+            buf[x] = 63;
+    }
+
+    int i = 0; //in buffer index
+    int j = i; //out buffer index
+
+    unsigned char z;
+    QString result;
+
+    while (i+1 < buf.size()) {
+        out[j] = buf[i] & (0x3F); //mask out top 2 bits
+        out[j] = out[j] << 2;
+        z = buf[i+1] >> 4;
+        out[j] = (out[j] | z);      //first byte retrieved
+
+        i++;
+        j++;
+
+        if (i+1 >= buf.size())
+            break;
+
+        out[j] = buf[i] & (0x0F);   //mask out top 4 bits
+        out[j] = out[j] << 4;
+        z = buf[i+1] >> 2;
+        z &= 0x0F;
+        out[j] = (out[j] | z);      //second byte retrieved
+
+        i++;
+        j++;
+
+        if (i+1 >= buf.size())
+            break;
+
+        out[j] = buf[i] & 0x03;   //mask out top 6 bits
+        out[j] = out[j] <<  6;
+        z = buf[i+1];
+        out[j] = out[j] | z;  //third byte retrieved
+
+        i += 2; //next byte
+        j++;
+    }
+
+    //go through the buffer and extract 16 bit unicode network byte order
+    for (int z = 0; z < out.count(); z += 2) {
+        unsigned short outcode = 0x0000;
+        outcode = out[z];
+        outcode <<= 8;
+        outcode &= 0xFF00;
+
+        unsigned short b = 0x0000;
+        b = out[z+1];
+        b &= 0x00FF;
+        outcode = outcode | b;
+        if (outcode)
+            result += QChar(outcode);
+    }
+
+    return result;
+}
+
+/*!
+    Encodes a \a text using modified UTF7 encoding according to RFC 3501, section 5.1.3
+
+    \sa QMailCodec::decodeModifiedUtf7
+*/
+QString QMailCodec::encodeModifiedUtf7(const QString &text)
+{
+    QString in = text;
+    int startIndex = 0;
+    int endIndex = 0;
+
+    while (startIndex < in.length()) {
+        // insert '-' after '&'
+        if (in[startIndex] == '&') {
+            startIndex++;
+            in.insert(startIndex, '-');
+            continue;
+        }
+
+        if (in[startIndex].unicode() < 0x20 || in[startIndex].unicode() > 0x7e) {
+            // get non-US-ASCII part
+            endIndex = startIndex;
+            while (endIndex < in.length() && (in[endIndex].unicode() < 0x20 || in[endIndex].unicode() > 0x7e))
+                endIndex++;
+
+            // encode non-US-ASCII part
+            QString unicodeString = in.mid(startIndex,(endIndex - startIndex));
+            QString mbase64 = encodeModifiedBase64(unicodeString);
+
+            // insert the encoded string
+            in.remove(startIndex,(endIndex-startIndex));
+            in.insert(startIndex, mbase64);
+
+            // set start index to the end of the encoded part
+            startIndex += mbase64.length() - 1;
+        }
+        startIndex++;
+    }
+    return in;
+}
+
+/*!
+    Decodes modified UTF7 \a text according to RFC 3501, section 5.1.3
+
+    \sa QMailCodec::encodeModifiedUtf7
+*/
+QString QMailCodec::decodeModifiedUtf7(const QString &text)
+{
+    QString in = text;
+    QRegExp reg("&[^&-]*-");
+
+    int startIndex = 0;
+    int endIndex = 0;
+
+    startIndex = in.indexOf(reg,endIndex);
+    while (startIndex != -1) {
+        endIndex = startIndex;
+        while (endIndex < in.length() && in[endIndex] != '-')
+            endIndex++;
+        endIndex++;
+
+        //extract the base64 string from the input string
+        QString mbase64 = in.mid(startIndex,(endIndex - startIndex));
+        QString unicodeString = decodeModifiedBase64(mbase64);
+
+        //remove encoding
+        in.remove(startIndex,(endIndex-startIndex));
+        in.insert(startIndex,unicodeString);
+
+        endIndex = startIndex + unicodeString.length();
+        startIndex = in.indexOf(reg,endIndex);
+    }
+
+    return in;
+}
+
diff --git a/src/libraries/qmfclient/qmailcodec.h b/src/libraries/qmfclient/qmailcodec.h
index 55d4ed00..cdb6f3c0 100644
--- a/src/libraries/qmfclient/qmailcodec.h
+++ b/src/libraries/qmfclient/qmailcodec.h
@@ -70,6 +70,8 @@ public:
     static void copy(QDataStream& out, QDataStream& in);
     static void copy(QTextStream& out, QTextStream& in);
     static QString autoDetectEncoding(const QByteArray& text);
+    static QString encodeModifiedUtf7(const QString &text);
+    static QString decodeModifiedUtf7(const QString &text);
 
 protected:
     // Helper functions to convert stream chunks
diff --git a/src/plugins/messageservices/imap/imapclient.cpp b/src/plugins/messageservices/imap/imapclient.cpp
index 477e2410..152455ee 100644
--- a/src/plugins/messageservices/imap/imapclient.cpp
+++ b/src/plugins/messageservices/imap/imapclient.cpp
@@ -41,6 +41,7 @@
 #include <qmailfolder.h>
 #include <qmailnamespace.h>
 #include <qmaildisconnected.h>
+#include <qmailcodec.h>
 #include <limits.h>
 #include <QFile>
 #include <QDir>
@@ -93,126 +94,6 @@ public:
 };
 
 namespace {
-
-    QString decodeModifiedBase64(QString in)
-    {
-        //remove  & -
-        in.remove(0,1);
-        in.remove(in.length()-1,1);
-
-        if(in.isEmpty())
-            return "&";
-
-        QByteArray buf(in.length(),static_cast<char>(0));
-        QByteArray out(in.length() * 3 / 4 + 2,static_cast<char>(0));
-
-        //chars to numeric
-        QByteArray latinChars = in.toLatin1();
-        for (int x = 0; x < in.length(); x++) {
-            int c = latinChars[x];
-            if ( c >= 'A' && c <= 'Z')
-                buf[x] = c - 'A';
-            if ( c >= 'a' && c <= 'z')
-                buf[x] = c - 'a' + 26;
-            if ( c >= '0' && c <= '9')
-                buf[x] = c - '0' + 52;
-            if ( c == '+')
-                buf[x] = 62;
-            if ( c == ',')
-                buf[x] = 63;
-        }
-
-        int i = 0; //in buffer index
-        int j = i; //out buffer index
-
-        unsigned char z;
-        QString result;
-
-        while(i+1 < buf.size())
-        {
-            out[j] = buf[i] & (0x3F); //mask out top 2 bits
-            out[j] = out[j] << 2;
-            z = buf[i+1] >> 4;
-            out[j] = (out[j] | z);      //first byte retrieved
-
-            i++;
-            j++;
-
-            if(i+1 >= buf.size())
-                break;
-
-            out[j] = buf[i] & (0x0F);   //mask out top 4 bits
-            out[j] = out[j] << 4;
-            z = buf[i+1] >> 2;
-            z &= 0x0F;
-            out[j] = (out[j] | z);      //second byte retrieved
-
-            i++;
-            j++;
-
-            if(i+1 >= buf.size())
-                break;
-
-            out[j] = buf[i] & 0x03;   //mask out top 6 bits
-            out[j] = out[j] <<  6;
-            z = buf[i+1];
-            out[j] = out[j] | z;  //third byte retrieved
-
-            i+=2; //next byte
-            j++;
-        }
-
-        //go through the buffer and extract 16 bit unicode network byte order
-        for(int z = 0; z < out.count(); z+=2) {
-            unsigned short outcode = 0x0000;
-            outcode = out[z];
-            outcode <<= 8;
-            outcode &= 0xFF00;
-
-            unsigned short b = 0x0000;
-            b = out[z+1];
-            b &= 0x00FF;
-            outcode = outcode | b;
-            if(outcode)
-                result += QChar(outcode);
-        }
-
-        return result;
-    }
-
-    QString decodeModUTF7(QString in)
-    {
-        QRegExp reg("&[^&-]*-");
-
-        int startIndex = 0;
-        int endIndex = 0;
-
-        startIndex = in.indexOf(reg,endIndex);
-        while (startIndex != -1) {
-            endIndex = startIndex;
-            while(endIndex < in.length() && in[endIndex] != '-')
-                endIndex++;
-            endIndex++;
-
-            //extract the base64 string from the input string
-            QString mbase64 = in.mid(startIndex,(endIndex - startIndex));
-            QString unicodeString = decodeModifiedBase64(mbase64);
-
-            //remove encoding
-            in.remove(startIndex,(endIndex-startIndex));
-            in.insert(startIndex,unicodeString);
-
-            endIndex = startIndex + unicodeString.length();
-            startIndex = in.indexOf(reg,endIndex);
-        }
-
-        return in;
-    }
-
-    QString decodeFolderName(const QString &name)
-    {
-        return decodeModUTF7(name);
-    }
     
     struct FlagInfo
     {
@@ -926,7 +807,7 @@ void ImapClient::mailboxListed(const QString &flags, const QString &path)
         } else {
             // This element needs to be created
             QMailFolder folder(mailboxPath, parentId, _config.id());
-            folder.setDisplayName(decodeFolderName(*it));
+            folder.setDisplayName(QMailCodec::decodeModifiedUtf7(*it));
             folder.setStatus(QMailFolder::SynchronizationEnabled, true);
             folder.setStatus(QMailFolder::Incoming, true);
 
diff --git a/src/plugins/messageservices/imap/imapprotocol.cpp b/src/plugins/messageservices/imap/imapprotocol.cpp
index 506ddbc1..fe5cfd64 100644
--- a/src/plugins/messageservices/imap/imapprotocol.cpp
+++ b/src/plugins/messageservices/imap/imapprotocol.cpp
@@ -50,6 +50,7 @@
 #include <qmailnamespace.h>
 #include <qmailtransport.h>
 #include <qmaildisconnected.h>
+#include <qmailcodec.h>
 
 #ifndef QT_NO_SSL
 #include <QSslError>
@@ -714,7 +715,7 @@ QString CreateState::makePath(ImapContext *c, const QMailFolderId &parent, const
             qWarning() << "Cannot create a child folder, without a delimiter";
 
     }
-    return (path + name);
+    return (path + QMailCodec::encodeModifiedUtf7(name));
 }
 
 class DeleteState : public ImapState
@@ -833,10 +834,11 @@ void RenameState::taggedResponse(ImapContext *c, const QString &line)
 QString RenameState::buildNewPath(ImapContext *c , const QMailFolder &folder, QString &newName)
 {
     QString path;
+    QString encodedNewName = QMailCodec::encodeModifiedUtf7(newName);
     if(c->protocol()->flatHierarchy() || folder.path().count(c->protocol()->delimiter()) == 0)
-        path = newName;
+        path = encodedNewName;
     else
-        path = folder.path().section(c->protocol()->delimiter(), 0, -2) + c->protocol()->delimiter() + newName;
+        path = folder.path().section(c->protocol()->delimiter(), 0, -2) + c->protocol()->delimiter() + encodedNewName;
     return path;
 }
 
diff --git a/src/plugins/messageservices/imap/imapstrategy.cpp b/src/plugins/messageservices/imap/imapstrategy.cpp
index c11b11f2..9dd792c6 100644
--- a/src/plugins/messageservices/imap/imapstrategy.cpp
+++ b/src/plugins/messageservices/imap/imapstrategy.cpp
@@ -43,6 +43,7 @@
 #include <qmailmessage.h>
 #include <qmailnamespace.h>
 #include <qmaildisconnected.h>
+#include <qmailcodec.h>
 #include <limits.h>
 #include <QDir>
 
@@ -854,7 +855,7 @@ void ImapRenameFolderStrategy::folderRenamed(ImapStrategyContextBase *context, c
 
     QMailFolder newFolder = folder;
     newFolder.setPath(newPath);
-    newFolder.setDisplayName(name);
+    newFolder.setDisplayName(QMailCodec::decodeModifiedUtf7(name));
 
     if(!QMailStore::instance()->updateFolder(&newFolder))
         qWarning() << "Unable to locally rename folder";
diff --git a/tests/tst_qmailcodec/tst_qmailcodec.cpp b/tests/tst_qmailcodec/tst_qmailcodec.cpp
index c522b28b..5212f9e7 100644
--- a/tests/tst_qmailcodec/tst_qmailcodec.cpp
+++ b/tests/tst_qmailcodec/tst_qmailcodec.cpp
@@ -66,6 +66,7 @@ private slots:
     void buffer_sizes();
     void embedded_newlines_data();
     void embedded_newlines();
+    void encodeDecodeModifiedUtf7();
 };
 
 QTEST_MAIN(tst_QMailCodec)
@@ -676,3 +677,36 @@ void tst_QMailCodec::embedded_newlines()
     QuotedPrintableMaxLineLength = originalQuotedPrintableMaxLineLength;
 }
 
+void tst_QMailCodec::encodeDecodeModifiedUtf7()
+{
+    // Test with some arabic characters, as per http://en.wikipedia.org/wiki/List_of_Unicode_characters
+    const QChar arabicChars[] = { 0x0636, 0x0669, 0x06a5, 0x06b4, 0x06a5, 0x0669, 0x0636};
+    QStringList arabicEncoded = QStringList()
+            << QString::fromLatin1("&BjY-")
+            << QString::fromLatin1("&BjYGaQ-")
+            << QString::fromLatin1("&BjYGaQal-")
+            << QString::fromLatin1("&BjYGaQalBrQ-")
+            << QString::fromLatin1("&BjYGaQalBrQGpQ-")
+            << QString::fromLatin1("&BjYGaQalBrQGpQZp-")
+            << QString::fromLatin1("&BjYGaQalBrQGpQZpBjY-");
+    int i;
+    for (i = 0; i < arabicEncoded.length(); i++) {
+       QCOMPARE(QMailCodec::encodeModifiedUtf7(QString(arabicChars, i+1)), arabicEncoded[i]);
+       QCOMPARE(QMailCodec::decodeModifiedUtf7(arabicEncoded[i]), QString(arabicChars, i+1));
+    }
+
+    // Test '&' symbols
+    QCOMPARE(QMailCodec::decodeModifiedUtf7(QString::fromLatin1("&-")), QString::fromLatin1("&"));
+    QCOMPARE(QMailCodec::encodeModifiedUtf7(QString::fromLatin1("&")), QString::fromLatin1("&-"));
+
+    // Test mixed arabic, '&' and Latin1
+    QString testString = QString::fromLatin1("abc") + QChar(0x0636) + QString::fromLatin1("& && &&&");
+    QString testEncodedString = QString::fromLatin1("abc&BjY-&- &-&- &-&-&-");
+    QCOMPARE(QMailCodec::decodeModifiedUtf7(testEncodedString), testString);
+    QCOMPARE(QMailCodec::encodeModifiedUtf7(testString), testEncodedString);
+
+    //Test empty string
+    QCOMPARE(QMailCodec::decodeModifiedUtf7(QString()), QString());
+    QCOMPARE(QMailCodec::encodeModifiedUtf7(QString()), QString());
+}
+