Fix QVCard21Writer for non-Latin-1 characters.

Change the encoding algorithm to:
- if value is not encodable in Latin-1 (or whatever the document global codec is)
    1. Encode as UTF-8
    2. Encode the UTF-8 in Quoted-Printable
    3. Write the result using the document global codec.

This mirrors what the QVersitReader does

Change-Id: Ia4815afedc7b9156c748d2ca2110b3ef893d3156
Reviewed-By: Michael Goddard
Task-number: QTMOBILITY-821

6 files changed, 71 insertions, 46 deletions

diff --git a/src/versit/qvcard21writer.cpp b/src/versit/qvcard21writer.cpp
index 074428e7a1..b9acf7de74 100644
--- a/src/versit/qvcard21writer.cpp
+++ b/src/versit/qvcard21writer.cpp
@@ -52,6 +52,17 @@ QVCard21Writer::QVCard21Writer(QVersitDocument::VersitType type) : QVersitDocume
 {
 }
 
+QTextEncoder* QVCard21Writer::utf8Encoder()
+{
+    static QTextEncoder* encoder = 0;
+    if (encoder == 0) {
+        encoder = QTextCodec::codecForName("UTF-8")->makeEncoder();
+        // Hack so the encoder doesn't output a byte order mark
+        encoder->fromUnicode(QString());
+    }
+    return encoder;
+}
+
 /*! Destroys a writer. */
 QVCard21Writer::~QVCard21Writer()
 {
@@ -68,7 +79,6 @@ void QVCard21Writer::encodeVersitProperty(const QVersitProperty& property)
 
     QString renderedValue;
     QByteArray renderedBytes;
-    bool useUtf8 = false;
 
     /* Structured values need to have their components backslash-escaped (in vCard 2.1, semicolons
        must be escaped for compound values and commas must be escaped for list values). */
@@ -90,7 +100,7 @@ void QVCard21Writer::encodeVersitProperty(const QVersitProperty& property)
         bool first = true;
         foreach (QString value, values) {
             if (!(value.isEmpty() && property.valueType() == QVersitProperty::ListType)) {
-                useUtf8 |= encodeVersitValue(parameters, value);
+                encodeVersitValue(parameters, value);
                 if (!first) {
                     renderedValue += separator;
                 }
@@ -100,7 +110,7 @@ void QVCard21Writer::encodeVersitProperty(const QVersitProperty& property)
         }
     } else if (variant.type() == QVariant::String) {
         renderedValue = variant.toString();
-        useUtf8 = encodeVersitValue(parameters, renderedValue);
+        encodeVersitValue(parameters, renderedValue);
     } else if (variant.type() == QVariant::ByteArray) {
         parameters.insert(QLatin1String("ENCODING"), QLatin1String("BASE64"));
         if (mCodecIsAsciiCompatible) // optimize by not converting to unicode
@@ -119,14 +129,14 @@ void QVCard21Writer::encodeVersitProperty(const QVersitProperty& property)
         QVersitDocument embeddedDocument = variant.value<QVersitDocument>();
         encodeVersitDocument(embeddedDocument);
     } else if (variant.type() == QVariant::String || variant.type() == QVariant::StringList) {
-        writeString(renderedValue, useUtf8);
+        writeString(renderedValue);
     } else if (variant.type() == QVariant::ByteArray) {
         // One extra folding before the value and
         // one extra line break after the value are needed in vCard 2.1
         writeCrlf();
         writeString(QLatin1String(" "));
         if (renderedBytes.isEmpty())
-            writeString(renderedValue, useUtf8);
+            writeString(renderedValue);
         else
             writeBytes(renderedBytes);
         writeCrlf();
@@ -136,18 +146,19 @@ void QVCard21Writer::encodeVersitProperty(const QVersitProperty& property)
 
 /*! Performs Quoted-Printable encoding and charset encoding on \a value as per vCard 2.1 spec.
     Returns true if the value will need to be encoded with UTF-8, false if mCodec is sufficient. */
-bool QVCard21Writer::encodeVersitValue(QMultiHash<QString,QString>& parameters, QString& value)
+void QVCard21Writer::encodeVersitValue(QMultiHash<QString,QString>& parameters, QString& value)
 {
-    // Quoted-Printable encode the value and add Quoted-Printable parameter, if necessary
-    if (quotedPrintableEncode(value))
-        parameters.insert(QLatin1String("ENCODING"), QLatin1String("QUOTED-PRINTABLE"));
-
     // Add the CHARSET parameter, if necessary and encode in UTF-8 later
-    if (!mCodec->canEncode(value)) {
+    if (!mCodec->canEncode(value)
+            // if codec is ASCII and there is a character > U+007F in value, encode it as UTF-8
+            || (mCodecIsAscii && containsNonAscii(value))) {
         parameters.insert(QLatin1String("CHARSET"), QLatin1String("UTF-8"));
-        return true;
+        value = QString::fromLatin1(utf8Encoder()->fromUnicode(value));
     }
-    return false;
+
+    // Quoted-Printable encode the value and add Quoted-Printable parameter, if necessary
+    if (quotedPrintableEncode(value))
+        parameters.insert(QLatin1String("ENCODING"), QLatin1String("QUOTED-PRINTABLE"));
 }
 
 int sortIndexOfTypeValue(const QString& type) {
@@ -196,14 +207,21 @@ void QVCard21Writer::encodeParameters(const QMultiHash<QString,QString>& paramet
     }
 }
 
-
+bool QVCard21Writer::containsNonAscii(const QString& str)
+{
+    for (int i = 0; i < str.length(); i++) {
+        if (str[i].unicode() > 127)
+            return true;
+    }
+    return false;
+}
 
 /*!
  * Encodes special characters in \a text
  * using Quoted-Printable encoding (RFC 1521).
  * Returns true if at least one character was encoded.
  */
-bool QVCard21Writer::quotedPrintableEncode(QString& text) const
+bool QVCard21Writer::quotedPrintableEncode(QString& text)
 {
     bool encoded = false;
     for (int i=0; i<text.length(); i++) {
@@ -223,7 +241,7 @@ bool QVCard21Writer::quotedPrintableEncode(QString& text) const
 /*!
  * Checks whether the \a chr should be Quoted-Printable encoded (RFC 1521).
  */
-bool QVCard21Writer::shouldBeQuotedPrintableEncoded(QChar chr) const
+bool QVCard21Writer::shouldBeQuotedPrintableEncoded(QChar chr)
 {
     int c = chr.unicode();
     return (c < 32 ||
diff --git a/src/versit/qvcard21writer_p.h b/src/versit/qvcard21writer_p.h
index 3bbb776766..8784dcb9d2 100644
--- a/src/versit/qvcard21writer_p.h
+++ b/src/versit/qvcard21writer_p.h
@@ -65,10 +65,14 @@ public:
     ~QVCard21Writer();
 
     void encodeVersitProperty(const QVersitProperty& property);
-    bool encodeVersitValue(QMultiHash<QString,QString>& parameters, QString& value);
+    void encodeVersitValue(QMultiHash<QString,QString>& parameters, QString& value);
     void encodeParameters(const QMultiHash<QString,QString>& parameters);
-    bool quotedPrintableEncode(QString& text) const;
-    bool shouldBeQuotedPrintableEncoded(QChar chr) const;
+    static bool containsNonAscii(const QString& str);
+    static bool quotedPrintableEncode(QString& text);
+    static bool shouldBeQuotedPrintableEncoded(QChar chr);
+
+private:
+    static QTextEncoder* utf8Encoder();
 };
 
 QTM_END_NAMESPACE
diff --git a/src/versit/qversitdocumentwriter_p.cpp b/src/versit/qversitdocumentwriter_p.cpp
index e6f7ca4dd7..f81cfdb006 100644
--- a/src/versit/qversitdocumentwriter_p.cpp
+++ b/src/versit/qversitdocumentwriter_p.cpp
@@ -64,20 +64,17 @@ QVersitDocumentWriter::QVersitDocumentWriter(QVersitDocument::VersitType type)
     : mType(type),
     mDevice(0),
     mCodec(0),
+    mCodecIsAscii(false),
     mEncoder(0),
-    mUtf8Encoder(QTextCodec::codecForName("UTF-8")->makeEncoder()),
     mSuccessful(true),
     mCurrentLineLength(0)
 {
-    // Hack so the encoder doesn't output a byte order mark for UTF-8.
-    mUtf8Encoder->fromUnicode(QString());
 }
 
 QVersitDocumentWriter::~QVersitDocumentWriter()
 {
     if (mEncoder)
         delete mEncoder;
-    delete mUtf8Encoder;
 }
 
 /*!
@@ -101,6 +98,15 @@ void QVersitDocumentWriter::setCodec(QTextCodec *codec)
 }
 
 /*!
+  Specifies that the codec is actually ASCII (setCodec must also be called with an ASCII-compatible
+  codec.
+ */
+void QVersitDocumentWriter::setAsciiCodec()
+{
+    mCodecIsAscii = true;
+}
+
+/*!
   Sets the device to write to.
   */
 void QVersitDocumentWriter::setDevice(QIODevice *device)
@@ -205,14 +211,12 @@ void QVersitDocumentWriter::writeBytes(const QByteArray &value)
 
 /*!
   Writes \a value to the device.
-  If \a useUtf8 is true, uses the UTF-8 codec instead of the one set in setCodec().
 
   This function tracks how many characters have been written to the line and folds (wraps) the line
   according to RFC2425.
   */
-void QVersitDocumentWriter::writeString(const QString &value, bool useUtf8)
+void QVersitDocumentWriter::writeString(const QString &value)
 {
-    QTextEncoder* encoder = useUtf8 ? mUtf8Encoder : mEncoder;
     int spaceRemaining = MAX_LINE_LENGTH - mCurrentLineLength;
     int charsWritten = 0;
     QString crlfSpace(QLatin1String("\r\n "));
@@ -220,14 +224,14 @@ void QVersitDocumentWriter::writeString(const QString &value, bool useUtf8)
         // Write the first "spaceRemaining" characters
         QStringRef line(&value, charsWritten, spaceRemaining);
         charsWritten += spaceRemaining;
-        if (mDevice->write(encoder->fromUnicode(line.constData(), line.length())) < 0
-               || mDevice->write(encoder->fromUnicode(crlfSpace)) < 0)
+        if (mDevice->write(mEncoder->fromUnicode(line.constData(), line.length())) < 0
+               || mDevice->write(mEncoder->fromUnicode(crlfSpace)) < 0)
             mSuccessful = false;
         spaceRemaining = MAX_LINE_LENGTH - 1; // minus 1 for the space at the front.
         mCurrentLineLength = 1;
     }
 
-    if (mDevice->write(encoder->fromUnicode(value.mid(charsWritten))) < 0)
+    if (mDevice->write(mEncoder->fromUnicode(value.mid(charsWritten))) < 0)
         mSuccessful = false;
     mCurrentLineLength += value.length() - charsWritten;
 }
diff --git a/src/versit/qversitdocumentwriter_p.h b/src/versit/qversitdocumentwriter_p.h
index ac43ec7242..f3c86dd18d 100644
--- a/src/versit/qversitdocumentwriter_p.h
+++ b/src/versit/qversitdocumentwriter_p.h
@@ -72,6 +72,7 @@ public:
     virtual ~QVersitDocumentWriter();
 
     void setCodec(QTextCodec* codec);
+    void setAsciiCodec();
     void setDevice(QIODevice* device);
 
     virtual void encodeVersitProperty(const QVersitProperty& property) = 0;
@@ -80,16 +81,16 @@ public:
     void encodeGroupsAndName(const QVersitProperty& property);
 
     void writeBytes(const QByteArray& value);
-    void writeString(const QString& value, bool useUtf8 = false);
+    void writeString(const QString& value);
     void writeCrlf();
 
 protected:
     QVersitDocument::VersitType mType;
     QIODevice* mDevice;
     QTextCodec* mCodec;
+    bool mCodecIsAscii;
     bool mCodecIsAsciiCompatible;
     QTextEncoder* mEncoder;
-    QTextEncoder* mUtf8Encoder;
     bool mSuccessful;
     int mCurrentLineLength;
 };
diff --git a/src/versit/qversitwriter_p.cpp b/src/versit/qversitwriter_p.cpp
index 5803ffe051..88971d7f72 100644
--- a/src/versit/qversitwriter_p.cpp
+++ b/src/versit/qversitwriter_p.cpp
@@ -93,16 +93,19 @@ void QVersitWriterPrivate::write()
             break;
         }
 
-        QScopedPointer<QVersitDocumentWriter> writer(
-                writerForType( // ... get type from the document if not specified in startWriting
-                    type == QVersitDocument::InvalidType ? document.type() : type,
-                    document));
+        // Get type from the document if not specified in startWriting
+        if (type == QVersitDocument::InvalidType)
+            type = document.type();
+
+        QScopedPointer<QVersitDocumentWriter> writer(writerForType(type, document));
         QTextCodec* codec = mDefaultCodec;
         if (codec == NULL) {
-            if (type == QVersitDocument::VCard21Type)
+            if (type == QVersitDocument::VCard21Type) {
                 codec = QTextCodec::codecForName("ISO-8859-1");
-            else
+                writer->setAsciiCodec();
+            } else {
                 codec = QTextCodec::codecForName("UTF-8");
+            }
         }
         writer->setCodec(codec);
         writer->setDevice(mIoDevice);
diff --git a/tests/auto/qvcard21writer/tst_qvcard21writer.cpp b/tests/auto/qvcard21writer/tst_qvcard21writer.cpp
index 53db651950..df6a5fc431 100644
--- a/tests/auto/qvcard21writer/tst_qvcard21writer.cpp
+++ b/tests/auto/qvcard21writer/tst_qvcard21writer.cpp
@@ -175,7 +175,10 @@ END:VCARD\r\n\
     QTest::newRow("base64 encoded") << property << expectedResult << codec;
 
     // Characters other than ASCII:
-    expectedResult = "ORG;CHARSET=UTF-8:" + KATAKANA_NOKIA.toUtf8() + "\r\n";
+    // Note: KATAKANA_NOKIA is defined as: QString::fromUtf8("\xe3\x83\x8e\xe3\x82\xad\xe3\x82\xa2")
+    // The expected behaviour is to convert to UTF8, then encode with quoted-printable
+    expectedResult = "ORG;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=E3=83=8E=E3=82=AD=E3=82=A2\r\n";
+
     property = QVersitProperty();
     property.setName(QLatin1String("ORG"));
     property.setValue(KATAKANA_NOKIA);
@@ -189,14 +192,6 @@ END:VCARD\r\n\
     property.setName(QLatin1String("ORG"));
     property.setValue(KATAKANA_NOKIA);
     QTest::newRow("JIS codec") << property << expectedResult << QByteArray("Shift-JIS");
-
-    // CHARSET and QUOTED-PRINTABLE
-    expectedResult = "EMAIL;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:john=40"
-                     + KATAKANA_NOKIA.toUtf8() + ".com\r\n";
-    property = QVersitProperty();
-    property.setName(QLatin1String("EMAIL"));
-    property.setValue(QString::fromAscii("john@%1.com").arg(KATAKANA_NOKIA));
-    QTest::newRow("Charset and QP") << property << expectedResult << codec;
 }
 
 void tst_QVCard21Writer::testEncodeParameters()