Markdown: fix several issues with lists and continuations

Importer fixes:
- the first list item after a heading doesn't keep the heading font
- the first text fragment after a bullet is the bullet text, not a
  separate paragraph
- detect continuation lines and append to the list item text
- detect continuation paragraphs and indent them properly
- indent nested list items properly
- add a test for QTextMarkdownImporter
Writer fixes:
- after bullet items, continuation lines and paragraphs are indented
- indentation of continuations isn't affected by checkboxes
- add extra newlines between list items in "loose" lists
- avoid writing triple newlines
- enhance the test for QTextMarkdownWriter

Change-Id: Ib1dda514832f6dc0cdad177aa9a423a7038ac8c6
Reviewed-by: Gatis Paeglis <gatis.paeglis@qt.io>

6 files changed, 258 insertions, 21 deletions

diff --git a/tests/auto/gui/text/qtextmarkdownimporter/data/headingBulletsContinuations.md b/tests/auto/gui/text/qtextmarkdownimporter/data/headingBulletsContinuations.md
new file mode 100644
index 0000000000..99eb633d17
--- /dev/null
+++ b/tests/auto/gui/text/qtextmarkdownimporter/data/headingBulletsContinuations.md
@@ -0,0 +1,28 @@
+# heading
+- bullet 1
+	continuation line 1, indented via tab
+- bullet 2
+    continuation line 2, indented via 4 spaces
+- bullet 3
+
+	continuation paragraph 3, indented via tab
+
+  - bullet 3.1
+
+    continuation paragraph 3.1, indented via 4 spaces
+
+  - bullet 3.2
+                continuation line, indented via 2 tabs
+- bullet 4
+
+    continuation paragraph 4, indented via 4 spaces
+    and continuing onto another line too
+
+- bullet 5
+
+  continuation paragraph 5, indented via 2 spaces and continuing onto another
+  line too
+
+- bullet 6
+
+plain old paragraph at the end
diff --git a/tests/auto/gui/text/qtextmarkdownimporter/qtextmarkdownimporter.pro b/tests/auto/gui/text/qtextmarkdownimporter/qtextmarkdownimporter.pro
new file mode 100644
index 0000000000..3b63a67228
--- /dev/null
+++ b/tests/auto/gui/text/qtextmarkdownimporter/qtextmarkdownimporter.pro
@@ -0,0 +1,7 @@
+CONFIG += testcase
+TARGET = tst_qtextmarkdownimporter
+QT += core-private gui-private testlib
+SOURCES += tst_qtextmarkdownimporter.cpp
+TESTDATA += data/headingBulletsContinuations.md
+
+DEFINES += SRCDIR=\\\"$$PWD\\\"
diff --git a/tests/auto/gui/text/qtextmarkdownimporter/tst_qtextmarkdownimporter.cpp b/tests/auto/gui/text/qtextmarkdownimporter/tst_qtextmarkdownimporter.cpp
new file mode 100644
index 0000000000..2ede2e73ad
--- /dev/null
+++ b/tests/auto/gui/text/qtextmarkdownimporter/tst_qtextmarkdownimporter.cpp
@@ -0,0 +1,121 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtTest/QtTest>
+#include <QTextDocument>
+#include <QTextCursor>
+#include <QTextBlock>
+#include <QTextList>
+#include <QTextTable>
+#include <QBuffer>
+#include <QDebug>
+
+#include <private/qtextmarkdownimporter_p.h>
+
+// #define DEBUG_WRITE_HTML
+
+Q_LOGGING_CATEGORY(lcTests, "qt.text.tests")
+
+static const QChar LineBreak = QChar(0x2028);
+static const QChar Tab = QLatin1Char('\t');
+static const QChar Space = QLatin1Char(' ');
+static const QChar Period = QLatin1Char('.');
+
+class tst_QTextMarkdownImporter : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void headingBulletsContinuations();
+};
+
+void tst_QTextMarkdownImporter::headingBulletsContinuations()
+{
+    const QStringList expectedBlocks = QStringList() <<
+        "" << // we could do without this blank line before the heading, but currently it happens
+        "heading" <<
+        "bullet 1 continuation line 1, indented via tab" <<
+        "bullet 2 continuation line 2, indented via 4 spaces" <<
+        "bullet 3" <<
+        "continuation paragraph 3, indented via tab" <<
+        "bullet 3.1" <<
+        "continuation paragraph 3.1, indented via 4 spaces" <<
+        "bullet 3.2 continuation line, indented via 2 tabs" <<
+        "bullet 4" <<
+        "continuation paragraph 4, indented via 4 spaces and continuing onto another line too" <<
+        "bullet 5" <<
+        // indenting by only 2 spaces is perhaps non-standard but currently is OK
+        "continuation paragraph 5, indented via 2 spaces and continuing onto another line too" <<
+        "bullet 6" <<
+        "plain old paragraph at the end";
+
+    QFile f(QFINDTESTDATA("data/headingBulletsContinuations.md"));
+    QVERIFY(f.open(QFile::ReadOnly | QIODevice::Text));
+    QString md = QString::fromUtf8(f.readAll());
+    f.close();
+
+    QTextDocument doc;
+    QTextMarkdownImporter(QTextMarkdownImporter::DialectGitHub).import(&doc, md);
+    QTextFrame::iterator iterator = doc.rootFrame()->begin();
+    QTextFrame *currentFrame = iterator.currentFrame();
+    QStringList::const_iterator expectedIt = expectedBlocks.constBegin();
+    int i = 0;
+    while (!iterator.atEnd()) {
+        // There are no child frames
+        QCOMPARE(iterator.currentFrame(), currentFrame);
+        // Check whether we got the right child block
+        QTextBlock block = iterator.currentBlock();
+        QCOMPARE(block.text().contains(LineBreak), false);
+        QCOMPARE(block.text().contains(Tab), false);
+        QVERIFY(!block.text().startsWith(Space));
+        int expectedIndentation = 0;
+        if (block.text().contains(QLatin1String("continuation paragraph")))
+            expectedIndentation = (block.text().contains(Period) ? 2 : 1);
+        qCDebug(lcTests) << i << "child block" << block.text() << "indentation" << block.blockFormat().indent();
+        QVERIFY(expectedIt != expectedBlocks.constEnd());
+        QCOMPARE(block.text(), *expectedIt);
+        if (i > 2)
+            QCOMPARE(block.blockFormat().indent(), expectedIndentation);
+        ++iterator;
+        ++expectedIt;
+        ++i;
+    }
+    QCOMPARE(expectedIt, expectedBlocks.constEnd());
+
+#ifdef DEBUG_WRITE_HTML
+    {
+        QFile out("/tmp/headingBulletsContinuations.html");
+        out.open(QFile::WriteOnly);
+        out.write(doc.toHtml().toLatin1());
+        out.close();
+    }
+#endif
+}
+
+QTEST_MAIN(tst_QTextMarkdownImporter)
+#include "tst_qtextmarkdownimporter.moc"
diff --git a/tests/auto/gui/text/qtextmarkdownwriter/data/example.md b/tests/auto/gui/text/qtextmarkdownwriter/data/example.md
index 3c63f209a2..0c3f34e09d 100644
--- a/tests/auto/gui/text/qtextmarkdownwriter/data/example.md
+++ b/tests/auto/gui/text/qtextmarkdownwriter/data/example.md
@@ -27,8 +27,8 @@ text layout changes.*
 Different kinds of lists can be included in rich text documents. Standard
 bullet lists can be nested, using different symbols for each level of the list:
 
-* Disc symbols are typically used for top-level list items.
-  - Circle symbols can be used to distinguish between items in lower-level
+- Disc symbols are typically used for top-level list items.
+  * Circle symbols can be used to distinguish between items in lower-level
     lists.
     + Square symbols provide a reasonable alternative to discs and circles.
 
@@ -36,13 +36,13 @@ Ordered lists can be created that can be used for tables of contents. Different
 characters can be used to enumerate items, and we can use both Roman and Arabic
 numerals in the same list structure:
 
-1. Introduction
-2. Qt Tools
-    1) Qt Assistant
-    2) Qt Designer
-        1. Form Editor
-        2. Component Architecture
-    3) Qt Linguist
+1.  Introduction
+2.  Qt Tools
+    1)  Qt Assistant
+    2)  Qt Designer
+        1.  Form Editor
+        2.  Component Architecture
+    3)  Qt Linguist
 
 The list will automatically be renumbered if you add or remove items. *Try
 adding new sections to the above list or removing existing item to see the
diff --git a/tests/auto/gui/text/qtextmarkdownwriter/tst_qtextmarkdownwriter.cpp b/tests/auto/gui/text/qtextmarkdownwriter/tst_qtextmarkdownwriter.cpp
index bf7c9708de..dca6d90a48 100644
--- a/tests/auto/gui/text/qtextmarkdownwriter/tst_qtextmarkdownwriter.cpp
+++ b/tests/auto/gui/text/qtextmarkdownwriter/tst_qtextmarkdownwriter.cpp
@@ -50,6 +50,7 @@ private slots:
     void testWriteParagraph_data();
     void testWriteParagraph();
     void testWriteList();
+    void testWriteNestedBulletLists_data();
     void testWriteNestedBulletLists();
     void testWriteNestedNumericLists();
     void testWriteTable();
@@ -122,9 +123,44 @@ void tst_QTextMarkdownWriter::testWriteList()
         "- ListItem 1\n- ListItem 2\n"));
 }
 
+void tst_QTextMarkdownWriter::testWriteNestedBulletLists_data()
+{
+    QTest::addColumn<bool>("checkbox");
+    QTest::addColumn<bool>("checked");
+    QTest::addColumn<bool>("continuationLine");
+    QTest::addColumn<bool>("continuationParagraph");
+    QTest::addColumn<QString>("expectedOutput");
+
+    QTest::newRow("plain bullets") << false << false << false << false <<
+        "- ListItem 1\n  * ListItem 2\n    + ListItem 3\n- ListItem 4\n  * ListItem 5\n";
+    QTest::newRow("bullets with continuation lines") << false << false << true << false <<
+        "- ListItem 1\n  * ListItem 2\n    + ListItem 3 with text that won't fit on one line and thus needs a\n      continuation\n- ListItem 4\n  * ListItem 5 with text that won't fit on one line and thus needs a\n    continuation\n";
+    QTest::newRow("bullets with continuation paragraphs") << false << false << false << true <<
+        "- ListItem 1\n\n  * ListItem 2\n    + ListItem 3\n\n      continuation\n\n- ListItem 4\n\n  * ListItem 5\n\n    continuation\n\n";
+    QTest::newRow("unchecked") << true << false << false << false <<
+        "- [ ] ListItem 1\n  * [ ] ListItem 2\n    + [ ] ListItem 3\n- [ ] ListItem 4\n  * [ ] ListItem 5\n";
+    QTest::newRow("checked") << true << true << false << false <<
+        "- [x] ListItem 1\n  * [x] ListItem 2\n    + [x] ListItem 3\n- [x] ListItem 4\n  * [x] ListItem 5\n";
+    QTest::newRow("checked with continuation lines") << true << true << true << false <<
+        "- [x] ListItem 1\n  * [x] ListItem 2\n    + [x] ListItem 3 with text that won't fit on one line and thus needs a\n      continuation\n- [x] ListItem 4\n  * [x] ListItem 5 with text that won't fit on one line and thus needs a\n    continuation\n";
+    QTest::newRow("checked with continuation paragraphs") << true << true << false << true <<
+        "- [x] ListItem 1\n\n  * [x] ListItem 2\n    + [x] ListItem 3\n\n      continuation\n\n- [x] ListItem 4\n\n  * [x] ListItem 5\n\n    continuation\n\n";
+}
+
 void tst_QTextMarkdownWriter::testWriteNestedBulletLists()
 {
+    QFETCH(bool, checkbox);
+    QFETCH(bool, checked);
+    QFETCH(bool, continuationParagraph);
+    QFETCH(bool, continuationLine);
+    QFETCH(QString, expectedOutput);
+
     QTextCursor cursor(document);
+    QTextBlockFormat blockFmt = cursor.blockFormat();
+    if (checkbox) {
+        blockFmt.setMarker(checked ? QTextBlockFormat::Checked : QTextBlockFormat::Unchecked);
+        cursor.setBlockFormat(blockFmt);
+    }
 
     QTextList *list1 = cursor.createList(QTextListFormat::ListDisc);
     cursor.insertText("ListItem 1");
@@ -140,18 +176,42 @@ void tst_QTextMarkdownWriter::testWriteNestedBulletLists()
     fmt3.setStyle(QTextListFormat::ListSquare);
     fmt3.setIndent(3);
     cursor.insertList(fmt3);
-    cursor.insertText("ListItem 3");
+    cursor.insertText(continuationLine ?
+                          "ListItem 3 with text that won't fit on one line and thus needs a continuation" :
+                          "ListItem 3");
+    if (continuationParagraph) {
+        QTextBlockFormat blockFmt;
+        blockFmt.setIndent(2);
+        cursor.insertBlock(blockFmt);
+        cursor.insertText("continuation");
+    }
 
-    cursor.insertBlock();
+    cursor.insertBlock(blockFmt);
     cursor.insertText("ListItem 4");
     list1->add(cursor.block());
 
     cursor.insertBlock();
-    cursor.insertText("ListItem 5");
+    cursor.insertText(continuationLine ?
+                          "ListItem 5 with text that won't fit on one line and thus needs a continuation" :
+                          "ListItem 5");
     list2->add(cursor.block());
+    if (continuationParagraph) {
+        QTextBlockFormat blockFmt;
+        blockFmt.setIndent(2);
+        cursor.insertBlock(blockFmt);
+        cursor.insertText("continuation");
+    }
 
-    QCOMPARE(documentToUnixMarkdown(), QString::fromLatin1(
-                 "- ListItem 1\n  * ListItem 2\n    + ListItem 3\n- ListItem 4\n  * ListItem 5\n"));
+    QString output = documentToUnixMarkdown();
+#ifdef DEBUG_WRITE_OUTPUT
+    {
+        QFile out("/tmp/" + QLatin1String(QTest::currentDataTag()) + ".md");
+        out.open(QFile::WriteOnly);
+        out.write(output.toUtf8());
+        out.close();
+    }
+#endif
+    QCOMPARE(documentToUnixMarkdown(), expectedOutput);
 }
 
 void tst_QTextMarkdownWriter::testWriteNestedNumericLists()
@@ -185,7 +245,7 @@ void tst_QTextMarkdownWriter::testWriteNestedNumericLists()
 
     // There's no QTextList API to set the starting number so we hard-coded all lists to start at 1 (QTBUG-65384)
     QCOMPARE(documentToUnixMarkdown(), QString::fromLatin1(
-                 "1 ListItem 1\n    1) ListItem 2\n        1 ListItem 3\n2 ListItem 4\n    2) ListItem 5\n"));
+                 "1.  ListItem 1\n    1)  ListItem 2\n        1.  ListItem 3\n2.  ListItem 4\n    2)  ListItem 5\n"));
 }
 
 void tst_QTextMarkdownWriter::testWriteTable()
@@ -312,8 +372,8 @@ void tst_QTextMarkdownWriter::rewriteDocument()
 
 void tst_QTextMarkdownWriter::fromHtml_data()
 {
-    QTest::addColumn<QString>("input");
-    QTest::addColumn<QString>("output");
+    QTest::addColumn<QString>("expectedInput");
+    QTest::addColumn<QString>("expectedOutput");
 
     QTest::newRow("long URL") <<
         "<span style=\"font-style:italic;\">https://www.example.com/dir/subdir/subsubdir/subsubsubdir/subsubsubsubdir/subsubsubsubsubdir/</span>" <<
@@ -329,6 +389,15 @@ void tst_QTextMarkdownWriter::fromHtml_data()
     QTest::newRow("escaped hyphen after newline") <<
         "The first sentence of this paragraph holds 80 characters, then there's a minus. - This is wrapped, but is <em>not</em> a bullet point." <<
         "The first sentence of this paragraph holds 80 characters, then there's a minus.\n\\- This is wrapped, but is *not* a bullet point.\n\n";
+    QTest::newRow("list items with indented continuations") <<
+        "<ul><li>bullet<p>continuation paragraph</p></li><li>another bullet<br/>continuation line</li></ul>" <<
+        "- bullet\n\n  continuation paragraph\n\n- another bullet\n  continuation line\n";
+    QTest::newRow("nested list items with continuations") <<
+        "<ul><li>bullet<p>continuation paragraph</p></li><li>another bullet<br/>continuation line</li><ul><li>bullet<p>continuation paragraph</p></li><li>another bullet<br/>continuation line</li></ul></ul>" <<
+        "- bullet\n\n  continuation paragraph\n\n- another bullet\n  continuation line\n\n  - bullet\n\n    continuation paragraph\n\n  - another bullet\n    continuation line\n";
+    QTest::newRow("nested ordered list items with continuations") <<
+        "<ol><li>item<p>continuation paragraph</p></li><li>another item<br/>continuation line</li><ol><li>item<p>continuation paragraph</p></li><li>another item<br/>continuation line</li></ol><li>another</li><li>another</li></ol>" <<
+        "1.  item\n\n    continuation paragraph\n\n2.  another item\n    continuation line\n\n    1.  item\n\n        continuation paragraph\n\n    2.  another item\n        continuation line\n\n3.  another\n4.  another\n";
     // TODO
 //    QTest::newRow("escaped number and paren after double newline") <<
 //        "<p>(The first sentence of this paragraph is a line, the next paragraph has a number</p>13) but that's not part of an ordered list" <<
@@ -340,11 +409,22 @@ void tst_QTextMarkdownWriter::fromHtml_data()
 
 void tst_QTextMarkdownWriter::fromHtml()
 {
-    QFETCH(QString, input);
-    QFETCH(QString, output);
+    QFETCH(QString, expectedInput);
+    QFETCH(QString, expectedOutput);
 
-    document->setHtml(input);
-    QCOMPARE(documentToUnixMarkdown(), output);
+    document->setHtml(expectedInput);
+    QString output = documentToUnixMarkdown();
+
+#ifdef DEBUG_WRITE_OUTPUT
+    {
+        QFile out("/tmp/" + QLatin1String(QTest::currentDataTag()) + ".md");
+        out.open(QFile::WriteOnly);
+        out.write(output.toUtf8());
+        out.close();
+    }
+#endif
+
+    QCOMPARE(output, expectedOutput);
 }
 
 QString tst_QTextMarkdownWriter::documentToUnixMarkdown()
diff --git a/tests/auto/gui/text/text.pro b/tests/auto/gui/text/text.pro
index a98debe35c..794d9ea8d3 100644
--- a/tests/auto/gui/text/text.pro
+++ b/tests/auto/gui/text/text.pro
@@ -28,6 +28,7 @@ SUBDIRS=\
 
 win32:SUBDIRS -= qtextpiecetable
 
+qtConfig(textmarkdownreader): SUBDIRS += qtextmarkdownimporter
 qtConfig(textmarkdownwriter): SUBDIRS += qtextmarkdownwriter
 
 !qtConfig(private_tests): SUBDIRS -= \