From de71c8a6f827b7ac1a0c394a92c113edbe6d5df7 Mon Sep 17 00:00:00 2001 From: Alexander Volkov Date: Thu, 30 Oct 2014 19:34:19 +0400 Subject: Add QTextStream::readLine() overload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The most common use case for QTextStream::readLine() is reading a file line by line in a loop. The existing readLine() method allocates new memory for each line, that results in a loss of speed. The introduced overload can use already allocated memory. Besides it allows you to not think about filesystem specifics. The current QFile documentation suggests a separate way to read files from /proc filesystem. With this overload it's possible to use the same idiom in all cases: QTextStream in(&file); QString line; while (in.readLine(&line)) { process_line(line); } The idea was inspired by the blog post of Ivan Čukić: http://ivan.fomentgroup.org/blog/2014/10/03/api-design-and-impact-on-the-performance-qt-vs-stl-example/ Change-Id: I0c62b4a52681870589bc099905e83ed69e03dd40 Reviewed-by: Martin Smith Reviewed-by: David Faure Reviewed-by: Oswald Buddenhagen --- src/corelib/io/qtextstream.cpp | 53 ++++++++- src/corelib/io/qtextstream.h | 1 + .../corelib/io/qtextstream/tst_qtextstream.cpp | 121 ++++++++++++++++++--- 3 files changed, 155 insertions(+), 20 deletions(-) diff --git a/src/corelib/io/qtextstream.cpp b/src/corelib/io/qtextstream.cpp index 771067fa14..ff3b007b21 100644 --- a/src/corelib/io/qtextstream.cpp +++ b/src/corelib/io/qtextstream.cpp @@ -1580,18 +1580,61 @@ QString QTextStream::readAll() \sa readAll(), QIODevice::readLine() */ QString QTextStream::readLine(qint64 maxlen) +{ + QString line; + + readLine(&line, maxlen); + return line; +} + +/*! + \since 5.5 + + Reads one line of text from the stream into \a line. + If \a line is 0, the read line is not stored. + + The maximum allowed line length is set to \a maxlen. If + the stream contains lines longer than this, then the lines will be + split after \a maxlen characters and returned in parts. + + If \a maxlen is 0, the lines can be of any length. + + The resulting line has no trailing end-of-line characters ("\\n" + or "\\r\\n"), so calling QString::trimmed() can be unnecessary. + + If \a line has sufficient capacity for the data that is about to be + read, this function may not need to allocate new memory. Because of + this, it can be faster than the other readLine() overload. + + Returns \c false if the stream has read to the end of the file or + an error has occurred; otherwise returns \c true. The contents in + \a line before the call are discarded in any case. + + \sa readAll(), QIODevice::readLine() +*/ +bool QTextStream::readLine(QString *line, qint64 maxlen) { Q_D(QTextStream); - CHECK_VALID_STREAM(QString()); + // keep in sync with CHECK_VALID_STREAM + if (!d->string && !d->device) { + qWarning("QTextStream: No device"); + if (line && !line->isNull()) + line->resize(0); + return false; + } const QChar *readPtr; int length; - if (!d->scan(&readPtr, &length, int(maxlen), QTextStreamPrivate::EndOfLine)) - return QString(); + if (!d->scan(&readPtr, &length, int(maxlen), QTextStreamPrivate::EndOfLine)) { + if (line && !line->isNull()) + line->resize(0); + return false; + } - QString tmp = QString(readPtr, length); + if (Q_LIKELY(line)) + line->setUnicode(readPtr, length); d->consumeLastToken(); - return tmp; + return true; } /*! diff --git a/src/corelib/io/qtextstream.h b/src/corelib/io/qtextstream.h index b15e7772e2..6f5e2473e4 100644 --- a/src/corelib/io/qtextstream.h +++ b/src/corelib/io/qtextstream.h @@ -124,6 +124,7 @@ public: void skipWhiteSpace(); QString readLine(qint64 maxlen = 0); + bool readLine(QString *line, qint64 maxlen = 0); QString readAll(); QString read(qint64 maxlen); diff --git a/tests/auto/corelib/io/qtextstream/tst_qtextstream.cpp b/tests/auto/corelib/io/qtextstream/tst_qtextstream.cpp index faa635fa96..84f9c53b21 100644 --- a/tests/auto/corelib/io/qtextstream/tst_qtextstream.cpp +++ b/tests/auto/corelib/io/qtextstream/tst_qtextstream.cpp @@ -81,6 +81,7 @@ private slots: void readLineMaxlen_data(); void readLineMaxlen(); void readLinesFromBufferCRCR(); + void readLineOverload(); // all void readAllFromDevice_data(); @@ -593,6 +594,63 @@ void tst_QTextStream::readLinesFromBufferCRCR() } } +class ErrorDevice : public QIODevice +{ +protected: + qint64 readData(char *data, qint64 maxlen) Q_DECL_OVERRIDE + { + Q_UNUSED(data) + Q_UNUSED(maxlen) + return -1; + } + + qint64 writeData(const char *data, qint64 len) Q_DECL_OVERRIDE + { + Q_UNUSED(data) + Q_UNUSED(len) + return -1; + } +}; + +void tst_QTextStream::readLineOverload() +{ + QByteArray data = "1\n2\n3"; + + QTextStream ts(&data); + QString line; + + ts.readLine(&line); + QCOMPARE(line, QStringLiteral("1")); + + ts.readLine(Q_NULLPTR, 0); // read the second line, but don't store it + + ts.readLine(&line); + QCOMPARE(line, QStringLiteral("3")); + + QVERIFY(!ts.readLine(&line)); + QVERIFY(line.isEmpty()); + + QFile file(m_rfc3261FilePath); + QVERIFY(file.open(QFile::ReadOnly)); + + ts.setDevice(&file); + line.reserve(1); + int maxLineCapacity = line.capacity(); + + while (ts.readLine(&line)) { + QVERIFY(line.capacity() >= maxLineCapacity); + maxLineCapacity = line.capacity(); + } + + line = "Test string"; + ErrorDevice errorDevice; + QVERIFY(errorDevice.open(QIODevice::ReadOnly)); + ts.setDevice(&errorDevice); + + QVERIFY(!ts.readLine(&line)); + QVERIFY(line.isEmpty()); +} + // ------------------------------------------------------------------------------ void tst_QTextStream::readLineFromString_data() { @@ -912,13 +970,28 @@ void tst_QTextStream::lineCount() } // ------------------------------------------------------------------------------ +struct CompareIndicesForArray +{ + int *array; + CompareIndicesForArray(int *array) : array(array) {} + bool operator() (const int i1, const int i2) + { + return array[i1] < array[i2]; + } +}; + void tst_QTextStream::performance() { // Phase #1 - test speed of reading a huge text file with QFile. QTime stopWatch; - int elapsed1 = 0; - int elapsed2 = 0; + const int N = 3; + const char * readMethods[N] = { + "QFile::readLine()", + "QTextStream::readLine()", + "QTextStream::readLine(QString *)" + }; + int elapsed[N] = {0, 0, 0}; stopWatch.restart(); int nlines1 = 0; @@ -930,7 +1003,7 @@ void tst_QTextStream::performance() file.readLine(); } - elapsed1 += stopWatch.elapsed(); + elapsed[0] = stopWatch.elapsed(); stopWatch.restart(); int nlines2 = 0; @@ -943,20 +1016,38 @@ void tst_QTextStream::performance() stream.readLine(); } - elapsed2 += stopWatch.elapsed(); + elapsed[1] = stopWatch.elapsed(); + stopWatch.restart(); + + int nlines3 = 0; + QFile file3(m_rfc3261FilePath); + QVERIFY(file3.open(QFile::ReadOnly)); + + QTextStream stream2(&file3); + QString line; + while (stream2.readLine(&line)) + ++nlines3; + + elapsed[2] = stopWatch.elapsed(); + QCOMPARE(nlines1, nlines2); + QCOMPARE(nlines2, nlines3); + + for (int i = 0; i < N; i++) { + qDebug("%s used %.3f seconds to read the file", readMethods[i], + elapsed[i] / 1000.0); + } + + int idx[N] = {0, 1, 2}; + std::sort(idx, idx + N, CompareIndicesForArray(elapsed)); - qDebug("QFile used %.2f seconds to read the file", - elapsed1 / 1000.0); - - qDebug("QTextStream used %.2f seconds to read the file", - elapsed2 / 1000.0); - if (elapsed2 > elapsed1) { - qDebug("QFile is %.2fx faster than QTextStream", - double(elapsed2) / double(elapsed1)); - } else { - qDebug("QTextStream is %.2fx faster than QFile", - double(elapsed1) / double(elapsed2)); + for (int i = 0; i < N-1; i++) { + int i1 = idx[i]; + int i2 = idx[i+1]; + qDebug("Reading by %s is %.2fx faster than by %s", + readMethods[i1], + double(elapsed[i2]) / double(elapsed[i1]), + readMethods[i2]); } } -- cgit v1.2.3