From b61933e6dfadd70655d1fad5344ba77da34587fc Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 25 Mar 2009 20:19:57 +0100 Subject: whitespace --- tools/linguist/shared/cpp.cpp | 2 +- tools/linguist/shared/po.cpp | 2 +- tools/linguist/shared/profileevaluator.cpp | 4 ++-- tools/linguist/shared/qph.cpp | 2 +- tools/linguist/shared/qscript.cpp | 2 +- tools/linguist/shared/qscript.g | 2 +- tools/linguist/shared/translator.h | 2 +- tools/linguist/shared/translatortools.cpp | 4 ++-- tools/linguist/shared/ts.cpp | 8 ++++---- tools/linguist/shared/ui.cpp | 4 ++-- 10 files changed, 16 insertions(+), 16 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/cpp.cpp b/tools/linguist/shared/cpp.cpp index 28616cc4bb..333ffbf709 100644 --- a/tools/linguist/shared/cpp.cpp +++ b/tools/linguist/shared/cpp.cpp @@ -784,7 +784,7 @@ static void parse(Translator *tor, const QString &initialContext, const QString if (yyTok == Tok_ColonColon) fullName.append(QString()); while (yyTok == Tok_ColonColon || yyTok == Tok_Ident) { - if (yyTok == Tok_Ident) + if (yyTok == Tok_Ident) fullName.append(yyIdent); yyTok = getToken(); } diff --git a/tools/linguist/shared/po.cpp b/tools/linguist/shared/po.cpp index e9375e96f1..5842771a55 100644 --- a/tools/linguist/shared/po.cpp +++ b/tools/linguist/shared/po.cpp @@ -389,7 +389,7 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd) for (; l != lines.size(); ++l) { QString line = lines.at(l); if (line.isEmpty()) - continue; + continue; if (isTranslationLine(line)) { bool isObsolete = line.startsWith(QLatin1String("#~ msgstr")); const QString prefix = QLatin1String(isObsolete ? "#~ " : ""); diff --git a/tools/linguist/shared/profileevaluator.cpp b/tools/linguist/shared/profileevaluator.cpp index 1e91f92ced..aa529c1f00 100644 --- a/tools/linguist/shared/profileevaluator.cpp +++ b/tools/linguist/shared/profileevaluator.cpp @@ -711,7 +711,7 @@ QStringList ProFileEvaluator::Private::qmakeFeaturePaths() // if (!specdir.cdUp() || specdir.isRoot()) // break; // if (QFile::exists(specdir.path() + QDir::separator() + "features")) { - // foreach (const QString &concat_it, concat) + // foreach (const QString &concat_it, concat) // feature_roots << (specdir.path() + concat_it); // break; // } @@ -1373,7 +1373,7 @@ bool ProFileEvaluator::Private::evaluateConditionalFunction(const QString &funct parents.append(proFile->fileName()); if (!parents.isEmpty()) parents.takeLast(); - if (parents.isEmpty()) + if (parents.isEmpty()) q->fileMessage(format("Project ERROR: %1").arg(msg)); else q->fileMessage(format("Project ERROR: %1. File was included from: '%2'") diff --git a/tools/linguist/shared/qph.cpp b/tools/linguist/shared/qph.cpp index 45d3a20a28..18f1634935 100644 --- a/tools/linguist/shared/qph.cpp +++ b/tools/linguist/shared/qph.cpp @@ -101,7 +101,7 @@ bool QPHReader::read(Translator &translator) m_currentField = TargetField; else if (name() == QLatin1String("definition")) m_currentField = DefinitionField; - else + else m_currentField = NoField; } else if (isWhiteSpace()) { // ignore these diff --git a/tools/linguist/shared/qscript.cpp b/tools/linguist/shared/qscript.cpp index 9ab5e160fd..7377cba61d 100644 --- a/tools/linguist/shared/qscript.cpp +++ b/tools/linguist/shared/qscript.cpp @@ -2382,7 +2382,7 @@ bool loadQScript(Translator &translator, QIODevice &dev, ConversionData &cd) return true; } -bool saveQScript(const Translator &translator, QIODevice &dev, ConversionData &cd) +bool saveQScript(const Translator &translator, QIODevice &dev, ConversionData &cd) { Q_UNUSED(dev); Q_UNUSED(translator); diff --git a/tools/linguist/shared/qscript.g b/tools/linguist/shared/qscript.g index c7ee80db70..8d332779a9 100644 --- a/tools/linguist/shared/qscript.g +++ b/tools/linguist/shared/qscript.g @@ -2012,7 +2012,7 @@ bool loadQScript(Translator &translator, QIODevice &dev, ConversionData &cd) return true; } -bool saveQScript(const Translator &translator, QIODevice &dev, ConversionData &cd) +bool saveQScript(const Translator &translator, QIODevice &dev, ConversionData &cd) { Q_UNUSED(dev); Q_UNUSED(translator); diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h index f4279da436..bbe765443e 100644 --- a/tools/linguist/shared/translator.h +++ b/tools/linguist/shared/translator.h @@ -88,7 +88,7 @@ public: QString m_targetFileName; QDir m_sourceDir; QDir m_targetDir; // FIXME: TS spefic - QStringList m_dropTags; // tags to be dropped + QStringList m_dropTags; // tags to be dropped QStringList m_errors; bool m_verbose; bool m_ignoreUnfinished; diff --git a/tools/linguist/shared/translatortools.cpp b/tools/linguist/shared/translatortools.cpp index dcff5468b4..96301d5d50 100644 --- a/tools/linguist/shared/translatortools.cpp +++ b/tools/linguist/shared/translatortools.cpp @@ -69,7 +69,7 @@ static int numberLength(const QString &s, int i) int pos = i; do { ++i; - } while (i < s.size() + } while (i < s.size() && (s.at(i).isDigit() || (isDigitFriendly(s[i]) && i + 1 < s.size() @@ -445,7 +445,7 @@ Translator merge(const Translator &tor, const Translator &virginTor, if (mv.sourceText().isEmpty()) { if (tor.contains(mv.context())) continue; - } else { + } else { if (tor.contains(mv.context(), mv.sourceText(), mv.comment())) continue; if (options & HeuristicSimilarText) { diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp index 2e7d40f331..d454e4ebd1 100644 --- a/tools/linguist/shared/ts.cpp +++ b/tools/linguist/shared/ts.cpp @@ -373,7 +373,7 @@ bool TSReader::read(Translator &translator) } else if (elementStarts(strtranslation)) { // QXmlStreamAttributes atts = attributes(); - QStringRef type = atts.value(strtype); + QStringRef type = atts.value(strtype); if (type == strunfinished) msg.setType(TranslatorMessage::Unfinished); else if (type == strobsolete) @@ -440,7 +440,7 @@ static QString protect(const QString &str) QString result; result.reserve(str.length() * 12 / 10); for (int i = 0; i != str.size(); ++i) { - uint c = str.at(i).unicode(); + uint c = str.at(i).unicode(); switch (c) { case '\"': result += QLatin1String("""); @@ -708,12 +708,12 @@ bool loadTS(Translator &translator, QIODevice &dev, ConversionData &cd) return reader.read(translator); } -bool saveTS11(const Translator &translator, QIODevice &dev, ConversionData &cd) +bool saveTS11(const Translator &translator, QIODevice &dev, ConversionData &cd) { return saveTS(translator, dev, cd, 11); } -bool saveTS20(const Translator &translator, QIODevice &dev, ConversionData &cd) +bool saveTS20(const Translator &translator, QIODevice &dev, ConversionData &cd) { return saveTS(translator, dev, cd, 20); } diff --git a/tools/linguist/shared/ui.cpp b/tools/linguist/shared/ui.cpp index 4b86714ee1..23a73e742c 100644 --- a/tools/linguist/shared/ui.cpp +++ b/tools/linguist/shared/ui.cpp @@ -151,7 +151,7 @@ bool UiReader::fatalError(const QXmlParseException &exception) msg.sprintf("XML error: Parse error at line %d, column %d (%s).", exception.lineNumber(), exception.columnNumber(), exception.message().toLatin1().data()); - m_cd.appendError(msg); + m_cd.appendError(msg); return false; } @@ -188,7 +188,7 @@ bool loadUI(Translator &translator, QIODevice &dev, ConversionData &cd) return result; } -bool saveUI(const Translator &translator, QIODevice &dev, ConversionData &cd) +bool saveUI(const Translator &translator, QIODevice &dev, ConversionData &cd) { Q_UNUSED(dev); Q_UNUSED(translator); -- cgit v1.2.3 From ddc1ff555604543cc2a3b1b797ff61b67abb0fe4 Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 25 Mar 2009 20:23:06 +0100 Subject: proper encoding and escaping --- tools/linguist/shared/qph.cpp | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/qph.cpp b/tools/linguist/shared/qph.cpp index 18f1634935..799bf7dd83 100644 --- a/tools/linguist/shared/qph.cpp +++ b/tools/linguist/shared/qph.cpp @@ -133,14 +133,47 @@ static bool loadQPH(Translator &translator, QIODevice &dev, ConversionData &cd) return reader.read(translator); } -static bool saveQPH(const Translator &translator, QIODevice &dev, ConversionData &cd) +static QString protect(const QString &str) +{ + QString result; + result.reserve(str.length() * 12 / 10); + for (int i = 0; i != str.size(); ++i) { + uint c = str.at(i).unicode(); + switch (c) { + case '\"': + result += QLatin1String("""); + break; + case '&': + result += QLatin1String("&"); + break; + case '>': + result += QLatin1String(">"); + break; + case '<': + result += QLatin1String("<"); + break; + case '\'': + result += QLatin1String("'"); + break; + default: + if (c < 0x20 && c != '\r' && c != '\n' && c != '\t') + result += QString(QLatin1String("&#%1;")).arg(c); + else // this also covers surrogates + result += QChar(c); + } + } + return result; +} + +static bool saveQPH(const Translator &translator, QIODevice &dev, ConversionData &) { QTextStream t(&dev); - t << "\n"; + t.setCodec(QTextCodec::codecForName("UTF-8")); + t << "\n\n"; foreach (const TranslatorMessage &msg, translator.messages()) { t << "\n"; - t << " " << msg.sourceText() << "\n"; - t << " " << msg.translations().join(QLatin1String("@")) + t << " " << protect(msg.sourceText()) << "\n"; + t << " " << protect(msg.translations().join(QLatin1String("@"))) << "\n"; if (!msg.context().isEmpty() || !msg.comment().isEmpty()) t << " " << msg.context() << msg.comment() -- cgit v1.2.3 From 6ac130466400197b8977f60d14ed96abfb0d2172 Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 25 Mar 2009 20:32:13 +0100 Subject: beautify / micro-optimize --- tools/linguist/shared/translator.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp index 3b5e8f3fba..6ae8059aec 100644 --- a/tools/linguist/shared/translator.cpp +++ b/tools/linguist/shared/translator.cpp @@ -105,13 +105,14 @@ void Translator::extend(const TranslatorMessage &msg) if (index == -1) { m_messages.append(msg); } else { - m_messages[index].addReferenceUniq(msg.fileName(), msg.lineNumber()); + TranslatorMessage &emsg = m_messages[index]; + emsg.addReferenceUniq(msg.fileName(), msg.lineNumber()); if (!msg.extraComment().isEmpty()) { - QString cmt = m_messages[index].extraComment(); + QString cmt = emsg.extraComment(); if (!cmt.isEmpty()) cmt.append(QLatin1String("\n----------\n")); cmt.append(msg.extraComment()); - m_messages[index].setExtraComment(cmt); + emsg.setExtraComment(cmt); } } } -- cgit v1.2.3 From 811eb47c3501479033ac3a484beeb896b6c0753f Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 25 Mar 2009 20:33:07 +0100 Subject: avoid empty codec name --- tools/linguist/shared/translator.cpp | 2 +- tools/linguist/shared/ts.cpp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp index 6ae8059aec..32404a5f7a 100644 --- a/tools/linguist/shared/translator.cpp +++ b/tools/linguist/shared/translator.cpp @@ -545,7 +545,7 @@ void Translator::setCodecName(const QByteArray &name) if (!codec) { if (!name.isEmpty()) qWarning("No QTextCodec for %s available. Using Latin1\n", name.constData()); - m_codecName.clear(); + m_codecName = "ISO-8859-1"; } else { m_codecName = codec->name(); } diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp index d454e4ebd1..3af9e593c6 100644 --- a/tools/linguist/shared/ts.cpp +++ b/tools/linguist/shared/ts.cpp @@ -277,7 +277,9 @@ bool TSReader::read(Translator &translator) // ignore these, just whitespace } else if (elementStarts(strdefaultcodec)) { // - translator.setCodecName(readElementText().toLatin1()); + const QString &codec = readElementText(); + if (!codec.isEmpty()) + translator.setCodecName(codec.toLatin1()); // } else if (isStartElement() && name().toString().startsWith(strextrans)) { -- cgit v1.2.3 From 39c52d840cd1ba3bf988937f4f9c9a9cdba57bfc Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 25 Mar 2009 21:04:48 +0100 Subject: actually ignore untranslated messages as it claims --- tools/linguist/shared/qm.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp index c197e2b1f2..1534e3fc01 100644 --- a/tools/linguist/shared/qm.cpp +++ b/tools/linguist/shared/qm.cpp @@ -646,10 +646,12 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData TranslatorMessage::Type typ = msg.type(); if (typ != TranslatorMessage::Obsolete) { if (typ == TranslatorMessage::Unfinished) { - if (msg.translation().isEmpty()) + if (msg.translation().isEmpty()) { ++untranslated; - else + continue; + } else { ++unfinished; + } } else { ++finished; } -- cgit v1.2.3 From 32798196d81439fa77b34425b95eb47556aebc8d Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 25 Mar 2009 21:05:25 +0100 Subject: micro-optimization: don't evaluate same condition twice --- tools/linguist/shared/qm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp index 1534e3fc01..4627b5d927 100644 --- a/tools/linguist/shared/qm.cpp +++ b/tools/linguist/shared/qm.cpp @@ -650,6 +650,8 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData ++untranslated; continue; } else { + if (cd.ignoreUnfinished()) + continue; ++unfinished; } } else { @@ -660,7 +662,6 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData QString comment = msg.comment(); QStringList translations = msg.translations(); - if (!cd.ignoreUnfinished() || typ != TranslatorMessage::Unfinished) { /* Drop the comment in (context, sourceText, comment), unless the context is empty, @@ -680,7 +681,6 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData //filename and lineNumbers will be ignored from now. releaser.insert(tm); } - } } } -- cgit v1.2.3 From d3be9fa7c981430c48de0a65938f3a2bd636bfcc Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Thu, 26 Mar 2009 23:13:30 +0100 Subject: properly deal with messages which appear in multiple encodings in ts 1.1 and qm files, messages appear in their native encoding. that means that a message can appear multiple times - once in utf8 and once in the codecForTr. however, in ts 2.0 files, everything is utf8 and messages can have a utf8 flag for the later transformation into qm. unfortunately, there was no flag to mark that the message is needed in *both* encodings, and the respective case was completely ignored when reading ts 1.1 and qm files (causing error messages). Task-number: 249022 AutoTest: 322690 --- tools/linguist/shared/qm.cpp | 291 +++++++++++++++------------- tools/linguist/shared/translator.cpp | 26 +++ tools/linguist/shared/translator.h | 1 + tools/linguist/shared/translatormessage.cpp | 4 +- tools/linguist/shared/translatormessage.h | 3 + tools/linguist/shared/ts.cpp | 194 +++++++++++-------- 6 files changed, 299 insertions(+), 220 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp index 4627b5d927..8a3658c7c0 100644 --- a/tools/linguist/shared/qm.cpp +++ b/tools/linguist/shared/qm.cpp @@ -103,6 +103,43 @@ static uint elfHash(const QByteArray &ba) return h; } +class ByteTranslatorMessage +{ +public: + ByteTranslatorMessage( + const QByteArray &context, + const QByteArray &sourceText, + const QByteArray &comment, + const QStringList &translations) : + m_context(context), + m_sourcetext(sourceText), + m_comment(comment), + m_translations(translations) + {} + const QByteArray &context() const { return m_context; } + const QByteArray &sourceText() const { return m_sourcetext; } + const QByteArray &comment() const { return m_comment; } + const QStringList &translations() const { return m_translations; } + bool operator<(const ByteTranslatorMessage& m) const; + +private: + QByteArray m_context; + QByteArray m_sourcetext; + QByteArray m_comment; + QStringList m_translations; +}; + +Q_DECLARE_TYPEINFO(ByteTranslatorMessage, Q_MOVABLE_TYPE); + +bool ByteTranslatorMessage::operator<(const ByteTranslatorMessage& m) const +{ + if (m_context != m.m_context) + return m_context < m.m_context; + if (m_sourcetext != m.m_sourcetext) + return m_sourcetext < m.m_sourcetext; + return m_comment < m.m_comment; +} + class Releaser { public: @@ -133,27 +170,12 @@ public: m_codec = QTextCodec::codecForName(codecName); } - TranslatorMessage findMessage(const QString &context, - const QString &sourceText, const QString &comment, - const QString &fileName = QString(), int lineNumber = -1) const; - bool save(QIODevice *iod); - void insert(const TranslatorMessage &); - void remove(const TranslatorMessage &); - - bool contains(const QString &context, const QString &sourceText, - const QString & comment) const; - - bool contains(const QString &context, const QString &comment, - const QString &fileName, int lineNumber) const; + void insert(const TranslatorMessage &msg, bool forceComment); void squeeze(TranslatorSaveMode mode); - QList messages() const; - - bool isEmpty() const; - void setNumerusRules(const QByteArray &rules); private: @@ -163,18 +185,20 @@ private: // on turn should be the same as passed to the actual tr(...) calls QByteArray originalBytes(const QString &str, bool isUtf8) const; - Prefix commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const; + void insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8); - uint msgHash(const TranslatorMessage &msg) const; + Prefix commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const; - void writeMessage(const TranslatorMessage & msg, QDataStream & stream, + uint msgHash(const ByteTranslatorMessage &msg) const; + + void writeMessage(const ByteTranslatorMessage & msg, QDataStream & stream, TranslatorSaveMode strip, Prefix prefix) const; // for squeezed but non-file data, this is what needs to be deleted QByteArray m_messageArray; QByteArray m_offsetArray; QByteArray m_contextArray; - QMap m_messages; + QMap m_messages; QByteArray m_numerusRules; // Used to reproduce the original bytes @@ -193,12 +217,12 @@ QByteArray Releaser::originalBytes(const QString &str, bool isUtf8) const return m_codec ? m_codec->fromUnicode(str) : str.toLatin1(); } -uint Releaser::msgHash(const TranslatorMessage &msg) const +uint Releaser::msgHash(const ByteTranslatorMessage &msg) const { - return elfHash(originalBytes(msg.sourceText() + msg.comment(), msg.isUtf8())); + return elfHash(msg.sourceText() + msg.comment()); } -Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const +Prefix Releaser::commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const { if (msgHash(m1) != msgHash(m2)) return NoPrefix; @@ -211,7 +235,7 @@ Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessa return HashContextSourceTextComment; } -void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream, +void Releaser::writeMessage(const ByteTranslatorMessage &msg, QDataStream &stream, TranslatorSaveMode mode, Prefix prefix) const { for (int i = 0; i < msg.translations().count(); ++i) { @@ -228,14 +252,14 @@ void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream, switch (prefix) { default: case HashContextSourceTextComment: - stream << quint8(Tag_Comment) << originalBytes(msg.comment(), msg.isUtf8()); + stream << quint8(Tag_Comment) << msg.comment(); // fall through case HashContextSourceText: - stream << quint8(Tag_SourceText) << originalBytes(msg.sourceText(), msg.isUtf8()); + stream << quint8(Tag_SourceText) << msg.sourceText(); // fall through case HashContext: - stream << quint8(Tag_Context) << originalBytes(msg.context(), msg.isUtf8()); - ; + stream << quint8(Tag_Context) << msg.context(); + break; } stream << quint8(Tag_End); @@ -275,7 +299,7 @@ void Releaser::squeeze(TranslatorSaveMode mode) if (m_messages.isEmpty() && mode == SaveEverything) return; - QMap messages = m_messages; + QMap messages = m_messages; // re-build contents m_messageArray.clear(); @@ -286,7 +310,7 @@ void Releaser::squeeze(TranslatorSaveMode mode) QMap offsets; QDataStream ms(&m_messageArray, QIODevice::WriteOnly); - QMap::const_iterator it, next; + QMap::const_iterator it, next; int cpPrev = 0, cpNext = 0; for (it = messages.constBegin(); it != messages.constEnd(); ++it) { cpPrev = cpNext; @@ -310,7 +334,7 @@ void Releaser::squeeze(TranslatorSaveMode mode) } if (mode == SaveStripped) { - QMap contextSet; + QMap contextSet; for (it = messages.constBegin(); it != messages.constEnd(); ++it) ++contextSet[it.key().context()]; @@ -322,10 +346,10 @@ void Releaser::squeeze(TranslatorSaveMode mode) else hTableSize = (contextSet.size() < 10000) ? 15013 : 3 * contextSet.size() / 2; - QMultiMap hashMap; - QMap::const_iterator c; + QMultiMap hashMap; + QMap::const_iterator c; for (c = contextSet.constBegin(); c != contextSet.constEnd(); ++c) - hashMap.insert(elfHash(originalBytes(c.key(), false /*FIXME*/)) % hTableSize, c.key()); + hashMap.insert(elfHash(c.key()) % hTableSize, c.key()); /* The contexts found in this translator are stored in a hash @@ -360,16 +384,14 @@ void Releaser::squeeze(TranslatorSaveMode mode) t << quint16(0); // the entry at offset 0 cannot be used uint upto = 2; - QMap::const_iterator entry = hashMap.constBegin(); + QMap::const_iterator entry = hashMap.constBegin(); while (entry != hashMap.constEnd()) { int i = entry.key(); hTable[i] = quint16(upto >> 1); do { - QString context = entry.value(); - QByteArray ba = context.toUtf8(); - const char *con = ba.data(); - uint len = uint(qstrlen(con)); + const char *con = entry.value().constData(); + uint len = uint(entry.value().length()); len = qMin(len, 255u); t << quint8(len); t.writeRawData(con, len); @@ -394,68 +416,28 @@ void Releaser::squeeze(TranslatorSaveMode mode) } } -bool Releaser::contains(const QString &context, const QString &sourceText, - const QString &comment) const -{ - return !findMessage(context, sourceText, comment).translation().isNull(); -} - -bool Releaser::contains(const QString &context, const QString &comment, - const QString &fileName, int lineNumber) const -{ - return !findMessage(context, QString(), comment, fileName, lineNumber).isNull(); -} - -void Releaser::insert(const TranslatorMessage &message) -{ - m_messages.insert(message, 0); -} - -void Releaser::remove(const TranslatorMessage &message) -{ - m_messages.remove(message); -} - - -TranslatorMessage Releaser::findMessage(const QString &context, - const QString &sourceText, const QString &comment, - const QString &fileName, int lineNumber) const +void Releaser::insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8) { - if (m_messages.isEmpty()) - return TranslatorMessage(); - - QMap::const_iterator it; - - // Either we want to find an item that matches context, sourcetext - // (and optionally comment) Or we want to find an item that - // matches context, filename, linenumber (and optionally comment) - TranslatorMessage msg(context, sourceText, comment, QString(), fileName, lineNumber); - it = m_messages.constFind(msg); - if (it != m_messages.constEnd()) - return it.key(); - - if (!comment.isEmpty()) { - it = m_messages.constFind(TranslatorMessage(context, sourceText, QString(), QString(), fileName, lineNumber)); - if (it != m_messages.constEnd()) - return it.key(); + ByteTranslatorMessage bmsg(originalBytes(message.context(), isUtf8), + originalBytes(message.sourceText(), isUtf8), + originalBytes(message.comment(), isUtf8), + message.translations()); + if (!forceComment) { + ByteTranslatorMessage bmsg2( + bmsg.context(), bmsg.sourceText(), QByteArray(""), bmsg.translations()); + if (!m_messages.contains(bmsg2)) { + m_messages.insert(bmsg2, 0); + return; + } } - - it = m_messages.constFind(TranslatorMessage(context, QString(), comment, QString(), fileName, lineNumber)); - if (it != m_messages.constEnd()) - return it.key(); - if (comment.isEmpty()) - return TranslatorMessage(); - - it = m_messages.constFind(TranslatorMessage(context, QString(), QString(), QString(), fileName, lineNumber)); - if (it != m_messages.constEnd()) - return it.key(); - return TranslatorMessage(); + m_messages.insert(bmsg, 0); } -bool Releaser::isEmpty() const +void Releaser::insert(const TranslatorMessage &message, bool forceComment) { - return m_messageArray.isEmpty() && m_offsetArray.isEmpty() - && m_contextArray.isEmpty() && m_messages.isEmpty(); + insertInternal(message, forceComment, message.isUtf8()); + if (message.isUtf8() && message.isNonUtf8()) + insertInternal(message, forceComment, false); } void Releaser::setNumerusRules(const QByteArray &rules) @@ -463,11 +445,6 @@ void Releaser::setNumerusRules(const QByteArray &rules) m_numerusRules = rules; } -QList Releaser::messages() const -{ - return m_messages.keys(); -} - static quint8 read8(const uchar *data) { return *data; @@ -478,6 +455,31 @@ static quint32 read32(const uchar *data) return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | (data[3]); } +static void fromBytes(const char *str, int len, QTextCodec *codec, QTextCodec *utf8Codec, + QString *out, QString *utf8Out, + bool *isSystem, bool *isUtf8, bool *needs8Bit) +{ + for (int i = 0; i < len; ++i) + if (str[i] & 0x80) { + if (utf8Codec) { + QTextCodec::ConverterState cvtState; + *utf8Out = utf8Codec->toUnicode(str, len, &cvtState); + *isUtf8 = !cvtState.invalidChars; + } + QTextCodec::ConverterState cvtState; + *out = codec->toUnicode(str, len, &cvtState); + *isSystem = !cvtState.invalidChars; + *needs8Bit = true; + return; + } + *out = QString::fromLatin1(str, len); + *isSystem = true; + if (utf8Codec) { + *utf8Out = *out; + *isUtf8 = true; + } + *needs8Bit = false; +} bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) { @@ -543,10 +545,19 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) size_t numItems = offsetLength / (2 * sizeof(quint32)); //qDebug() << "NUMITEMS: " << numItems; - TranslatorMessage msg; - // FIXME: that's just a guess, the original locale data is lost... QTextCodec *codec = QTextCodec::codecForLocale(); + QTextCodec *utf8Codec = 0; + if (codec->name() != "UTF-8") + utf8Codec = QTextCodec::codecForName("UTF-8"); + + QString context, contextUtf8; + bool contextIsSystem, contextIsUtf8, contextNeeds8Bit; + QString sourcetext, sourcetextUtf8; + bool sourcetextIsSystem, sourcetextIsUtf8, sourcetextNeeds8Bit; + QString comment, commentUtf8; + bool commentIsSystem, commentIsUtf8, commentNeeds8Bit; + QStringList translations; for (const uchar *start = offsetArray; start != offsetArray + (numItems << 3); start += 8) { //quint32 hash = read32(start); @@ -575,7 +586,7 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) } str.replace(QChar(Translator::InternalVariantSeparator), QChar(Translator::DefaultVariantSeparator)); - msg.appendTranslation(str); + translations << str; m += len; break; } @@ -588,7 +599,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) m += 4; //qDebug() << "SOURCE LEN: " << len; //qDebug() << "SOURCE: " << QByteArray((const char*)m, len); - msg.setSourceText(codec->toUnicode(QByteArray((const char*)m, len))); + fromBytes((const char*)m, len, codec, utf8Codec, + &sourcetext, &sourcetextUtf8, + &sourcetextIsSystem, &sourcetextIsUtf8, &sourcetextNeeds8Bit); m += len; break; } @@ -597,7 +610,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) m += 4; //qDebug() << "CONTEXT LEN: " << len; //qDebug() << "CONTEXT: " << QByteArray((const char*)m, len); - msg.setContext(codec->toUnicode(QByteArray((const char*)m, len))); + fromBytes((const char*)m, len, codec, utf8Codec, + &context, &contextUtf8, + &contextIsSystem, &contextIsUtf8, &contextNeeds8Bit); m += len; break; } @@ -606,7 +621,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) m += 4; //qDebug() << "COMMENT LEN: " << len; //qDebug() << "COMMENT: " << QByteArray((const char*)m, len); - msg.setComment(codec->toUnicode(QByteArray((const char*)m, len))); + fromBytes((const char*)m, len, codec, utf8Codec, + &comment, &commentUtf8, + &commentIsSystem, &commentIsUtf8, &commentNeeds8Bit); m += len; break; } @@ -616,11 +633,33 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) } } end:; + TranslatorMessage msg; msg.setType(TranslatorMessage::Finished); + msg.setTranslations(translations); + translations.clear(); + if (contextNeeds8Bit || sourcetextNeeds8Bit || commentNeeds8Bit) { + if (utf8Codec && contextIsUtf8 && sourcetextIsUtf8 && commentIsUtf8) { + // The message is utf-8, but file is not. + msg.setUtf8(true); + msg.setContext(contextUtf8); + msg.setSourceText(sourcetextUtf8); + msg.setComment(commentUtf8); + translator.append(msg); + continue; + } + if (!(contextIsSystem && sourcetextIsSystem && commentIsSystem)) { + cd.appendError(QLatin1String( + "Cannot read file with current system character codec")); + return false; + } + // The message is 8-bit in the file's encoding (utf-8 or not). + } + msg.setContext(context); + msg.setSourceText(sourcetext); + msg.setComment(comment); translator.append(msg); - //qDebug() << "\nHASH:" << hash << msg.sourceText() << msg.context(); - msg.setTranslations(QStringList()); } + translator.resolveDualEncoded(); return ok; } @@ -657,30 +696,16 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData } else { ++finished; } - QString context = msg.context(); - QString sourceText = msg.sourceText(); - QString comment = msg.comment(); - QStringList translations = msg.translations(); - - /* - Drop the comment in (context, sourceText, comment), - unless the context is empty, - unless (context, sourceText, "") already exists or - unless we already dropped the comment of (context, - sourceText, comment0). - */ - if (comment.isEmpty() - || context.isEmpty() - || translator.contains(context, sourceText, QString()) - || !releaser.findMessage(context, sourceText, QString()).translation() - .isNull() ) { - releaser.insert(msg); - } else { - TranslatorMessage tm(context, sourceText, QString(), - QString(), QString(), -1, translations); - //filename and lineNumbers will be ignored from now. - releaser.insert(tm); - } + // Drop the comment in (context, sourceText, comment), + // unless the context is empty, + // unless (context, sourceText, "") already exists or + // unless we already dropped the comment of (context, + // sourceText, comment0). + bool forceComment = + msg.comment().isEmpty() + || msg.context().isEmpty() + || translator.contains(msg.context(), msg.sourceText(), QString()); + releaser.insert(msg, forceComment); } } diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp index 32404a5f7a..3721204173 100644 --- a/tools/linguist/shared/translator.cpp +++ b/tools/linguist/shared/translator.cpp @@ -114,6 +114,10 @@ void Translator::extend(const TranslatorMessage &msg) cmt.append(msg.extraComment()); emsg.setExtraComment(cmt); } + if (msg.isUtf8() != emsg.isUtf8()) { + emsg.setUtf8(true); + emsg.setNonUtf8(true); + } } } @@ -425,6 +429,28 @@ QList Translator::findDuplicates() const return ret; } +void Translator::resolveDualEncoded() +{ + QHash dups; + for (int i = 0; i < m_messages.count();) { + const TranslatorMessage &msg = m_messages.at(i); + QHash::ConstIterator it = dups.constFind(msg); + if (it != dups.constEnd()) { + TranslatorMessage &omsg = m_messages[*it]; + if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) { + omsg.setUtf8(true); + omsg.setNonUtf8(true); + m_messages.removeAt(i); + continue; + } + // Regular dupe; will complain later + } else { + dups[msg] = i; + } + ++i; + } +} + // Used by lupdate to be able to search using absolute paths during merging void Translator::makeFileNamesAbsolute(const QDir &originalPath) { diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h index bbe765443e..ba719ec56c 100644 --- a/tools/linguist/shared/translator.h +++ b/tools/linguist/shared/translator.h @@ -130,6 +130,7 @@ public: void stripIdenticalSourceTranslations(); void dropTranslations(); QList findDuplicates() const; + void resolveDualEncoded(); void makeFileNamesAbsolute(const QDir &originalPath); void setCodecName(const QByteArray &name); diff --git a/tools/linguist/shared/translatormessage.cpp b/tools/linguist/shared/translatormessage.cpp index ab4301fbb3..afe66fe888 100644 --- a/tools/linguist/shared/translatormessage.cpp +++ b/tools/linguist/shared/translatormessage.cpp @@ -54,7 +54,7 @@ QT_BEGIN_NAMESPACE TranslatorMessage::TranslatorMessage() - : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_plural(false) + : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_nonUtf8(false), m_plural(false) { } @@ -66,7 +66,7 @@ TranslatorMessage::TranslatorMessage(const QString &context, : m_context(context), m_sourcetext(sourceText), m_comment(comment), m_userData(userData), m_translations(translations), m_fileName(fileName), m_lineNumber(lineNumber), - m_type(type), m_utf8(false), m_plural(plural) + m_type(type), m_utf8(false), m_nonUtf8(false), m_plural(plural) { } diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h index fa37ff5ce7..2818e28686 100644 --- a/tools/linguist/shared/translatormessage.h +++ b/tools/linguist/shared/translatormessage.h @@ -136,6 +136,8 @@ public: void setType(Type t) { m_type = t; } bool isUtf8() const { return m_utf8; } // codecForTr override void setUtf8(bool on) { m_utf8 = on; } + bool isNonUtf8() const { return m_nonUtf8; } // codecForTr override + void setNonUtf8(bool on) { m_nonUtf8 = on; } bool isPlural() const { return m_plural; } void setPlural(bool isplural) { m_plural = isplural; } @@ -169,6 +171,7 @@ private: Type m_type; bool m_utf8; + bool m_nonUtf8; bool m_plural; }; diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp index 3af9e593c6..8a6d365a8e 100644 --- a/tools/linguist/shared/ts.cpp +++ b/tools/linguist/shared/ts.cpp @@ -212,6 +212,7 @@ QString TSReader::readTransContents() bool TSReader::read(Translator &translator) { + STRING(both); STRING(byte); STRING(comment); STRING(context); @@ -265,13 +266,15 @@ bool TSReader::read(Translator &translator) QString currentFile; QXmlStreamAttributes atts = attributes(); - //QString version = atts.value(strversion).toString(); + QString version = atts.value(strversion).toString(); translator.setLanguageCode(atts.value(strlanguage).toString()); translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString()); while (!atEnd()) { readNext(); if (isEndElement()) { // found, finish local loop + if (version == QLatin1String("1.1")) + translator.resolveDualEncoded(); break; } else if (isWhiteSpace()) { // ignore these, just whitespace @@ -311,7 +314,9 @@ bool TSReader::read(Translator &translator) msg.setContext(context); msg.setType(TranslatorMessage::Finished); msg.setPlural(attributes().value(strnumerus) == stryes); - msg.setUtf8(attributes().value(strutf8) == strtrue + const QStringRef &utf8Attr = attributes().value(strutf8); + msg.setNonUtf8(utf8Attr == strboth); + msg.setUtf8(msg.isNonUtf8() || utf8Attr == strtrue || attributes().value(strencoding) == strUtf8); while (!atEnd()) { readNext(); @@ -594,107 +599,126 @@ bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd, in foreach (const TranslatorMessage &msg, messageOrder[context]) { //msg.dump(); - t << " \n"; - if (translator.locationsType() != Translator::NoLocations) { - QString cfile = currentFile; - bool first = true; - foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) { - QString fn = cd.m_targetDir.relativeFilePath(ref.fileName()) - .replace(QLatin1Char('\\'),QLatin1Char('/')); - int ln = ref.lineNumber(); - QString ld; - if (translator.locationsType() == Translator::RelativeLocations) { - if (ln != -1) { - int dlt = ln - currentLine[fn]; - if (dlt >= 0) - ld.append(QLatin1Char('+')); - ld.append(QString::number(dlt)); - currentLine[fn] = ln; + bool isUtf8 = msg.isUtf8(); + bool second = false; + forever { + + t << " \n"; + if (translator.locationsType() != Translator::NoLocations) { + QString cfile = currentFile; + bool first = true; + foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) { + QString fn = cd.m_targetDir.relativeFilePath(ref.fileName()) + .replace(QLatin1Char('\\'),QLatin1Char('/')); + int ln = ref.lineNumber(); + QString ld; + if (translator.locationsType() == Translator::RelativeLocations) { + if (ln != -1) { + int dlt = ln - currentLine[fn]; + if (dlt >= 0) + ld.append(QLatin1Char('+')); + ld.append(QString::number(dlt)); + currentLine[fn] = ln; + } - if (fn != cfile) { - if (first) - currentFile = fn; - cfile = fn; + if (fn != cfile) { + if (first) + currentFile = fn; + cfile = fn; + } else { + fn.clear(); + } + first = false; } else { - fn.clear(); + if (ln != -1) + ld = QString::number(ln); } - first = false; - } else { - if (ln != -1) - ld = QString::number(ln); + t << " \n"; } - t << " \n"; } - } - t << " " - << evilBytes(msg.sourceText(), msg.isUtf8(), format, codecName) - << "\n"; + t << " " + << evilBytes(msg.sourceText(), isUtf8, format, codecName) + << "\n"; - if (format != 11 && !msg.oldSourceText().isEmpty()) - t << " " << protect(msg.oldSourceText()) << "\n"; + if (format != 11 && !msg.oldSourceText().isEmpty()) + t << " " << protect(msg.oldSourceText()) << "\n"; - if (!msg.comment().isEmpty()) { - t << " " - << evilBytes(msg.comment(), msg.isUtf8(), format, codecName) - << "\n"; - } + if (!msg.comment().isEmpty()) { + t << " " + << evilBytes(msg.comment(), isUtf8, format, codecName) + << "\n"; + } - if (format != 11) { + if (format != 11) { - if (!msg.oldComment().isEmpty()) - t << " " << protect(msg.oldComment()) << "\n"; + if (!msg.oldComment().isEmpty()) + t << " " << protect(msg.oldComment()) << "\n"; - if (!msg.extraComment().isEmpty()) - t << " " << protect(msg.extraComment()) - << "\n"; + if (!msg.extraComment().isEmpty()) + t << " " << protect(msg.extraComment()) + << "\n"; - if (!msg.translatorComment().isEmpty()) - t << " " << protect(msg.translatorComment()) - << "\n"; + if (!msg.translatorComment().isEmpty()) + t << " " << protect(msg.translatorComment()) + << "\n"; - } + } - t << " "; - QStringList translns = translator.normalizedTranslations(msg, cd, &result); - for (int j = 0; j < qMax(1, translns.count()); ++j) { - t << "\n "; + t << " "; + QStringList translns = translator.normalizedTranslations(msg, cd, &result); + for (int j = 0; j < qMax(1, translns.count()); ++j) { + t << "\n "; + } + t << "\n "; + } else { + writeVariants(t, " ", msg.translation()); } - t << "\n "; - } else { - writeVariants(t, " ", msg.translation()); - } - t << "\n"; + t << "\n"; + + if (format != 11) + writeExtras(t, " ", msg.extras(), drops); - if (format != 11) - writeExtras(t, " ", msg.extras(), drops); + if (!msg.userData().isEmpty()) + t << " " << msg.userData() << "\n"; + t << " \n"; - if (!msg.userData().isEmpty()) - t << " " << msg.userData() << "\n"; - t << " \n"; + if (format != 11 || second || !msg.isUtf8() || !msg.isNonUtf8()) + break; + isUtf8 = false; + second = true; + } } t << "\n"; } -- cgit v1.2.3 From f3ec88cb128ee912457e273c420839d4bd295572 Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Thu, 26 Mar 2009 19:02:25 +0100 Subject: declare type info next to the type, not "somewhere" --- tools/linguist/shared/translator.h | 2 -- tools/linguist/shared/translatormessage.h | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h index ba719ec56c..8908305780 100644 --- a/tools/linguist/shared/translator.h +++ b/tools/linguist/shared/translator.h @@ -52,8 +52,6 @@ QT_BEGIN_NAMESPACE -Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE); - class QIODevice; // A struct of "interesting" data passed to and from the load and save routines diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h index 2818e28686..363019ede5 100644 --- a/tools/linguist/shared/translatormessage.h +++ b/tools/linguist/shared/translatormessage.h @@ -175,6 +175,8 @@ private: bool m_plural; }; +Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE); + int qHash(const TranslatorMessage &msg); QT_END_NAMESPACE -- cgit v1.2.3 From 65aadc316caca9c31a9aaf4011e4e02b3f8af19f Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Thu, 26 Mar 2009 19:58:57 +0100 Subject: static is even better than const --- tools/linguist/shared/qm.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp index 8a3658c7c0..d19b5198c1 100644 --- a/tools/linguist/shared/qm.cpp +++ b/tools/linguist/shared/qm.cpp @@ -187,9 +187,9 @@ private: void insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8); - Prefix commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const; + static Prefix commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2); - uint msgHash(const ByteTranslatorMessage &msg) const; + static uint msgHash(const ByteTranslatorMessage &msg); void writeMessage(const ByteTranslatorMessage & msg, QDataStream & stream, TranslatorSaveMode strip, Prefix prefix) const; @@ -217,12 +217,12 @@ QByteArray Releaser::originalBytes(const QString &str, bool isUtf8) const return m_codec ? m_codec->fromUnicode(str) : str.toLatin1(); } -uint Releaser::msgHash(const ByteTranslatorMessage &msg) const +uint Releaser::msgHash(const ByteTranslatorMessage &msg) { return elfHash(msg.sourceText() + msg.comment()); } -Prefix Releaser::commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const +Prefix Releaser::commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) { if (msgHash(m1) != msgHash(m2)) return NoPrefix; -- cgit v1.2.3 From b3f88e8629390a49d12fd7684e711280b7b2c7c2 Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Fri, 27 Mar 2009 13:08:00 +0100 Subject: properly process backslash line continuations this is a backport of be5a9587865f9f042cac8feb5296b71b11a1a80f Task-number: 249633 --- tools/linguist/shared/cpp.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'tools/linguist/shared') diff --git a/tools/linguist/shared/cpp.cpp b/tools/linguist/shared/cpp.cpp index 333ffbf709..2e137cfa6f 100644 --- a/tools/linguist/shared/cpp.cpp +++ b/tools/linguist/shared/cpp.cpp @@ -130,12 +130,19 @@ static int yyInPos; static uint getChar() { - if (yyInPos >= yyInStr.size()) - return EOF; - QChar c = yyInStr[yyInPos++]; - if (c.unicode() == '\n') - ++yyCurLineNo; - return c.unicode(); + forever { + if (yyInPos >= yyInStr.size()) + return EOF; + uint c = yyInStr[yyInPos++].unicode(); + if (c == '\\' && yyInPos < yyInStr.size() && yyInStr[yyInPos].unicode() == '\n') { + ++yyCurLineNo; + ++yyInPos; + continue; + } + if (c == '\n') + ++yyCurLineNo; + return c; + } } static uint getToken() -- cgit v1.2.3