lupdate/Python: Handle f/r strings

Fixes: PYSIDE-2380
Change-Id: I44ed60039af4df82d4f06bfae9e229847ad1fad6
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>
(cherry picked from commit 5577432ba554e4835ceaa76f6bd97d76f6b7e8aa)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

3 files changed, 68 insertions, 9 deletions

diff --git a/src/linguist/lupdate/python.cpp b/src/linguist/lupdate/python.cpp
index 1cadadc59..0bc3bf5e8 100644
--- a/src/linguist/lupdate/python.cpp
+++ b/src/linguist/lupdate/python.cpp
@@ -30,6 +30,14 @@ enum Token { Tok_Eof, Tok_class, Tok_def, Tok_return, Tok_tr,
              Tok_LeftParen, Tok_RightParen,
              Tok_Comma, Tok_None, Tok_Integer};
 
+enum class StringType
+{
+    NoString,
+    String,
+    FormatString,
+    RawString
+};
+
 /*
   The tokenizer maintains the following global variables. The names
   should be self-explanatory.
@@ -125,7 +133,7 @@ static void startTokenizer(const QString &fileName, int (*getCharFunc)(),
     yyContextStack.clear();
 }
 
-static bool parseStringEscape()
+static bool parseStringEscape(int quoteChar, StringType stringType)
 {
     static const char tab[] = "abfnrtv";
     static const char backTab[] = "\a\b\f\n\r\t\v";
@@ -134,6 +142,14 @@ static bool parseStringEscape()
     if (yyCh == EOF)
         return false;
 
+    if (stringType == StringType::RawString) {
+        if (yyCh != quoteChar) // Only quotes can be escaped in raw strings
+            yyString[yyStringLen++] = '\\';
+        yyString[yyStringLen++] = yyCh;
+        yyCh = getChar();
+        return true;
+    }
+
     if (yyCh == 'x') {
         QByteArray hex = "0";
         yyCh = getChar();
@@ -185,7 +201,7 @@ static bool parseStringEscape()
     return true;
 }
 
-static Token parseString()
+static Token parseString(StringType stringType = StringType::NoString)
 {
     int quoteChar = yyCh;
     bool tripleQuote = false;
@@ -226,7 +242,7 @@ static Token parseString()
         }
 
         if (yyCh == '\\') {
-            if (!parseStringEscape())
+            if (!parseStringEscape(quoteChar, stringType))
                 return Tok_Eof;
         } else {
             char *yStart = yyString + yyStringLen;
@@ -266,7 +282,7 @@ static QByteArray readLine()
     return result;
 }
 
-static Token getToken()
+static Token getToken(StringType stringType = StringType::NoString)
 {
     yyIdent.clear();
     yyCommentLen = 0;
@@ -304,7 +320,7 @@ static Token getToken()
             break;
         case '"':
         case '\'':
-            return parseString();
+            return parseString(stringType);
         case '(':
             yyParenDepth++;
             yyCh = getChar();
@@ -376,15 +392,34 @@ static bool match(Token t)
     return matches;
 }
 
+static bool matchStringStart()
+{
+    if (yyTok == Tok_String)
+        return true;
+    // Check for f"bla{var}" and raw strings r"bla".
+    if (yyTok == Tok_Ident && yyIdent.size() == 1) {
+        switch (yyIdent.at(0)) {
+        case 'r':
+            yyTok = getToken(StringType::RawString);
+            return yyTok == Tok_String;
+        case 'f':
+            yyTok = getToken(StringType::FormatString);
+            return yyTok == Tok_String;
+        }
+    }
+    return false;
+}
+
 static bool matchString(QByteArray *s)
 {
-    const bool matches = (yyTok == Tok_String);
     s->clear();
-    while (yyTok == Tok_String) {
+    bool ok = false;
+    while (matchStringStart()) {
         *s += yyString;
         yyTok = getToken();
+        ok = true;
     }
-    return matches;
+    return ok;
 }
 
 static bool matchEncoding(bool *utf8)
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsepython/main.py b/tests/auto/linguist/lupdate/testdata/good/parsepython/main.py
index e2bf9ac93..5c0c77851 100644
--- a/tests/auto/linguist/lupdate/testdata/good/parsepython/main.py
+++ b/tests/auto/linguist/lupdate/testdata/good/parsepython/main.py
@@ -72,6 +72,10 @@ class Window(QMainWindow):
 
     def window_method(self):  # PYSIDE-2379, Don't put this into NestedClass
         msg = self.tr("Window Message")
+        msg = self.tr(f"An f-string\\")
+        msg = self.tr(r"A raw strin\g")
+        msg = self.tr(r"A raw strin\g""continued\\")
+        msg = self.tr(r"A raw string with escaped quote\"bla")
 
 
 if __name__ == '__main__':
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsepython/project.ts.result b/tests/auto/linguist/lupdate/testdata/good/parsepython/project.ts.result
index 658485339..df1788672 100644
--- a/tests/auto/linguist/lupdate/testdata/good/parsepython/project.ts.result
+++ b/tests/auto/linguist/lupdate/testdata/good/parsepython/project.ts.result
@@ -81,9 +81,29 @@
         </translation>
     </message>
     <message>
-        <location filename="main.py" line="77"/>
+        <location filename="main.py" line="75"/>
         <source>Window Message</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <location filename="main.py" line="76"/>
+        <source>An f-string\</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.py" line="77"/>
+        <source>A raw strin\g</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.py" line="78"/>
+        <source>A raw strin\gcontinued\</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.py" line="81"/>
+        <source>A raw string with escaped quote&quot;bla</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 </TS>