Fix retrieval performance regression.2009W12

Retrieval performance decreased with respect to the number of files
in the content directory, due to the use of QDir::entryList to find
files related to the old content identifier when a message was
updated.  Part files are now stored in a per-message subdirectory
of the content location.

The Qtopiamailfile content manager performs an automatic migration
from the old format to the new one - to support this, the
QMailContentManager and QMailContentManagerFactory classes have
an init() function which is called by the mail store after it
completes its own initialisation.

7 files changed, 200 insertions, 26 deletions

diff --git a/src/libraries/qtopiamail/CHANGES b/src/libraries/qtopiamail/CHANGES
index d0c867ce..1a071729 100644
--- a/src/libraries/qtopiamail/CHANGES
+++ b/src/libraries/qtopiamail/CHANGES
@@ -4,6 +4,9 @@
 
 API changes since the development preview release on 06/03/09:
 
+1. Added 'QMailContentManager::init()' and 
+         'QMailContentManagerFactory::init()'.
+
 /***************************************************************************/
 
 API changes since the development preview release on 16/01/09:
diff --git a/src/libraries/qtopiamail/qmailcontentmanager.cpp b/src/libraries/qtopiamail/qmailcontentmanager.cpp
index ea554242..acfe2544 100644
--- a/src/libraries/qtopiamail/qmailcontentmanager.cpp
+++ b/src/libraries/qtopiamail/qmailcontentmanager.cpp
@@ -101,6 +101,19 @@ QMailContentManager *QMailContentManagerFactory::create(const QString &scheme)
 }
 
 /*!
+    Performs any initialization tasks for content managers known to the factory.
+    Returns false if any content managers are unable to perform initialiation tasks.
+*/
+bool QMailContentManagerFactory::init()
+{
+    foreach (QMailContentManager *manager, pluginMap().values())
+        if (!manager->init())
+            return false;
+
+    return true;
+}
+
+/*!
     Clears the content managed by all content managers known to the factory.
 */
 void QMailContentManagerFactory::clearContent()
@@ -251,6 +264,17 @@ QMailContentManager::~QMailContentManager()
 }
 
 /*!
+    Directs the content manager to perform any initialization tasks required.
+    The content manager should return false if unable to perform initialization tasks; otherwise return true.
+
+    This function is called by the mail store after it has been successfully initialized.
+*/
+bool QMailContentManager::init()
+{
+    return true;
+}
+
+/*!
     Directs the content manager to clear any message content that it is responsible for.
 
     This function is called by the mail store to remove all existing data, typically in test conditions.
diff --git a/src/libraries/qtopiamail/qmailcontentmanager.h b/src/libraries/qtopiamail/qmailcontentmanager.h
index f2b0b64d..5eebf84a 100644
--- a/src/libraries/qtopiamail/qmailcontentmanager.h
+++ b/src/libraries/qtopiamail/qmailcontentmanager.h
@@ -29,6 +29,7 @@ public:
 
     static QMailContentManager *create(const QString &scheme);
 
+    static bool init();
     static void clearContent();
 };
 
@@ -71,6 +72,7 @@ public:
     virtual QMailStore::ErrorCode remove(const QString &identifier) = 0;
     virtual QMailStore::ErrorCode load(const QString &identifier, QMailMessage *message) = 0;
 
+    virtual bool init();
     virtual void clearContent();
 };
 
diff --git a/src/libraries/qtopiamail/qmailstore_p.cpp b/src/libraries/qtopiamail/qmailstore_p.cpp
index a81e3d5c..61d33bb0 100644
--- a/src/libraries/qtopiamail/qmailstore_p.cpp
+++ b/src/libraries/qtopiamail/qmailstore_p.cpp
@@ -2010,6 +2010,11 @@ bool QMailStorePrivate::initStore()
     query.exec(QLatin1String("PRAGMA cache_size=50"));
 #endif
 
+    if (!QMailContentManagerFactory::init()) {
+        qMailLog(Messaging) << "Could not initialize content manager factory";
+        return false;
+    }
+
     // We are now correctly initialized
     init = true;
     return true;
diff --git a/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.cpp b/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.cpp
index 0dc162d5..770c5f74 100644
--- a/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.cpp
+++ b/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.cpp
@@ -103,6 +103,52 @@ bool pathOnDefault(const QString &path)
     return path.startsWith(defaultPath());
 }
 
+bool migrateAccountToVersion101(const QMailAccountId &accountId)
+{
+    foreach (const QMailMessageId &id, QMailStore::instance()->queryMessages(QMailMessageKey::parentAccountId(accountId))) {
+        const QMailMessage message(id);
+
+        if (message.multipartType() != QMailMessage::MultipartNone) {
+            // Any part files for this message need to be moved to the new location
+            QString fileName(message.contentIdentifier());
+
+            QFileInfo fi(fileName);
+            QDir path(fi.dir());
+            path.setNameFilters(QStringList() << (fi.fileName() + "-[0-9]*"));
+
+            QStringList entries(path.entryList());
+            if (!entries.isEmpty()) {
+                // Does the part directory exist?
+                QString partDirectory(QtopiamailfileManager::messagePartDirectory(fileName));
+                if (!QDir(partDirectory).exists()) {
+                    if (!QDir::root().mkpath(partDirectory)) {
+                        qMailLog(Messaging) << "Unable to create directory for message part content:" << partDirectory;
+                        return false;
+                    }
+                }
+
+                foreach (const QString &entry, entries) {
+                    QFile existing(path.filePath(entry));
+
+                    // We need to move this file to the subdirectory
+                    QString newName(partDirectory + '/');
+                    int index = entry.lastIndexOf('-');
+                    if (index != -1) {
+                        newName.append(entry.mid(index + 1));
+                    }
+
+                    if (!existing.rename(newName)) {
+                        qMailLog(Messaging) << "Unable to move part file to new name:" << newName;
+                        return false;
+                    }
+                }
+            }
+        }
+    }
+
+    return true;
+}
+
 }
 
 
@@ -175,7 +221,7 @@ QMailStore::ErrorCode QtopiamailfileManager::addOrRename(QMailMessage *message,
         }
 
         // Try to remove any parts that were created
-        removeParts(*message, message->contentIdentifier());
+        removeParts(message->contentIdentifier());
 
         return QMailStore::FrameworkFault;
     }
@@ -211,14 +257,14 @@ QMailStore::ErrorCode QtopiamailfileManager::remove(const QString &identifier)
 
     QFileInfo fi(identifier);
     QDir path(fi.dir());
-    path.setNameFilters(QStringList() << (fi.fileName() + '*'));
+    if (!path.remove(fi.fileName())) {
+        qMailLog(Messaging) << "Unable to remove content file:" << identifier;
+        result = QMailStore::FrameworkFault;
+    }
 
-    foreach (const QString &entry, path.entryList()) {
-        QString filePath(path.filePath(entry));
-        if (!QFile::remove(filePath)) {
-            qMailLog() << "Unable to remove content file:" << filePath;
-            result = QMailStore::FrameworkFault;
-        }
+    if (!removeParts(identifier)) {
+        qMailLog(Messaging) << "Unable to remove part content files for:" << identifier;
+        result = QMailStore::FrameworkFault;
     }
 
     return result;
@@ -249,6 +295,79 @@ QMailStore::ErrorCode QtopiamailfileManager::load(const QString &identifier, QMa
     return QMailStore::NoError;
 }
 
+bool QtopiamailfileManager::init()
+{
+    // It used to be possible for accounts to not have a storage service configured.
+    // If so, add a configuration to those accounts for qtopiamailfile
+    foreach (const QMailAccountId &accountId, QMailStore::instance()->queryAccounts()) {
+        QMailAccountConfiguration config(accountId);
+
+        if (!config.services().contains(gKey)) {
+            bool storageConfigured(false);
+
+            // See if any of these services are for storage
+            foreach (const QString &service, config.services()) {
+                QMailAccountConfiguration::ServiceConfiguration &svcCfg(config.serviceConfiguration(service));
+
+                if (svcCfg.value("servicetype") == "storage") {
+                    storageConfigured = true;
+                    break;
+                }
+            }
+
+            if (!storageConfigured) {
+                // Add a configuration for our default service
+                config.addServiceConfiguration(gKey);
+
+                QMailAccountConfiguration::ServiceConfiguration &svcCfg(config.serviceConfiguration(gKey));
+                svcCfg.setValue("version", "100");
+                svcCfg.setValue("servicetype", "storage");
+
+                if (QMailStore::instance()->updateAccountConfiguration(&config)) {
+                    qMailLog(Messaging) << "Added storage configuration for account" << accountId;
+                } else {
+                    qWarning() << "Unable to add missing storage configuration for account:" << accountId;
+                    return false;
+                }
+            }
+        }
+    }
+
+    // Migrate any data in older formats
+    foreach (const QMailAccountId &accountId, QMailStore::instance()->queryAccounts()) {
+        QMailAccountConfiguration config(accountId);
+
+        if (config.services().contains(gKey)) {
+            // This account uses our content manager
+            QMailAccountConfiguration::ServiceConfiguration &svcCfg = config.serviceConfiguration(gKey);
+            int version = svcCfg.value("version").toInt();
+
+            if (version == 100) {
+                // Version 100 - part files are not in subdirectories
+                if (!migrateAccountToVersion101(accountId)) {
+                    qWarning() << "Unable to migrate account data to version 101 for account:" << accountId;
+                    return false;
+                }
+
+                version = 101;
+            }
+
+            if (svcCfg.value("version").toInt() != version) {
+                svcCfg.setValue("version", QString::number(version));
+
+                if (QMailStore::instance()->updateAccountConfiguration(&config)) {
+                    qMailLog(Messaging) << "Migrated content data for account" << accountId << "to version" << version;
+                } else {
+                    qWarning() << "Unable to update account configuration for account:" << accountId;
+                    return false;
+                }
+            }
+        }
+    }
+
+    return true;
+}
+
 void QtopiamailfileManager::clearContent()
 {
     // Delete all content files
@@ -289,11 +408,25 @@ QString QtopiamailfileManager::messageFilePath(const QString &fileName, const QM
 
 QString QtopiamailfileManager::messagePartFilePath(const QMailMessagePart &part, const QString &fileName)
 {
-    return fileName + '-' + part.location().toString(false); 
+    return messagePartDirectory(fileName) + '/' + part.location().toString(false); 
+}
+
+QString QtopiamailfileManager::messagePartDirectory(const QString &fileName)
+{
+    return fileName + "-parts";
 }
 
 bool QtopiamailfileManager::addOrRenameParts(QMailMessage *message, const QMailMessagePartContainer &container, const QString &fileName, const QString &existing)
 {
+    // Ensure that the part directory exists
+    QString partDirectory(messagePartDirectory(fileName));
+    if (!QDir(partDirectory).exists()) {
+        if (!QDir::root().mkpath(partDirectory)) {
+            qMailLog(Messaging) << "Unable to create directory for message part content:" << partDirectory;
+            return false;
+        }
+    }
+
     for (uint i = 0; i < container.partCount(); ++i) {
         const QMailMessagePart &part(container.partAt(i));
         QString loc = part.location().toString(false);
@@ -407,26 +540,31 @@ bool QtopiamailfileManager::loadParts(QMailMessage *message, QMailMessagePartCon
     return true;
 }
 
-void QtopiamailfileManager::removeParts(const QMailMessagePartContainer &container, const QString &fileName)
+bool QtopiamailfileManager::removeParts(const QString &fileName)
 {
-    for (uint i = 0; i < container.partCount(); ++i) {
-        const QMailMessagePart &part(container.partAt(i));
-
-        if (part.multipartType() == QMailMessagePartContainer::MultipartNone) {
-            if (part.hasBody()) {
-                QString partFilePath(messagePartFilePath(part, fileName));
-
-                if (QFile::exists(partFilePath)) {
-                    if (!QFile::remove(partFilePath)){
-                        qMailLog(Messaging) << "Unable to remove message part content file:" << partFilePath;
-                    }
+    bool result(true);
+
+    QString partDirectory(messagePartDirectory(fileName));
+
+    QDir dir(partDirectory);
+    if (dir.exists()) {
+        // Remove any files in this directory
+        foreach (const QString &entry, dir.entryList()) {
+            if ((entry != QLatin1String(".")) && (entry != QLatin1String(".."))) {
+                if (!dir.remove(entry)) {
+                    qMailLog(Messaging) << "Unable to remove part file:" << entry;
+                    result = false;
                 }
             }
-        } else {
-            // Remove any sub-parts of this part
-            removeParts(part, fileName);
+        }
+
+        if (!QDir::root().rmdir(dir.absolutePath())) {
+            qMailLog(Messaging) << "Unable to remove directory for message part content:" << partDirectory;
+            result = false;
         }
     }
+
+    return result;
 }
 
 Q_EXPORT_PLUGIN2(qtopiamailfilemanager,QtopiamailfileManagerPlugin)
diff --git a/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.h b/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.h
index 0287ddd0..c08f5895 100644
--- a/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.h
+++ b/src/plugins/contentmanagers/qtopiamailfile/qtopiamailfilemanager.h
@@ -31,10 +31,12 @@ public:
     QMailStore::ErrorCode remove(const QString &identifier);
     QMailStore::ErrorCode load(const QString &identifier, QMailMessage *message);
 
+    bool init();
     void clearContent();
 
     static const QString &messagesBodyPath(const QMailAccountId &accountId);
     static QString messageFilePath(const QString &fileName, const QMailAccountId &accountId);
+    static QString messagePartDirectory(const QString &fileName);
     static QString messagePartFilePath(const QMailMessagePart &part, const QString &fileName);
 
 protected slots:
@@ -45,7 +47,7 @@ private:
 
     bool addOrRenameParts(QMailMessage *message, const QMailMessagePartContainer &container, const QString &fileName, const QString &existing);
     bool loadParts(QMailMessage *message, QMailMessagePartContainer *container, const QString &fileName);
-    void removeParts(const QMailMessagePartContainer &container, const QString &fileName);
+    bool removeParts(const QString &fileName);
 };
 
 
diff --git a/src/plugins/messageservices/qtopiamailfile/settings.cpp b/src/plugins/messageservices/qtopiamailfile/settings.cpp
index 4ff75a62..ed25181e 100644
--- a/src/plugins/messageservices/qtopiamailfile/settings.cpp
+++ b/src/plugins/messageservices/qtopiamailfile/settings.cpp
@@ -62,7 +62,7 @@ bool QtopiamailfileSettings::updateAccount(QMailAccount *, QMailAccountConfigura
         config->addServiceConfiguration(serviceKey);
 
     QMailAccountConfiguration::ServiceConfiguration &svcCfg(config->serviceConfiguration(serviceKey));
-    svcCfg.setValue("version", "100");
+    svcCfg.setValue("version", "101");
     svcCfg.setValue("servicetype", "storage");
 
     svcCfg.setValue("basePath", locations[locationSelector->currentIndex()].second);