OSM Geoservice plugin improvements, including offline data support

This patch contains improvements for the OSM geoservice plugin, in form
of property-controllable disk cache, and the support for a directory
containing an offline dataset.

The offline directory can contain subdirectories also in form of
symlinks. If duplicate tiles are present, the newest is used.

The population of the offline tiles registry is offloaded to a
thread to prevent blocking the startup of the application.
No synchronization is used (except on cache destruction),
so that, as the tiles are being indexed, they can be used in
the application.

For the time being, no support is given to populate such directory
(mostly due to current API limitations), but tiles can be placed
manually, for example copying them from the disk cache directory.

Task-number: QTBUG-45284
Change-Id: I518980669a3ee474545025adf05adc69cdd29781
Reviewed-by: Paolo Angelelli <paolo.angelelli@theqtcompany.com>
Reviewed-by: Alex Blasche <alexander.blasche@theqtcompany.com>

8 files changed, 315 insertions, 38 deletions

diff --git a/src/location/doc/src/plugins/osm.qdoc b/src/location/doc/src/plugins/osm.qdoc
index 1c924fd3..9f4979b5 100644
--- a/src/location/doc/src/plugins/osm.qdoc
+++ b/src/location/doc/src/plugins/osm.qdoc
@@ -86,6 +86,29 @@ a prefix.
         If not specified the default  \l {http://nominatim.openstreetmap.org/search}{url}
         will be used.
         \note The API documentation is available at \l {https://wiki.openstreetmap.org/wiki/Nominatim}{Project OSM Nominatim}.
+\row
+    \li osm.mapping.cache.directory
+    \li Absolute path to map tile cache directory used as network disk cache.
+
+    The default place for the cache is \c{QtLocation/osm} directory in \l {QStandardPaths::writableLocation()} {QStandardPaths::writableLocation}(\l{QStandardPaths::GenericCacheLocation}).
+    On systems that have no concept of a shared cache, the application-specific \l{QStandardPaths::CacheLocation} is used instead.
+\row
+    \li osm.mapping.offline.directory
+    \li Absolute path to a directory containing map tiles used as an offline storage. If specified, it will work together with the network disk cache, but tiles won't get automatically
+    inserted, removed or updated. The format of the tiles is the same used by the network disk cache.
+
+    There is no default value, and if this property is not set, no directory will be indexed and only the network disk cache will be used
+    to reduce network usage or to act as an offline storage for the currently cached tiles.
+\row
+    \li osm.mapping.cache.disk.size
+    \li Map tile disk cache size in bytes. Default size of the cache is 100MB.
+\row
+    \li osm.mapping.cache.memory.size
+    \li Map tile memory cache size in bytes. Default size of the cache is 3MB.
+\row
+    \li osm.mapping.cache.texture.size
+    \li Map tile texture cache size in bytes. Default size of the cache is 6MB. Note that the texture cache has a hard minimum size which depends on the size of the map viewport (it must contain enough data to display the tiles currently visible on the display). This value is the amount of cache to be used in addition to the bare minimum.
+
 \endtable
 
 \section1 Parameter Usage Example
diff --git a/src/location/maps/qgeofiletilecache.cpp b/src/location/maps/qgeofiletilecache.cpp
index 520649ea..fd6bbc38 100644
--- a/src/location/maps/qgeofiletilecache.cpp
+++ b/src/location/maps/qgeofiletilecache.cpp
@@ -280,43 +280,10 @@ void QGeoFileTileCache::clearAll()
 
 QSharedPointer<QGeoTileTexture> QGeoFileTileCache::get(const QGeoTileSpec &spec)
 {
-    QSharedPointer<QGeoTileTexture> tt = textureCache_.object(spec);
+    QSharedPointer<QGeoTileTexture> tt = getFromMemory(spec);
     if (tt)
         return tt;
-
-    QSharedPointer<QGeoCachedTileMemory> tm = memoryCache_.object(spec);
-    if (tm) {
-        QImage image;
-        if (!image.loadFromData(tm->bytes)) {
-            handleError(spec, QLatin1String("Problem with tile image"));
-            return QSharedPointer<QGeoTileTexture>(0);
-        }
-        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(spec, image);
-        if (tt)
-            return tt;
-    }
-
-    QSharedPointer<QGeoCachedTileDisk> td = diskCache_.object(spec);
-    if (td) {
-        const QString format = QFileInfo(td->filename).suffix();
-        QFile file(td->filename);
-        file.open(QIODevice::ReadOnly);
-        QByteArray bytes = file.readAll();
-        file.close();
-
-        QImage image;
-        if (!image.loadFromData(bytes)) {
-            handleError(spec, QLatin1String("Problem with tile image"));
-            return QSharedPointer<QGeoTileTexture>(0);
-        }
-
-        addToMemoryCache(spec, bytes, format);
-        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(td->spec, image);
-        if (tt)
-            return tt;
-    }
-
-    return QSharedPointer<QGeoTileTexture>();
+    return getFromDisk(spec);
 }
 
 void QGeoFileTileCache::insert(const QGeoTileSpec &spec,
@@ -394,6 +361,51 @@ QSharedPointer<QGeoTileTexture> QGeoFileTileCache::addToTextureCache(const QGeoT
     return tt;
 }
 
+QSharedPointer<QGeoTileTexture> QGeoFileTileCache::getFromMemory(const QGeoTileSpec &spec)
+{
+    QSharedPointer<QGeoTileTexture> tt = textureCache_.object(spec);
+    if (tt)
+        return tt;
+
+    QSharedPointer<QGeoCachedTileMemory> tm = memoryCache_.object(spec);
+    if (tm) {
+        QImage image;
+        if (!image.loadFromData(tm->bytes)) {
+            handleError(spec, QLatin1String("Problem with tile image"));
+            return QSharedPointer<QGeoTileTexture>(0);
+        }
+        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(spec, image);
+        if (tt)
+            return tt;
+    }
+    return QSharedPointer<QGeoTileTexture>();
+}
+
+QSharedPointer<QGeoTileTexture> QGeoFileTileCache::getFromDisk(const QGeoTileSpec &spec)
+{
+    QSharedPointer<QGeoCachedTileDisk> td = diskCache_.object(spec);
+    if (td) {
+        const QString format = QFileInfo(td->filename).suffix();
+        QFile file(td->filename);
+        file.open(QIODevice::ReadOnly);
+        QByteArray bytes = file.readAll();
+        file.close();
+
+        QImage image;
+        if (!image.loadFromData(bytes)) {
+            handleError(spec, QLatin1String("Problem with tile image"));
+            return QSharedPointer<QGeoTileTexture>(0);
+        }
+
+        addToMemoryCache(spec, bytes, format);
+        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(td->spec, image);
+        if (tt)
+            return tt;
+    }
+
+    return QSharedPointer<QGeoTileTexture>();
+}
+
 QString QGeoFileTileCache::tileSpecToFilename(const QGeoTileSpec &spec, const QString &format, const QString &directory) const
 {
     QString filename = spec.plugin();
diff --git a/src/location/maps/qgeofiletilecache_p.h b/src/location/maps/qgeofiletilecache_p.h
index 920ee56c..b6d4919f 100644
--- a/src/location/maps/qgeofiletilecache_p.h
+++ b/src/location/maps/qgeofiletilecache_p.h
@@ -138,6 +138,8 @@ protected:
     QSharedPointer<QGeoCachedTileDisk> addToDiskCache(const QGeoTileSpec &spec, const QString &filename);
     QSharedPointer<QGeoCachedTileMemory> addToMemoryCache(const QGeoTileSpec &spec, const QByteArray &bytes, const QString &format);
     QSharedPointer<QGeoTileTexture> addToTextureCache(const QGeoTileSpec &spec, const QImage &image);
+    QSharedPointer<QGeoTileTexture> getFromMemory(const QGeoTileSpec &spec);
+    QSharedPointer<QGeoTileTexture> getFromDisk(const QGeoTileSpec &spec);
 
     virtual QString tileSpecToFilename(const QGeoTileSpec &spec, const QString &format, const QString &directory) const;
     virtual QGeoTileSpec filenameToTileSpec(const QString &filename) const;
diff --git a/src/plugins/geoservices/osm/osm.pro b/src/plugins/geoservices/osm/osm.pro
index e73c16d7..3de72cea 100644
--- a/src/plugins/geoservices/osm/osm.pro
+++ b/src/plugins/geoservices/osm/osm.pro
@@ -1,6 +1,6 @@
 TARGET = qtgeoservices_osm
 
-QT += location-private positioning-private network
+QT += location-private positioning-private network concurrent
 
 HEADERS += \
     qgeoserviceproviderpluginosm.h \
@@ -14,7 +14,8 @@ HEADERS += \
     qplacemanagerengineosm.h \
     qplacesearchreplyosm.h \
     qplacecategoriesreplyosm.h \
-    qgeotiledmaposm.h
+    qgeotiledmaposm.h \
+    qgeofiletilecacheosm.h
 
 SOURCES += \
     qgeoserviceproviderpluginosm.cpp \
@@ -28,7 +29,8 @@ SOURCES += \
     qplacemanagerengineosm.cpp \
     qplacesearchreplyosm.cpp \
     qplacecategoriesreplyosm.cpp \
-    qgeotiledmaposm.cpp
+    qgeotiledmaposm.cpp \
+    qgeofiletilecacheosm.cpp
 
 
 OTHER_FILES += \
diff --git a/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp b/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
new file mode 100644
index 00000000..a780faa9
--- /dev/null
+++ b/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
@@ -0,0 +1,131 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the QtLocation module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qgeofiletilecacheosm.h"
+#include <QtLocation/private/qgeotilespec_p.h>
+#include <QDir>
+#include <QDirIterator>
+#include <QPair>
+#include <QDateTime>
+#include <QtConcurrent>
+#include <QThread>
+
+QT_BEGIN_NAMESPACE
+
+QGeoFileTileCacheOsm::QGeoFileTileCacheOsm(const QString &offlineDirectory, const QString &directory, QObject *parent)
+    :QGeoFileTileCache(directory, parent), m_offlineDirectory(offlineDirectory),
+     m_requestCancel(0)
+{
+}
+
+QGeoFileTileCacheOsm::~QGeoFileTileCacheOsm()
+{
+    m_requestCancel = 1;
+    m_future.waitForFinished();
+}
+
+QSharedPointer<QGeoTileTexture> QGeoFileTileCacheOsm::get(const QGeoTileSpec &spec)
+{
+    QSharedPointer<QGeoTileTexture> tt = getFromMemory(spec);
+    if (tt)
+        return tt;
+    if (tt = getFromOfflineStorage(spec))
+        return tt;
+    return getFromDisk(spec);
+}
+
+void QGeoFileTileCacheOsm::init()
+{
+    QGeoFileTileCache::init();
+    if (!m_offlineDirectory.isEmpty())
+        m_future = QtConcurrent::run(this, &QGeoFileTileCacheOsm::initOfflineRegistry);
+}
+
+QSharedPointer<QGeoTileTexture> QGeoFileTileCacheOsm::getFromOfflineStorage(const QGeoTileSpec &spec)
+{
+    QMutexLocker locker(&storageLock);
+    if (m_tilespecToOfflineFilepath.contains(spec)) {
+        QFile file(m_tilespecToOfflineFilepath[spec]);
+        file.open(QIODevice::ReadOnly);
+        QByteArray bytes = file.readAll();
+        file.close();
+        locker.unlock();
+
+        QImage image;
+        if (!image.loadFromData(bytes)) {
+            handleError(spec, QLatin1String("Problem with tile image"));
+            return QSharedPointer<QGeoTileTexture>(0);
+        }
+
+        addToMemoryCache(spec, bytes, QString());
+        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(spec, image);
+        if (tt)
+            return tt;
+    }
+
+    return QSharedPointer<QGeoTileTexture>();
+}
+
+void QGeoFileTileCacheOsm::initOfflineRegistry()
+{
+    // Dealing with duplicates: picking the newest
+    QMap<QString, QPair<QString, QDateTime> > fileDates; // key is filename, value is <filepath, lastmodified>
+    QDirIterator it(m_offlineDirectory, QStringList() << "*.*", QDir::Files, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks );
+    while (it.hasNext()) {
+        const QString &path = it.next();
+        QFileInfo f(path);
+        if (!fileDates.contains(f.fileName()) || fileDates[f.fileName()].second < f.lastModified())
+            fileDates[f.fileName()] = QPair<QString, QDateTime>(f.filePath(), f.lastModified());
+        if (m_requestCancel)
+            return;
+    }
+
+    int count = 0;
+    for (auto i= fileDates.begin(); i != fileDates.end(); ++i) {
+        QGeoTileSpec spec = filenameToTileSpec(i.key());
+        if (spec.zoom() == -1)
+            continue;
+        count++;
+        storageLock.lock();
+        m_tilespecToOfflineFilepath[spec] = i.value().first;
+        storageLock.unlock();
+        if (m_requestCancel)
+            return;
+    }
+    qWarning() << "OSM Offline tiles: "<<count;
+}
+
+QT_END_NAMESPACE
diff --git a/src/plugins/geoservices/osm/qgeofiletilecacheosm.h b/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
new file mode 100644
index 00000000..f48fad7f
--- /dev/null
+++ b/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the QtLocation module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QGEOFILETILECACHEOSM_H
+#define QGEOFILETILECACHEOSM_H
+
+#include <QtLocation/private/qgeofiletilecache_p.h>
+#include <QHash>
+#include <QtConcurrent>
+#include <qatomic.h>
+
+QT_BEGIN_NAMESPACE
+
+class QGeoFileTileCacheOsm : public QGeoFileTileCache
+{
+    Q_OBJECT
+public:
+    QGeoFileTileCacheOsm(const QString &offlineDirectory = QString(), const QString &directory = QString(), QObject *parent = 0);
+    ~QGeoFileTileCacheOsm();
+
+    QSharedPointer<QGeoTileTexture> get(const QGeoTileSpec &spec) Q_DECL_OVERRIDE;
+
+protected:
+    void init() Q_DECL_OVERRIDE;
+    QSharedPointer<QGeoTileTexture> getFromOfflineStorage(const QGeoTileSpec &spec);
+
+    void initOfflineRegistry();
+
+    QString m_offlineDirectory;
+    QHash<QGeoTileSpec, QString> m_tilespecToOfflineFilepath;
+    QAtomicInt m_requestCancel;
+    QFuture<void> m_future;
+    QMutex storageLock;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGEOFILETILECACHEOSM_H
diff --git a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
index d5490559..043f3f29 100644
--- a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
+++ b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
@@ -40,6 +40,7 @@
 #include "qgeotiledmappingmanagerengineosm.h"
 #include "qgeotilefetcherosm.h"
 #include "qgeotiledmaposm.h"
+#include "qgeofiletilecacheosm.h"
 
 #include <QtLocation/private/qgeocameracapabilities_p.h>
 #include <QtLocation/private/qgeomaptype_p.h>
@@ -84,6 +85,39 @@ QGeoTiledMappingManagerEngineOsm::QGeoTiledMappingManagerEngineOsm(const QVarian
 
     setTileFetcher(tileFetcher);
 
+
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.directory"))) {
+        m_cacheDirectory = parameters.value(QStringLiteral("osm.mapping.cache.directory")).toString();
+    } else {
+        // managerName() is not yet set, we have to hardcode the plugin name below
+        m_cacheDirectory = QAbstractGeoTileCache::baseCacheDirectory() + QLatin1String("osm");
+    }
+    if (parameters.contains(QStringLiteral("osm.mapping.offline.directory"))) {
+        m_offlineDirectory = parameters.value(QStringLiteral("osm.mapping.offline.directory")).toString();
+    }
+    QAbstractGeoTileCache *tileCache = new QGeoFileTileCacheOsm(m_offlineDirectory, m_cacheDirectory);
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.disk.size"))) {
+        bool ok = false;
+        int cacheSize = parameters.value(QStringLiteral("osm.mapping.cache.disk.size")).toString().toInt(&ok);
+        if (ok)
+            tileCache->setMaxDiskUsage(cacheSize);
+    } else {
+        tileCache->setMaxDiskUsage(100 * 1024 * 1024);
+    }
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.memory.size"))) {
+        bool ok = false;
+        int cacheSize = parameters.value(QStringLiteral("osm.mapping.cache.memory.size")).toString().toInt(&ok);
+        if (ok)
+            tileCache->setMaxMemoryUsage(cacheSize);
+    }
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.texture.size"))) {
+        bool ok = false;
+        int cacheSize = parameters.value(QStringLiteral("osm.mapping.cache.texture.size")).toString().toInt(&ok);
+        if (ok)
+            tileCache->setExtraTextureUsage(cacheSize);
+    }
+    setTileCache(tileCache);
+
     *error = QGeoServiceProvider::NoError;
     errorString->clear();
 }
diff --git a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h
index e0f7d873..4998caf9 100644
--- a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h
+++ b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h
@@ -60,6 +60,8 @@ public:
 
 private:
     QString m_customCopyright;
+    QString m_cacheDirectory;
+    QString m_offlineDirectory;
 };
 
 QT_END_NAMESPACE