diff options
author | Alexander Shalamov <alexander.shalamov@mapbox.com> | 2020-01-31 23:32:58 +0200 |
---|---|---|
committer | Alexander Shalamov <alexander.shalamov@mapbox.com> | 2020-01-31 23:32:58 +0200 |
commit | 6479d0da8e17ccc7d38521096d051a1e91921b14 (patch) | |
tree | bd52940b53d703d9fae49e8a7dc6a51b4fed341f | |
parent | d7c8104decbbea8203bd79a95c0647bc4e1a2196 (diff) |
Revert "[core] Modularize FileSource codebase (#15768)"
This reverts commit 879c44f661c5eb762c93a721b657859a71aabfc7.
82 files changed, 1329 insertions, 1840 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 381f4e2fd..75b6454cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,14 +133,6 @@ Introduces public [mblg::Renderer::render](https://github.com/mapbox/mapbox-gl-native/pull/16127/files#diff-5a977e0401792825d7fcf522d48df11fR34) API break. -- [core] Refactor DefaultFileSource codebase ([#15768](https://github.com/mapbox/mapbox-gl-native/pull/15768)) - - Adds `FileSourceManager` interface that provides access to `FileSource` instances and means of registering / unregistering `FileSource` factories - - Splits `DefaultFileSource` into smaller parts - - Adds `DatabaseFileSource` interface and it's default implementation - - Removes inter-dependencies between concrete `FileSource` classes - - All sources operate on dedicated thread, except `MainResourceLoader` that acts as a dispatcher and works on thread that requested it. - - Removes `ResourceOptions::withCacheOnlyRequestsSupport` method - - [core] Remove Map::cycleDebugOptions ([#16005](https://github.com/mapbox/mapbox-gl-native/pull/16005)) This function was mostly used by the Android API, which is no longer necessary. diff --git a/bin/cache.cpp b/bin/cache.cpp index 1df782f75..eee1d61b3 100644 --- a/bin/cache.cpp +++ b/bin/cache.cpp @@ -1,6 +1,5 @@ -#include <mbgl/storage/file_source_manager.hpp> +#include <mbgl/storage/default_file_source.hpp> #include <mbgl/storage/resource.hpp> -#include <mbgl/storage/resource_options.hpp> #include <mbgl/util/chrono.hpp> #include <mbgl/util/run_loop.hpp> @@ -90,9 +89,9 @@ int main(int argc, char* argv[]) { } mbgl::util::RunLoop loop; - auto dbfs = mbgl::FileSourceManager::get()->getFileSource( - mbgl::FileSourceType::Database, mbgl::ResourceOptions().withCachePath(args::get(cacheValue))); - dbfs->forward(resource, response, [&loop] { loop.stop(); }); - loop.run(); + mbgl::DefaultFileSource fileSource(args::get(cacheValue), "."); + + fileSource.put(resource, response); + return 0; } diff --git a/bin/offline.cpp b/bin/offline.cpp index c18fc3181..fcc6adc3e 100644 --- a/bin/offline.cpp +++ b/bin/offline.cpp @@ -3,9 +3,7 @@ #include <mbgl/util/string.hpp> #include <mbgl/util/geojson.hpp> -#include <mbgl/storage/database_file_source.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/resource_options.hpp> +#include <mbgl/storage/default_file_source.hpp> #include <args.hxx> @@ -163,16 +161,17 @@ int main(int argc, char *argv[]) { util::RunLoop loop; - std::shared_ptr<DatabaseFileSource> fileSource = - std::static_pointer_cast<DatabaseFileSource>(FileSourceManager::get()->getFileSource( - FileSourceType::Database, - ResourceOptions().withAccessToken(token).withBaseURL(apiBaseURL).withCachePath(output))); - + DefaultFileSource fileSource(output, "."); std::unique_ptr<OfflineRegion> region; - if (inputDb && mergePath) { - DatabaseFileSource inputSource(ResourceOptions().withCachePath(*inputDb)); + fileSource.setAccessToken(token); + fileSource.setAPIBaseURL(apiBaseURL); + if (inputDb && mergePath) { + DefaultFileSource inputSource(*inputDb, "."); + inputSource.setAccessToken(token); + inputSource.setAPIBaseURL(apiBaseURL); + int retCode = 0; std::cout << "Start Merge" << std::endl; inputSource.mergeOfflineRegions(*mergePath, [&] (mbgl::expected<std::vector<OfflineRegion>, std::exception_ptr> result) { @@ -194,15 +193,13 @@ int main(int argc, char *argv[]) { class Observer : public OfflineRegionObserver { public: - Observer(OfflineRegion& region_, - std::shared_ptr<DatabaseFileSource> fileSource_, - util::RunLoop& loop_, - mbgl::optional<std::string> mergePath_) + Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_, mbgl::optional<std::string> mergePath_) : region(region_), - fileSource(std::move(fileSource_)), + fileSource(fileSource_), loop(loop_), mergePath(std::move(mergePath_)), - start(util::now()) {} + start(util::now()) { + } void statusChanged(OfflineRegionStatus status) override { if (status.downloadState == OfflineRegionDownloadState::Inactive) { @@ -218,11 +215,14 @@ int main(int argc, char *argv[]) { bytesPerSecond = util::toString(status.completedResourceSize / elapsedSeconds); } - std::cout << status.completedResourceCount << " / " << status.requiredResourceCount << " resources | " - << status.completedTileCount << " / " << status.requiredTileCount << " tiles" - << (status.requiredResourceCountIsPrecise ? " | " : " (indeterminate); ") + std::cout << status.completedResourceCount << " / " << status.requiredResourceCount + << " resources" + << status.completedTileCount << " / " << status.requiredTileCount + << "tiles" + << (status.requiredResourceCountIsPrecise ? "; " : " (indeterminate); ") << status.completedResourceSize << " bytes downloaded" - << " (" << bytesPerSecond << " bytes/sec)" << std::endl; + << " (" << bytesPerSecond << " bytes/sec)" + << std::endl; if (status.complete()) { std::cout << "Finished Download" << std::endl; @@ -239,7 +239,7 @@ int main(int argc, char *argv[]) { } OfflineRegion& region; - std::shared_ptr<DatabaseFileSource> fileSource; + DefaultFileSource& fileSource; util::RunLoop& loop; mbgl::optional<std::string> mergePath; Timestamp start; @@ -248,26 +248,24 @@ int main(int argc, char *argv[]) { static auto stop = [&] { if (region) { std::cout << "Stopping download... "; - fileSource->setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Inactive); + fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Inactive); } }; std::signal(SIGINT, [] (int) { stop(); }); - fileSource->createOfflineRegion( - definition, metadata, [&](mbgl::expected<OfflineRegion, std::exception_ptr> region_) { - if (!region_) { - std::cerr << "Error creating region: " << util::toString(region_.error()) << std::endl; - loop.stop(); - exit(1); - } else { - assert(region_); - region = std::make_unique<OfflineRegion>(std::move(*region_)); - fileSource->setOfflineRegionObserver(*region, - std::make_unique<Observer>(*region, fileSource, loop, mergePath)); - fileSource->setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active); - } - }); + fileSource.createOfflineRegion(definition, metadata, [&] (mbgl::expected<OfflineRegion, std::exception_ptr> region_) { + if (!region_) { + std::cerr << "Error creating region: " << util::toString(region_.error()) << std::endl; + loop.stop(); + exit(1); + } else { + assert(region_); + region = std::make_unique<OfflineRegion>(std::move(*region_)); + fileSource.setOfflineRegionObserver(*region, std::make_unique<Observer>(*region, fileSource, loop, mergePath)); + fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active); + } + }); loop.run(); return 0; diff --git a/include/mbgl/storage/database_file_source.hpp b/include/mbgl/storage/default_file_source.hpp index 81e5315fd..fdd430cca 100644 --- a/include/mbgl/storage/database_file_source.hpp +++ b/include/mbgl/storage/default_file_source.hpp @@ -1,130 +1,45 @@ #pragma once +#include <mbgl/actor/actor_ref.hpp> #include <mbgl/storage/file_source.hpp> #include <mbgl/storage/offline.hpp> -#include <mbgl/util/expected.hpp> +#include <mbgl/util/constants.hpp> #include <mbgl/util/optional.hpp> +#include <mbgl/util/expected.hpp> -namespace mbgl { +#include <vector> +#include <mutex> -class ResourceOptions; +namespace mbgl { -// TODO: Split DatabaseFileSource into Ambient cache and Database interfaces. -class DatabaseFileSource : public FileSource { -public: - explicit DatabaseFileSource(const ResourceOptions& options); - ~DatabaseFileSource() override; +namespace util { +template <typename T> class Thread; +} // namespace util - // FileSource overrides - std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - void forward(const Resource&, const Response&, std::function<void()> callback) override; - bool canRequest(const Resource&) const override; - void setProperty(const std::string&, const mapbox::base::Value&) override; - void pause() override; - void resume() override; +class ResourceTransform; - // Methods common to Ambient cache and Offline functionality +// TODO: the callback should include a potential error info when https://github.com/mapbox/mapbox-gl-native/issues/14759 is resolved +using PathChangeCallback = std::function<void ()>; - /* - * Sets path of a database to be used by DatabaseFileSource and invokes provided - * callback when a database path is set. - */ - virtual void setDatabasePath(const std::string&, std::function<void()> callback); +class DefaultFileSource : public FileSource { +public: + DefaultFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true); + DefaultFileSource(const std::string& cachePath, std::unique_ptr<FileSource>&& assetFileSource, bool supportCacheOnlyRequests = true); + ~DefaultFileSource() override; - /* - * Delete existing database and re-initialize. - * - * When the operation is complete or encounters an error, the given callback will be - * executed on the database thread; it is the responsibility of the SDK bindings - * to re-execute a user-provided callback on the main thread. - */ - virtual void resetDatabase(std::function<void(std::exception_ptr)>); + bool supportsCacheOnlyRequests() const override; - /* - * Packs the existing database file into a minimal amount of disk space. - * - * This operation has a performance impact as it will vacuum the database, - * forcing it to move pages on the filesystem. - * - * When the operation is complete or encounters an error, the given callback will be - * executed on the database thread; it is the responsibility of the SDK bindings - * to re-execute a user-provided callback on the main thread. - */ - virtual void packDatabase(std::function<void(std::exception_ptr)> callback); + void setAPIBaseURL(const std::string&); + std::string getAPIBaseURL(); - /* - * Sets whether packing the database file occurs automatically after an offline - * region is deleted (deleteOfflineRegion()) or the ambient cache is cleared - * (clearAmbientCache()). - * - * By default, packing is enabled. If disabled, disk space will not be freed - * after resources are removed unless packDatabase() is explicitly called. - */ - virtual void runPackDatabaseAutomatically(bool); + void setAccessToken(const std::string&); + std::string getAccessToken(); - // Ambient cache + void setResourceTransform(optional<ActorRef<ResourceTransform>>&&); - /* - * Insert the provided resource into the ambient cache - * - * Consumers of the resource will expect the uncompressed version; the - * OfflineDatabase will determine whether to compress the data on disk. - * This call is asynchronous: the data may not be immediately available - * for in-progress requests, although subsequent requests should have - * access to the cached data. - */ - virtual void put(const Resource&, const Response&); + void setResourceCachePath(const std::string&, optional<ActorRef<PathChangeCallback>>&&); - /* - * Forces revalidation of the ambient cache. - * - * Forces Mapbox GL Native to revalidate resources stored in the ambient - * cache with the tile server before using them, making sure they - * are the latest version. This is more efficient than cleaning the - * cache because if the resource is considered valid after the server - * lookup, it will not get downloaded again. - * - * Resources overlapping with offline regions will not be affected - * by this call. - */ - virtual void invalidateAmbientCache(std::function<void(std::exception_ptr)>); - - /* - * Erase resources from the ambient cache, freeing storage space. - * - * Erases the ambient cache, freeing resources. - * - * Note that this operation can be potentially slow if packing the database - * occurs automatically (see runPackDatabaseAutomatically() and packDatabase()). - * - * Resources overlapping with offline regions will not be affected - * by this call. - */ - virtual void clearAmbientCache(std::function<void(std::exception_ptr)>); - - /* - * Sets the maximum size in bytes for the ambient cache. - * - * This call is potentially expensive because it will try - * to trim the data in case the database is larger than the - * size defined. The size of offline regions are not affected - * by this settings, but the ambient cache will always try - * to not exceed the maximum size defined, taking into account - * the current size for the offline regions. - * - * If the maximum size is set to 50 MB and 40 MB are already - * used by offline regions, the cache size will be effectively - * 10 MB. - * - * Setting the size to 0 will disable the cache if there is no - * offline region on the database. - * - * This method should always be called before using the database, - * otherwise the default maximum size will be used. - */ - virtual void setMaximumAmbientCacheSize(uint64_t size, std::function<void(std::exception_ptr)> callback); - - // Offline + std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; /* * Retrieve all regions in the offline database. @@ -133,7 +48,7 @@ public: * callback, which will be executed on the database thread; it is the responsibility * of the SDK bindings to re-execute a user-provided callback on the main thread. */ - virtual void listOfflineRegions(std::function<void(expected<OfflineRegions, std::exception_ptr>)>); + void listOfflineRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)>); /* * Create an offline region in the database. @@ -146,25 +61,25 @@ public: * downloading resources, call `setOfflineRegionDownloadState(OfflineRegionDownloadState::Active)`, * optionally registering an `OfflineRegionObserver` beforehand. */ - virtual void createOfflineRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata, - std::function<void(expected<OfflineRegion, std::exception_ptr>)>); + void createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)>); + /* * Update an offline region metadata in the database. */ - virtual void updateOfflineMetadata(const int64_t regionID, - const OfflineRegionMetadata& metadata, - std::function<void(expected<OfflineRegionMetadata, std::exception_ptr>)>); - + void updateOfflineMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)>); /* * Register an observer to be notified when the state of the region changes. */ - virtual void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr<OfflineRegionObserver>); + void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr<OfflineRegionObserver>); /* * Pause or resume downloading of regional resources. */ - virtual void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState); + void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState); /* * Retrieve the current status of the region. The query will be executed @@ -172,8 +87,9 @@ public: * executed on the database thread; it is the responsibility of the SDK bindings * to re-execute a user-provided callback on the main thread. */ - virtual void getOfflineRegionStatus(OfflineRegion&, - std::function<void(expected<OfflineRegionStatus, std::exception_ptr>)>) const; + void getOfflineRegionStatus( + OfflineRegion&, + std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)>) const; /* * Merge offline regions from a secondary database into the main offline database. @@ -195,8 +111,8 @@ public: * Merged regions may not be in a completed status if the secondary database * does not contain all the tiles or resources required by the region definition. */ - virtual void mergeOfflineRegions(const std::string& sideDatabasePath, - std::function<void(expected<OfflineRegions, std::exception_ptr>)>); + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)>); /* * Remove an offline region from the database and perform any resources evictions @@ -216,7 +132,7 @@ public: * executed on the database thread; it is the responsibility of the SDK bindings * to re-execute a user-provided callback on the main thread. */ - virtual void deleteOfflineRegion(OfflineRegion, std::function<void(std::exception_ptr)>); + void deleteOfflineRegion(OfflineRegion&&, std::function<void(std::exception_ptr)>); /* * Invalidate all the tiles from an offline region forcing Mapbox GL to revalidate @@ -224,17 +140,140 @@ public: * offline region and downloading it again because if the data on the cache matches * the server, no new data gets transmitted. */ - virtual void invalidateOfflineRegion(OfflineRegion&, std::function<void(std::exception_ptr)>); + void invalidateOfflineRegion(OfflineRegion&, std::function<void(std::exception_ptr)>); /* * Changing or bypassing this limit without permission from Mapbox is prohibited * by the Mapbox Terms of Service. */ - virtual void setOfflineMapboxTileCountLimit(uint64_t) const; + void setOfflineMapboxTileCountLimit(uint64_t) const; + + /* + * Pause file request activity. + * + * If pause is called then no revalidation or network request activity + * will occur. + */ + void pause(); + + /* + * Resume file request activity. + * + * Calling resume will unpause the file source and process any tasks that + * expired while the file source was paused. + */ + void resume(); + + /* + * Insert the provided resource into the ambient cache + * + * Consumers of the resource will expect the uncompressed version; the + * OfflineDatabase will determine whether to compress the data on disk. + * This call is asynchronous: the data may not be immediately available + * for in-progress requests, although subsequent requests should have + * access to the cached data. + */ + void put(const Resource&, const Response&); + + /* + * Delete existing database and re-initialize. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void resetDatabase(std::function<void(std::exception_ptr)>); + + /* + * Packs the existing database file into a minimal amount of disk space. + * + * This operation has a performance impact as it will vacuum the database, + * forcing it to move pages on the filesystem. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void packDatabase(std::function<void(std::exception_ptr)> callback); + + /* + * Sets whether packing the database file occurs automatically after an offline + * region is deleted (deleteOfflineRegion()) or the ambient cache is cleared + * (clearAmbientCache()). + * + * By default, packing is enabled. If disabled, disk space will not be freed + * after resources are removed unless packDatabase() is explicitly called. + */ + void runPackDatabaseAutomatically(bool); + + /* + * Forces revalidation of the ambient cache. + * + * Forces Mapbox GL Native to revalidate resources stored in the ambient + * cache with the tile server before using them, making sure they + * are the latest version. This is more efficient than cleaning the + * cache because if the resource is considered valid after the server + * lookup, it will not get downloaded again. + * + * Resources overlapping with offline regions will not be affected + * by this call. + */ + void invalidateAmbientCache(std::function<void (std::exception_ptr)>); + + /* + * Erase resources from the ambient cache, freeing storage space. + * + * Erases the ambient cache, freeing resources. + * + * Note that this operation can be potentially slow if packing the database + * occurs automatically (see runPackDatabaseAutomatically() and packDatabase()). + * + * Resources overlapping with offline regions will not be affected + * by this call. + */ + void clearAmbientCache(std::function<void (std::exception_ptr)>); + + /* + * Sets the maximum size in bytes for the ambient cache. + * + * This call is potentially expensive because it will try + * to trim the data in case the database is larger than the + * size defined. The size of offline regions are not affected + * by this settings, but the ambient cache will always try + * to not exceed the maximum size defined, taking into account + * the current size for the offline regions. + * + * If the maximum size is set to 50 MB and 40 MB are already + * used by offline regions, the cache size will be effectively + * 10 MB. + * + * Setting the size to 0 will disable the cache if there is no + * offline region on the database. + * + * This method should always be called before using the database, + * otherwise the default maximum size will be used. + */ + void setMaximumAmbientCacheSize(uint64_t size, std::function<void (std::exception_ptr)> callback); + + // For testing only. + void setOnlineStatus(bool); + void reopenDatabaseReadOnlyForTesting(); + void setMaximumConcurrentRequests(uint32_t); -private: class Impl; - const std::unique_ptr<Impl> impl; + +private: + // Shared so destruction is done on this thread + const std::shared_ptr<FileSource> assetFileSource; + const std::unique_ptr<util::Thread<Impl>> impl; + + std::mutex cachedBaseURLMutex; + std::string cachedBaseURL = mbgl::util::API_BASE_URL; + + std::mutex cachedAccessTokenMutex; + std::string cachedAccessToken; + + const bool supportCacheOnlyRequests; }; } // namespace mbgl diff --git a/include/mbgl/storage/file_source.hpp b/include/mbgl/storage/file_source.hpp index f05f1473e..2270038c4 100644 --- a/include/mbgl/storage/file_source.hpp +++ b/include/mbgl/storage/file_source.hpp @@ -1,34 +1,19 @@ #pragma once +#include <mbgl/actor/actor_ref.hpp> #include <mbgl/storage/response.hpp> +#include <mbgl/storage/resource.hpp> -#include <mapbox/value.hpp> +#include <mbgl/util/async_request.hpp> #include <functional> #include <memory> namespace mbgl { -class AsyncRequest; -class Resource; +class ResourceOptions; +class ResourceTransform; -// TODO: Rename to ResourceProviderType -enum FileSourceType : uint8_t { - Asset, - // TODO: split to separate types - // - Cache for fast KV store (FASTER, LevelDB, RocksDB) - // - Database for read-only offline use-cases - Database, - FileSystem, - Network, - // Resource loader acts as a proxy and has logic - // for request delegation to Asset, Cache, and other - // file sources. - ResourceLoader -}; - -// TODO: Rename to ResourceProvider to avoid confusion with -// GeoJSONSource, RasterSource, VectorSource, CustomGeometrySource and other *Sources. class FileSource { public: FileSource(const FileSource&) = delete; @@ -44,44 +29,22 @@ public: // not be executed. virtual std::unique_ptr<AsyncRequest> request(const Resource&, Callback) = 0; - // Allows to forward response from one source to another. - // Optionally, callback can be provided to receive notification for forward - // operation. - virtual void forward(const Resource&, const Response&, std::function<void()> = {}) {} - // When a file source supports consulting a local cache only, it must return true. // Cache-only requests are requests that aren't as urgent, but could be useful, e.g. // to cover part of the map while loading. The FileSource should only do cheap actions to // retrieve the data, e.g. load it from a cache, but not from the internet. - virtual bool supportsCacheOnlyRequests() const { return false; } - - // Checks whether a resource could be requested from this file source. - virtual bool canRequest(const Resource&) const = 0; - - /* - * Pause file request activity. - * - * If pause is called then no revalidation or network request activity - * will occur. - */ - virtual void pause() {} - - /* - * Resume file request activity. - * - * Calling resume will unpause the file source and process any tasks that - * expired while the file source was paused. - */ - virtual void resume() {} + virtual bool supportsCacheOnlyRequests() const { + return false; + } - /* - * Generic property setter / getter methods. - */ - virtual void setProperty(const std::string&, const mapbox::base::Value&){}; - virtual mapbox::base::Value getProperty(const std::string&) const { return {}; }; + // Singleton for obtaining the shared platform-specific file source. A single instance of a file source is provided + // for each unique combination of a Mapbox API base URL, access token, cache path and platform context. + static std::shared_ptr<FileSource> getSharedFileSource(const ResourceOptions&); protected: FileSource() = default; + // Factory for creating a platform-specific file source. + static std::shared_ptr<FileSource> createPlatformFileSource(const ResourceOptions&); }; } // namespace mbgl diff --git a/include/mbgl/storage/file_source_manager.hpp b/include/mbgl/storage/file_source_manager.hpp deleted file mode 100644 index 2b2a43cbe..000000000 --- a/include/mbgl/storage/file_source_manager.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include <mbgl/storage/file_source.hpp> - -namespace mbgl { - -class ResourceOptions; - -/** - * @brief A singleton class responsible for managing file sources. - * - * The FileSourceManager provides following functionality: - * - * - provides access to file sources of a specific type and configuration - * - caches previously created file sources of a (type, configuration) tuples - * - allows to register and unregister file source factories - */ -class FileSourceManager { -public: - using FileSourceFactory = std::function<std::unique_ptr<FileSource>(const ResourceOptions&)>; - - /** - * @brief A singleton getter. - * - * @return FileSourceManager* - */ - static FileSourceManager* get() noexcept; - - // Returns shared instance of a file source for (type, options) tuple. - // Creates new instance via registered factory if needed. If new instance cannot be - // created, nullptr would be returned. - std::shared_ptr<FileSource> getFileSource(FileSourceType, const ResourceOptions&) noexcept; - - // Registers file source factory for a provided FileSourceType type. If factory for the - // same type was already registered, will unregister previously registered factory. - // Provided factory must not be null. - virtual void registerFileSourceFactory(FileSourceType, FileSourceFactory&&) noexcept; - - // Unregisters file source factory. If there are no registered factories for a FileSourceType - // invocation has no effect. - virtual FileSourceFactory unRegisterFileSourceFactory(FileSourceType) noexcept; - -protected: - FileSourceManager(); - class Impl; - std::unique_ptr<Impl> impl; - virtual ~FileSourceManager(); -}; - -} // namespace mbgl diff --git a/include/mbgl/storage/online_file_source.hpp b/include/mbgl/storage/online_file_source.hpp index 8969d2187..b2e9b43e5 100644 --- a/include/mbgl/storage/online_file_source.hpp +++ b/include/mbgl/storage/online_file_source.hpp @@ -1,46 +1,42 @@ #pragma once +#include <mbgl/actor/actor_ref.hpp> #include <mbgl/storage/file_source.hpp> +#include <mbgl/util/constants.hpp> #include <mbgl/util/optional.hpp> namespace mbgl { class ResourceTransform; -// Properties that may be supported by online file sources. - -// Property name to set / get an access token. -// type: std::string -constexpr const char* ACCESS_TOKEN_KEY = "access-token"; - -// Property name to set / get base url. -// type: std::string -constexpr const char* API_BASE_URL_KEY = "api-base-url"; - -// Property name to set / get maximum number of concurrent requests. -// type: unsigned -constexpr const char* MAX_CONCURRENT_REQUESTS_KEY = "max-concurrent-requests"; - class OnlineFileSource : public FileSource { public: OnlineFileSource(); ~OnlineFileSource() override; - // FileSource overrides + void setAPIBaseURL(const std::string& t) { apiBaseURL = t; } + std::string getAPIBaseURL() const { return apiBaseURL; } + + void setAccessToken(const std::string& t) { accessToken = t; } + std::string getAccessToken() const { return accessToken; } + + void setResourceTransform(optional<ActorRef<ResourceTransform>>&&); + std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource&) const override; - void pause() override; - void resume() override; - void setProperty(const std::string&, const mapbox::base::Value&) override; - mapbox::base::Value getProperty(const std::string&) const override; - // OnlineFileSource interface. - // TODO: Would be nice to drop it to get uniform interface. - virtual void setResourceTransform(ResourceTransform); + void setMaximumConcurrentRequests(uint32_t); + uint32_t getMaximumConcurrentRequests() const; + + // For testing only. + void setOnlineStatus(bool); private: + friend class OnlineFileRequest; + class Impl; const std::unique_ptr<Impl> impl; + std::string accessToken; + std::string apiBaseURL = mbgl::util::API_BASE_URL; }; } // namespace mbgl diff --git a/include/mbgl/storage/resource.hpp b/include/mbgl/storage/resource.hpp index b21265ef5..d00f33666 100644 --- a/include/mbgl/storage/resource.hpp +++ b/include/mbgl/storage/resource.hpp @@ -64,7 +64,7 @@ public: void setPriority(Priority p) { priority = p; } void setUsage(Usage u) { usage = u; } - bool hasLoadingMethod(LoadingMethod method) const; + bool hasLoadingMethod(LoadingMethod method); static Resource style(const std::string& url); static Resource source(const std::string& url); @@ -97,7 +97,7 @@ public: std::shared_ptr<const std::string> priorData; }; -inline bool Resource::hasLoadingMethod(Resource::LoadingMethod method) const { +inline bool Resource::hasLoadingMethod(Resource::LoadingMethod method) { return (loadingMethod & method); } diff --git a/include/mbgl/storage/resource_options.hpp b/include/mbgl/storage/resource_options.hpp index 6d603b8cc..00dc6e10d 100644 --- a/include/mbgl/storage/resource_options.hpp +++ b/include/mbgl/storage/resource_options.hpp @@ -97,6 +97,21 @@ public: uint64_t maximumCacheSize() const; /** + * @brief Sets whether to support cache-only requests. + * + * @return Whether or not cache-only requests are supported. + */ + bool supportsCacheOnlyRequests() const; + + /** + * @brief Gets the previously set (or default) support for cache-only requests. + * + * @param cacheOnly Whether or not cache-only requests are supported. + * @return reference to ResourceOptions for chaining options together. + */ + ResourceOptions& withCacheOnlyRequestsSupport(bool cacheOnly); + + /** * @brief Sets the platform context. A platform context is usually an object * that assists the creation of a file source. * diff --git a/include/mbgl/storage/resource_transform.hpp b/include/mbgl/storage/resource_transform.hpp index 29d19bb4e..b8e3dbac7 100644 --- a/include/mbgl/storage/resource_transform.hpp +++ b/include/mbgl/storage/resource_transform.hpp @@ -1,5 +1,6 @@ #pragma once +#include <mbgl/actor/actor_ref.hpp> #include <mbgl/storage/resource.hpp> #include <functional> @@ -7,14 +8,16 @@ namespace mbgl { +class Mailbox; + class ResourceTransform { public: - using FinishedCallback = std::function<void(const std::string&)>; - using TransformCallback = std::function<void(Resource::Kind kind, const std::string& url, FinishedCallback)>; + using TransformCallback = std::function<std::string(Resource::Kind kind, const std::string& url)>; + using FinishedCallback = std::function<void(const std::string&&)>; + + ResourceTransform(ActorRef<ResourceTransform>, TransformCallback&&); - ResourceTransform(TransformCallback = {}); - void transform(Resource::Kind, const std::string& url, FinishedCallback); - explicit operator bool() const { return bool(transformCallback); } + void transform(Resource::Kind, const std::string& url, FinishedCallback&&); private: TransformCallback transformCallback; diff --git a/include/mbgl/style/style.hpp b/include/mbgl/style/style.hpp index 83d6ad5bb..4a6a542b8 100644 --- a/include/mbgl/style/style.hpp +++ b/include/mbgl/style/style.hpp @@ -21,7 +21,7 @@ class Layer; class Style { public: - Style(std::shared_ptr<FileSource>, float pixelRatio); + Style(FileSource&, float pixelRatio); ~Style(); void loadJSON(const std::string&); diff --git a/include/mbgl/util/constants.hpp b/include/mbgl/util/constants.hpp index 56f42ac89..ffdc4b2b3 100644 --- a/include/mbgl/util/constants.hpp +++ b/include/mbgl/util/constants.hpp @@ -60,9 +60,6 @@ constexpr UnitBezier DEFAULT_TRANSITION_EASE = { 0, 0, 0.25, 1 }; constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5; constexpr const char* API_BASE_URL = "https://api.mapbox.com"; -constexpr const char* ASSET_PROTOCOL = "asset://"; -constexpr const char* FILE_PROTOCOL = "file://"; -constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15; diff --git a/next/CMakeLists.txt b/next/CMakeLists.txt index 1950a1144..f0beebdb4 100644 --- a/next/CMakeLists.txt +++ b/next/CMakeLists.txt @@ -130,9 +130,8 @@ add_library( ${MBGL_ROOT}/include/mbgl/renderer/renderer_frontend.hpp ${MBGL_ROOT}/include/mbgl/renderer/renderer_observer.hpp ${MBGL_ROOT}/include/mbgl/renderer/renderer_state.hpp - ${MBGL_ROOT}/include/mbgl/storage/database_file_source.hpp + ${MBGL_ROOT}/include/mbgl/storage/default_file_source.hpp ${MBGL_ROOT}/include/mbgl/storage/file_source.hpp - ${MBGL_ROOT}/include/mbgl/storage/file_source_manager.hpp ${MBGL_ROOT}/include/mbgl/storage/network_status.hpp ${MBGL_ROOT}/include/mbgl/storage/offline.hpp ${MBGL_ROOT}/include/mbgl/storage/online_file_source.hpp @@ -597,10 +596,9 @@ add_library( ${MBGL_ROOT}/src/mbgl/sprite/sprite_parser.cpp ${MBGL_ROOT}/src/mbgl/sprite/sprite_parser.hpp ${MBGL_ROOT}/src/mbgl/storage/asset_file_source.hpp - ${MBGL_ROOT}/src/mbgl/storage/file_source_manager.cpp + ${MBGL_ROOT}/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/src/mbgl/storage/http_file_source.hpp ${MBGL_ROOT}/src/mbgl/storage/local_file_source.hpp - ${MBGL_ROOT}/src/mbgl/storage/main_resource_loader.hpp ${MBGL_ROOT}/src/mbgl/storage/network_status.cpp ${MBGL_ROOT}/src/mbgl/storage/resource.cpp ${MBGL_ROOT}/src/mbgl/storage/resource_options.cpp diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake index 454e11bcd..429963b87 100644 --- a/next/platform/android/android.cmake +++ b/next/platform/android/android.cmake @@ -214,12 +214,10 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/gl/headless_backend.cpp ${MBGL_ROOT}/platform/default/src/mbgl/map/map_snapshotter.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp @@ -329,6 +327,7 @@ add_library( ${MBGL_ROOT}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp ${MBGL_ROOT}/platform/android/src/test/collator_test_stub.cpp ${MBGL_ROOT}/platform/android/src/test/number_format_test_stub.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/android/src/test/http_file_source_test_stub.cpp ) @@ -378,6 +377,7 @@ add_library( ${MBGL_ROOT}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp ${MBGL_ROOT}/platform/android/src/test/collator_test_stub.cpp ${MBGL_ROOT}/platform/android/src/test/number_format_test_stub.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/android/src/test/http_file_source_test_stub.cpp ) diff --git a/next/platform/ios/ios.cmake b/next/platform/ios/ios.cmake index 68578e0df..ca7e9ae4d 100644 --- a/next/platform/ios/ios.cmake +++ b/next/platform/ios/ios.cmake @@ -40,12 +40,11 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/gl/headless_backend.cpp ${MBGL_ROOT}/platform/default/src/mbgl/map/map_snapshotter.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake index 6f2ac12e4..6c06a75e6 100644 --- a/next/platform/linux/linux.cmake +++ b/next/platform/linux/linux.cmake @@ -19,13 +19,12 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/i18n/number_format.cpp ${MBGL_ROOT}/platform/default/src/mbgl/layermanager/layer_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/http_file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake index 2b28b7e1e..9a7f538db 100644 --- a/next/platform/macos/macos.cmake +++ b/next/platform/macos/macos.cmake @@ -95,12 +95,11 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/gl/headless_backend.cpp ${MBGL_ROOT}/platform/default/src/mbgl/map/map_snapshotter.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/platform/qt/qt.cmake b/next/platform/qt/qt.cmake index 74cea29ef..4d8189194 100644 --- a/next/platform/qt/qt.cmake +++ b/next/platform/qt/qt.cmake @@ -32,12 +32,11 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/i18n/collator.cpp ${MBGL_ROOT}/platform/default/src/mbgl/layermanager/layer_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/test/CMakeLists.txt b/next/test/CMakeLists.txt index cb0519ade..06d8045d0 100644 --- a/next/test/CMakeLists.txt +++ b/next/test/CMakeLists.txt @@ -35,11 +35,10 @@ add_library( ${MBGL_ROOT}/test/src/mbgl/test/test.cpp ${MBGL_ROOT}/test/src/mbgl/test/util.cpp ${MBGL_ROOT}/test/storage/asset_file_source.test.cpp - ${MBGL_ROOT}/test/storage/database_file_source.test.cpp + ${MBGL_ROOT}/test/storage/default_file_source.test.cpp ${MBGL_ROOT}/test/storage/headers.test.cpp ${MBGL_ROOT}/test/storage/http_file_source.test.cpp ${MBGL_ROOT}/test/storage/local_file_source.test.cpp - ${MBGL_ROOT}/test/storage/main_resource_loader.test.cpp ${MBGL_ROOT}/test/storage/offline.test.cpp ${MBGL_ROOT}/test/storage/offline_database.test.cpp ${MBGL_ROOT}/test/storage/offline_download.test.cpp diff --git a/platform/android/src/test/http_file_source_test_stub.cpp b/platform/android/src/test/http_file_source_test_stub.cpp index 48e6a965f..930a20907 100644 --- a/platform/android/src/test/http_file_source_test_stub.cpp +++ b/platform/android/src/test/http_file_source_test_stub.cpp @@ -1,6 +1,6 @@ #include <mbgl/storage/http_file_source.hpp> #include <mbgl/storage/resource.hpp> -#include <mbgl/util/async_request.hpp> + #include <mbgl/util/async_task.hpp> namespace mbgl { diff --git a/platform/darwin/filesource-files.json b/platform/darwin/filesource-files.json index 4b2fe21c0..62043a0dc 100644 --- a/platform/darwin/filesource-files.json +++ b/platform/darwin/filesource-files.json @@ -4,6 +4,7 @@ "platform/darwin/src/MGLLoggingConfiguration.mm", "platform/darwin/src/MGLNetworkConfiguration.m", "platform/darwin/src/http_file_source.mm", + "platform/default/src/mbgl/storage/file_source.cpp", "platform/default/src/mbgl/storage/sqlite3.cpp" ], "public_headers": {}, diff --git a/platform/default/filesource-files.json b/platform/default/filesource-files.json index 72e76670b..f61aa6a33 100644 --- a/platform/default/filesource-files.json +++ b/platform/default/filesource-files.json @@ -2,22 +2,19 @@ "//": "This file can be edited manually and is the canonical source.", "sources": [ "platform/default/src/mbgl/storage/asset_file_source.cpp", - "platform/default/src/mbgl/storage/database_file_source.cpp", - "platform/default/src/mbgl/storage/file_source_manager.cpp", + "platform/default/src/mbgl/storage/default_file_source.cpp", "platform/default/src/mbgl/storage/file_source_request.cpp", "platform/default/src/mbgl/storage/local_file_request.cpp", "platform/default/src/mbgl/storage/local_file_source.cpp", - "platform/default/src/mbgl/storage/main_resource_loader.cpp", "platform/default/src/mbgl/storage/offline.cpp", "platform/default/src/mbgl/storage/offline_database.cpp", "platform/default/src/mbgl/storage/offline_download.cpp", "platform/default/src/mbgl/storage/online_file_source.cpp" ], "public_headers": { - "mbgl/storage/offline_file_source.hpp": "include/mbgl/storage/database_file_source.hpp", + "mbgl/storage/default_file_source.hpp": "include/mbgl/storage/default_file_source.hpp", "mbgl/storage/offline.hpp": "include/mbgl/storage/offline.hpp", "mbgl/storage/online_file_source.hpp": "include/mbgl/storage/online_file_source.hpp", - "mbgl/storage/file_source_manager.hpp": "include/mbgl/storage/file_source_manager.hpp", "mbgl/storage/file_source_request.hpp": "platform/default/include/mbgl/storage/file_source_request.hpp", "mbgl/storage/local_file_request.hpp": "platform/default/include/mbgl/storage/local_file_request.hpp", "mbgl/storage/merge_sideloaded.hpp": "platform/default/include/mbgl/storage/merge_sideloaded.hpp", @@ -27,7 +24,6 @@ "mbgl/storage/sqlite3.hpp": "platform/default/include/mbgl/storage/sqlite3.hpp" }, "private_headers": { - "mbgl/storage/main_resource_loader.hpp": "src/mbgl/storage/main_resource_loader.hpp", "mbgl/storage/asset_file_source.hpp": "src/mbgl/storage/asset_file_source.hpp", "mbgl/storage/http_file_source.hpp": "src/mbgl/storage/http_file_source.hpp", "mbgl/storage/local_file_source.hpp": "src/mbgl/storage/local_file_source.hpp" diff --git a/platform/default/include/mbgl/storage/offline_download.hpp b/platform/default/include/mbgl/storage/offline_download.hpp index 3a5159470..53b42ae9d 100644 --- a/platform/default/include/mbgl/storage/offline_download.hpp +++ b/platform/default/include/mbgl/storage/offline_download.hpp @@ -1,8 +1,8 @@ #pragma once -#include <mbgl/storage/file_source.hpp> #include <mbgl/storage/offline.hpp> #include <mbgl/storage/resource.hpp> +#include <mbgl/storage/online_file_source.hpp> #include <list> #include <unordered_set> @@ -28,7 +28,7 @@ class Parser; */ class OfflineDownload { public: - OfflineDownload(int64_t id, OfflineRegionDefinition, OfflineDatabase& offline, FileSource& online); + OfflineDownload(int64_t id, OfflineRegionDefinition&&, OfflineDatabase& offline, OnlineFileSource& online); ~OfflineDownload(); void setObserver(std::unique_ptr<OfflineRegionObserver>); @@ -53,7 +53,7 @@ private: int64_t id; OfflineRegionDefinition definition; OfflineDatabase& offlineDatabase; - FileSource& onlineFileSource; + OnlineFileSource& onlineFileSource; OfflineRegionStatus status; std::unique_ptr<OfflineRegionObserver> observer; diff --git a/platform/default/src/mbgl/storage/asset_file_source.cpp b/platform/default/src/mbgl/storage/asset_file_source.cpp index 7abd609b1..b14d73045 100644 --- a/platform/default/src/mbgl/storage/asset_file_source.cpp +++ b/platform/default/src/mbgl/storage/asset_file_source.cpp @@ -1,17 +1,15 @@ #include <mbgl/storage/asset_file_source.hpp> #include <mbgl/storage/file_source_request.hpp> #include <mbgl/storage/local_file_request.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/storage/response.hpp> -#include <mbgl/util/constants.hpp> #include <mbgl/util/string.hpp> #include <mbgl/util/thread.hpp> #include <mbgl/util/url.hpp> namespace { -bool acceptsURL(const std::string& url) { - return 0 == url.rfind(mbgl::util::ASSET_PROTOCOL, 0); -} + +const std::string assetProtocol = "asset://"; + } // namespace namespace mbgl { @@ -32,8 +30,7 @@ public: } // Cut off the protocol and prefix with path. - const auto path = - root + "/" + mbgl::util::percentDecode(url.substr(std::char_traits<char>::length(util::ASSET_PROTOCOL))); + const auto path = root + "/" + mbgl::util::percentDecode(url.substr(assetProtocol.size())); requestLocalFile(path, std::move(req)); } @@ -55,16 +52,8 @@ std::unique_ptr<AsyncRequest> AssetFileSource::request(const Resource& resource, return std::move(req); } -bool AssetFileSource::canRequest(const Resource& resource) const { - return acceptsURL(resource.url); -} - -void AssetFileSource::pause() { - impl->pause(); -} - -void AssetFileSource::resume() { - impl->resume(); +bool AssetFileSource::acceptsURL(const std::string& url) { + return 0 == url.rfind(assetProtocol, 0); } } // namespace mbgl diff --git a/platform/default/src/mbgl/storage/database_file_source.cpp b/platform/default/src/mbgl/storage/database_file_source.cpp deleted file mode 100644 index b46693849..000000000 --- a/platform/default/src/mbgl/storage/database_file_source.cpp +++ /dev/null @@ -1,291 +0,0 @@ -#include <mbgl/storage/database_file_source.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/file_source_request.hpp> -#include <mbgl/storage/offline_database.hpp> -#include <mbgl/storage/offline_download.hpp> -#include <mbgl/storage/resource_options.hpp> -#include <mbgl/storage/response.hpp> -#include <mbgl/util/constants.hpp> -#include <mbgl/util/logging.hpp> -#include <mbgl/util/thread.hpp> - -namespace mbgl { - -// For testing use only -constexpr const char* READ_ONLY_MODE_KEY = "read-only-mode"; - -class DatabaseFileSourceThread { -public: - DatabaseFileSourceThread(std::shared_ptr<FileSource> onlineFileSource_, std::string cachePath) - : db(std::make_unique<OfflineDatabase>(cachePath)), onlineFileSource(onlineFileSource_) {} - - void request(const Resource& resource, ActorRef<FileSourceRequest> req) { - auto offlineResponse = db->get(resource); - if (!offlineResponse) { - offlineResponse.emplace(); - offlineResponse->noContent = true; - offlineResponse->error = - std::make_unique<Response::Error>(Response::Error::Reason::NotFound, "Not found in offline database"); - } else if (!offlineResponse->isUsable()) { - offlineResponse->error = - std::make_unique<Response::Error>(Response::Error::Reason::NotFound, "Cached resource is unusable"); - } - req.invoke(&FileSourceRequest::setResponse, *offlineResponse); - } - - void setDatabasePath(const std::string& path, std::function<void()> callback) { - db->changePath(path); - if (callback) { - callback(); - } - } - - void forward(const Resource& resource, const Response& response, std::function<void()> callback) { - db->put(resource, response); - if (callback) { - callback(); - } - } - - void resetDatabase(std::function<void(std::exception_ptr)> callback) { callback(db->resetDatabase()); } - - void packDatabase(std::function<void(std::exception_ptr)> callback) { callback(db->pack()); } - - void runPackDatabaseAutomatically(bool autopack) { db->runPackDatabaseAutomatically(autopack); } - - void put(const Resource& resource, const Response& response) { db->put(resource, response); } - - void invalidateAmbientCache(std::function<void(std::exception_ptr)> callback) { - callback(db->invalidateAmbientCache()); - } - - void clearAmbientCache(std::function<void(std::exception_ptr)> callback) { callback(db->clearAmbientCache()); } - - void setMaximumAmbientCacheSize(uint64_t size, std::function<void(std::exception_ptr)> callback) { - callback(db->setMaximumAmbientCacheSize(size)); - } - - void listRegions(std::function<void(expected<OfflineRegions, std::exception_ptr>)> callback) { - callback(db->listRegions()); - } - - void createRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata, - std::function<void(expected<OfflineRegion, std::exception_ptr>)> callback) { - callback(db->createRegion(definition, metadata)); - } - - void mergeOfflineRegions(const std::string& sideDatabasePath, - std::function<void(expected<OfflineRegions, std::exception_ptr>)> callback) { - callback(db->mergeDatabase(sideDatabasePath)); - } - - void updateMetadata(const int64_t regionID, - const OfflineRegionMetadata& metadata, - std::function<void(expected<OfflineRegionMetadata, std::exception_ptr>)> callback) { - callback(db->updateMetadata(regionID, metadata)); - } - - void getRegionStatus(int64_t regionID, - std::function<void(expected<OfflineRegionStatus, std::exception_ptr>)> callback) { - if (auto download = getDownload(regionID)) { - callback(download.value()->getStatus()); - } else { - callback(unexpected<std::exception_ptr>(download.error())); - } - } - - void deleteRegion(OfflineRegion region, std::function<void(std::exception_ptr)> callback) { - downloads.erase(region.getID()); - callback(db->deleteRegion(std::move(region))); - } - - void invalidateRegion(int64_t regionID, std::function<void(std::exception_ptr)> callback) { - callback(db->invalidateRegion(regionID)); - } - - void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) { - if (auto download = getDownload(regionID)) { - download.value()->setObserver(std::move(observer)); - } - } - - void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { - if (auto download = getDownload(regionID)) { - download.value()->setState(state); - } - } - - void setOfflineMapboxTileCountLimit(uint64_t limit) { db->setOfflineMapboxTileCountLimit(limit); } - - void reopenDatabaseReadOnlyForTesting() { db->reopenDatabaseReadOnlyForTesting(); } - -private: - expected<OfflineDownload*, std::exception_ptr> getDownload(int64_t regionID) { - if (!onlineFileSource) { - return unexpected<std::exception_ptr>( - std::make_exception_ptr(std::runtime_error("Network file source unavailable."))); - } - - auto it = downloads.find(regionID); - if (it != downloads.end()) { - return it->second.get(); - } - auto definition = db->getRegionDefinition(regionID); - if (!definition) { - return unexpected<std::exception_ptr>(definition.error()); - } - auto download = - std::make_unique<OfflineDownload>(regionID, std::move(definition.value()), *db, *onlineFileSource); - return downloads.emplace(regionID, std::move(download)).first->second.get(); - } - - std::unique_ptr<OfflineDatabase> db; - std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads; - std::shared_ptr<FileSource> onlineFileSource; -}; - -class DatabaseFileSource::Impl { -public: - Impl(std::shared_ptr<FileSource> onlineFileSource, const std::string& cachePath) - : thread(std::make_unique<util::Thread<DatabaseFileSourceThread>>( - "DatabaseFileSource", std::move(onlineFileSource), cachePath)) {} - - ActorRef<DatabaseFileSourceThread> actor() const { return thread->actor(); } - - void pause() { thread->pause(); } - void resume() { thread->resume(); } - -private: - const std::unique_ptr<util::Thread<DatabaseFileSourceThread>> thread; -}; - -DatabaseFileSource::DatabaseFileSource(const ResourceOptions& options) - : impl(std::make_unique<Impl>(FileSourceManager::get()->getFileSource(FileSourceType::Network, options), - options.cachePath())) {} - -DatabaseFileSource::~DatabaseFileSource() = default; - -std::unique_ptr<AsyncRequest> DatabaseFileSource::request(const Resource& resource, Callback callback) { - auto req = std::make_unique<FileSourceRequest>(std::move(callback)); - impl->actor().invoke(&DatabaseFileSourceThread::request, resource, req->actor()); - return std::move(req); -} - -void DatabaseFileSource::forward(const Resource& res, const Response& response, std::function<void()> callback) { - std::function<void()> wrapper; - if (callback) { - wrapper = Scheduler::GetCurrent()->bindOnce(std::move(callback)); - } - impl->actor().invoke(&DatabaseFileSourceThread::forward, res, response, std::move(wrapper)); -} - -bool DatabaseFileSource::canRequest(const Resource& resource) const { - return resource.hasLoadingMethod(Resource::LoadingMethod::Cache) && - resource.url.rfind(mbgl::util::ASSET_PROTOCOL, 0) == std::string::npos && - resource.url.rfind(mbgl::util::FILE_PROTOCOL, 0) == std::string::npos; -} - -void DatabaseFileSource::setDatabasePath(const std::string& path, std::function<void()> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::setDatabasePath, path, std::move(callback)); -} - -void DatabaseFileSource::resetDatabase(std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::resetDatabase, std::move(callback)); -} - -void DatabaseFileSource::packDatabase(std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::packDatabase, std::move(callback)); -} - -void DatabaseFileSource::runPackDatabaseAutomatically(bool autopack) { - impl->actor().invoke(&DatabaseFileSourceThread::runPackDatabaseAutomatically, autopack); -} - -void DatabaseFileSource::put(const Resource& resource, const Response& response) { - impl->actor().invoke(&DatabaseFileSourceThread::put, resource, response); -} - -void DatabaseFileSource::invalidateAmbientCache(std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::invalidateAmbientCache, std::move(callback)); -} - -void DatabaseFileSource::clearAmbientCache(std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::clearAmbientCache, std::move(callback)); -} - -void DatabaseFileSource::setMaximumAmbientCacheSize(uint64_t size, std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::setMaximumAmbientCacheSize, size, std::move(callback)); -} - -void DatabaseFileSource::listOfflineRegions( - std::function<void(expected<OfflineRegions, std::exception_ptr>)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::listRegions, callback); -} - -void DatabaseFileSource::createOfflineRegion( - const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata, - std::function<void(expected<OfflineRegion, std::exception_ptr>)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::createRegion, definition, metadata, callback); -} - -void DatabaseFileSource::mergeOfflineRegions( - const std::string& sideDatabasePath, std::function<void(expected<OfflineRegions, std::exception_ptr>)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::mergeOfflineRegions, sideDatabasePath, callback); -} - -void DatabaseFileSource::updateOfflineMetadata( - const int64_t regionID, - const OfflineRegionMetadata& metadata, - std::function<void(expected<OfflineRegionMetadata, std::exception_ptr>)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::updateMetadata, regionID, metadata, callback); -} - -void DatabaseFileSource::deleteOfflineRegion(OfflineRegion region, std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::deleteRegion, std::move(region), callback); -} - -void DatabaseFileSource::invalidateOfflineRegion(OfflineRegion& region, - std::function<void(std::exception_ptr)> callback) { - impl->actor().invoke(&DatabaseFileSourceThread::invalidateRegion, region.getID(), callback); -} - -void DatabaseFileSource::setOfflineRegionObserver(OfflineRegion& region, - std::unique_ptr<OfflineRegionObserver> observer) { - impl->actor().invoke(&DatabaseFileSourceThread::setRegionObserver, region.getID(), std::move(observer)); -} - -void DatabaseFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { - impl->actor().invoke(&DatabaseFileSourceThread::setRegionDownloadState, region.getID(), state); -} - -void DatabaseFileSource::getOfflineRegionStatus( - OfflineRegion& region, std::function<void(expected<OfflineRegionStatus, std::exception_ptr>)> callback) const { - impl->actor().invoke(&DatabaseFileSourceThread::getRegionStatus, region.getID(), callback); -} - -void DatabaseFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const { - impl->actor().invoke(&DatabaseFileSourceThread::setOfflineMapboxTileCountLimit, limit); -} - -void DatabaseFileSource::setProperty(const std::string& key, const mapbox::base::Value& value) { - if (key == READ_ONLY_MODE_KEY && value.getBool()) { - if (*value.getBool()) { - impl->actor().invoke(&DatabaseFileSourceThread::reopenDatabaseReadOnlyForTesting); - } - } else { - std::string message = "Resource provider does not support property " + key; - Log::Error(Event::General, message.c_str()); - } -} - -void DatabaseFileSource::pause() { - impl->pause(); -} - -void DatabaseFileSource::resume() { - impl->resume(); -} - -} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/default_file_source.cpp b/platform/default/src/mbgl/storage/default_file_source.cpp new file mode 100644 index 000000000..2d96a5a9a --- /dev/null +++ b/platform/default/src/mbgl/storage/default_file_source.cpp @@ -0,0 +1,398 @@ +#include <mbgl/storage/default_file_source.hpp> +#include <mbgl/storage/asset_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/local_file_source.hpp> +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/resource_transform.hpp> + +#include <mbgl/util/platform.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/work_request.hpp> +#include <mbgl/util/stopwatch.hpp> + +#include <cassert> +#include <utility> + +namespace mbgl { + +class DefaultFileSource::Impl { +public: + Impl(std::shared_ptr<FileSource> assetFileSource_, std::string cachePath) + : assetFileSource(std::move(assetFileSource_)) + , localFileSource(std::make_unique<LocalFileSource>()) + , offlineDatabase(std::make_unique<OfflineDatabase>(std::move(cachePath))) { + } + + void setAPIBaseURL(const std::string& url) { + onlineFileSource.setAPIBaseURL(url); + } + + std::string getAPIBaseURL() const{ + return onlineFileSource.getAPIBaseURL(); + } + + void setAccessToken(const std::string& accessToken) { + onlineFileSource.setAccessToken(accessToken); + } + + std::string getAccessToken() const { + return onlineFileSource.getAccessToken(); + } + + void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + onlineFileSource.setResourceTransform(std::move(transform)); + } + + void setResourceCachePath(const std::string& path, optional<ActorRef<PathChangeCallback>>&& callback) { + offlineDatabase->changePath(path); + if (callback) { + callback->invoke(&PathChangeCallback::operator()); + } + } + + void listRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + callback(offlineDatabase->listRegions()); + } + + void createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) { + callback(offlineDatabase->createRegion(definition, metadata)); + } + + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + callback(offlineDatabase->mergeDatabase(sideDatabasePath)); + } + + void updateMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) { + callback(offlineDatabase->updateMetadata(regionID, metadata)); + } + + void getRegionStatus(int64_t regionID, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) { + if (auto download = getDownload(regionID)) { + callback(download.value()->getStatus()); + } else { + callback(unexpected<std::exception_ptr>(download.error())); + } + } + + void deleteRegion(OfflineRegion&& region, std::function<void(std::exception_ptr)> callback) { + downloads.erase(region.getID()); + callback(offlineDatabase->deleteRegion(std::move(region))); + } + + void invalidateRegion(int64_t regionID, std::function<void (std::exception_ptr)> callback) { + callback(offlineDatabase->invalidateRegion(regionID)); + } + + void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) { + if (auto download = getDownload(regionID)) { + download.value()->setObserver(std::move(observer)); + } + } + + void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { + if (auto download = getDownload(regionID)) { + download.value()->setState(state); + } + } + + void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) { + auto callback = [ref] (const Response& res) { + ref.invoke(&FileSourceRequest::setResponse, res); + }; + + if (AssetFileSource::acceptsURL(resource.url)) { + //Asset request + tasks[req] = assetFileSource->request(resource, callback); + } else if (LocalFileSource::acceptsURL(resource.url)) { + //Local file request + tasks[req] = localFileSource->request(resource, callback); + } else { + // Try the offline database + if (resource.hasLoadingMethod(Resource::LoadingMethod::Cache)) { + auto offlineResponse = offlineDatabase->get(resource); + + if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) { + if (!offlineResponse) { + // Ensure there's always a response that we can send, so the caller knows that + // there's no optional data available in the cache, when it's the only place + // we're supposed to load from. + offlineResponse.emplace(); + offlineResponse->noContent = true; + offlineResponse->error = std::make_unique<Response::Error>( + Response::Error::Reason::NotFound, "Not found in offline database"); + } else if (!offlineResponse->isUsable()) { + // Don't return resources the server requested not to show when they're stale. + // Even if we can't directly use the response, we may still use it to send a + // conditional HTTP request, which is why we're saving it above. + offlineResponse->error = std::make_unique<Response::Error>( + Response::Error::Reason::NotFound, "Cached resource is unusable"); + } + callback(*offlineResponse); + } else if (offlineResponse) { + // Copy over the fields so that we can use them when making a refresh request. + resource.priorModified = offlineResponse->modified; + resource.priorExpires = offlineResponse->expires; + resource.priorEtag = offlineResponse->etag; + resource.priorData = offlineResponse->data; + + if (offlineResponse->isUsable()) { + callback(*offlineResponse); + // Set the priority of existing resource to low if it's expired but usable. + resource.setPriority(Resource::Priority::Low); + } + } + } + + // Get from the online file source + if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) { + MBGL_TIMING_START(watch); + tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) { + this->offlineDatabase->put(resource, onlineResponse); + if (resource.kind == Resource::Kind::Tile) { + // onlineResponse.data will be null if data not modified + MBGL_TIMING_FINISH(watch, + " Action: " << "Requesting," << + " URL: " << resource.url.c_str() << + " Size: " << (onlineResponse.data != nullptr ? onlineResponse.data->size() : 0) << "B," << + " Time") + } + callback(onlineResponse); + }); + } + } + } + + void cancel(AsyncRequest* req) { + tasks.erase(req); + } + + void setOfflineMapboxTileCountLimit(uint64_t limit) { + offlineDatabase->setOfflineMapboxTileCountLimit(limit); + } + + void setOnlineStatus(const bool status) { + onlineFileSource.setOnlineStatus(status); + } + + void reopenDatabaseReadOnlyForTesting() { offlineDatabase->reopenDatabaseReadOnlyForTesting(); } + + void setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + onlineFileSource.setMaximumConcurrentRequests(maximumConcurrentRequests_); + } + + void put(const Resource& resource, const Response& response) { + offlineDatabase->put(resource, response); + } + + void resetDatabase(std::function<void (std::exception_ptr)> callback) { + callback(offlineDatabase->resetDatabase()); + } + + void invalidateAmbientCache(std::function<void (std::exception_ptr)> callback) { + callback(offlineDatabase->invalidateAmbientCache()); + } + + void clearAmbientCache(std::function<void (std::exception_ptr)> callback) { + callback(offlineDatabase->clearAmbientCache()); + } + + void setMaximumAmbientCacheSize(uint64_t size, std::function<void (std::exception_ptr)> callback) { + callback(offlineDatabase->setMaximumAmbientCacheSize(size)); + } + + void packDatabase(std::function<void(std::exception_ptr)> callback) { callback(offlineDatabase->pack()); } + + void runPackDatabaseAutomatically(bool autopack) { offlineDatabase->runPackDatabaseAutomatically(autopack); } + +private: + expected<OfflineDownload*, std::exception_ptr> getDownload(int64_t regionID) { + auto it = downloads.find(regionID); + if (it != downloads.end()) { + return it->second.get(); + } + auto definition = offlineDatabase->getRegionDefinition(regionID); + if (!definition) { + return unexpected<std::exception_ptr>(definition.error()); + } + auto download = std::make_unique<OfflineDownload>(regionID, std::move(definition.value()), + *offlineDatabase, onlineFileSource); + return downloads.emplace(regionID, std::move(download)).first->second.get(); + } + + // shared so that destruction is done on the creating thread + const std::shared_ptr<FileSource> assetFileSource; + const std::unique_ptr<FileSource> localFileSource; + std::unique_ptr<OfflineDatabase> offlineDatabase; + OnlineFileSource onlineFileSource; + std::unordered_map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks; + std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads; +}; + +DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests_) + : DefaultFileSource(cachePath, std::make_unique<AssetFileSource>(assetPath), supportCacheOnlyRequests_) { +} + +DefaultFileSource::DefaultFileSource(const std::string& cachePath, std::unique_ptr<FileSource>&& assetFileSource_, bool supportCacheOnlyRequests_) + : assetFileSource(std::move(assetFileSource_)) + , impl(std::make_unique<util::Thread<Impl>>("DefaultFileSource", assetFileSource, cachePath)) + , supportCacheOnlyRequests(supportCacheOnlyRequests_) { +} + +DefaultFileSource::~DefaultFileSource() = default; + +bool DefaultFileSource::supportsCacheOnlyRequests() const { + return supportCacheOnlyRequests; +} + +void DefaultFileSource::setAPIBaseURL(const std::string& baseURL) { + impl->actor().invoke(&Impl::setAPIBaseURL, baseURL); + + { + std::lock_guard<std::mutex> lock(cachedBaseURLMutex); + cachedBaseURL = baseURL; + } +} + +std::string DefaultFileSource::getAPIBaseURL() { + std::lock_guard<std::mutex> lock(cachedBaseURLMutex); + return cachedBaseURL; +} + +void DefaultFileSource::setAccessToken(const std::string& accessToken) { + impl->actor().invoke(&Impl::setAccessToken, accessToken); + + { + std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); + cachedAccessToken = accessToken; + } +} + +std::string DefaultFileSource::getAccessToken() { + std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); + return cachedAccessToken; +} + +void DefaultFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + impl->actor().invoke(&Impl::setResourceTransform, std::move(transform)); +} + +void DefaultFileSource::setResourceCachePath(const std::string& path, optional<ActorRef<PathChangeCallback>>&& callback) { + impl->actor().invoke(&Impl::setResourceCachePath, path, std::move(callback)); +} + +std::unique_ptr<AsyncRequest> DefaultFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + req->onCancel([fs = impl->actor(), req = req.get()] () { fs.invoke(&Impl::cancel, req); }); + + impl->actor().invoke(&Impl::request, req.get(), resource, req->actor()); + + return std::move(req); +} + +void DefaultFileSource::listOfflineRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::listRegions, callback); +} + +void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::createRegion, definition, metadata, callback); +} + +void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback); +} + +void DefaultFileSource::updateOfflineMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, + std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback); +} + +void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function<void(std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::deleteRegion, std::move(region), callback); +} + +void DefaultFileSource::invalidateOfflineRegion(OfflineRegion& region, + std::function<void(std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::invalidateRegion, region.getID(), callback); +} + +void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr<OfflineRegionObserver> observer) { + impl->actor().invoke(&Impl::setRegionObserver, region.getID(), std::move(observer)); +} + +void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { + impl->actor().invoke(&Impl::setRegionDownloadState, region.getID(), state); +} + +void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) const { + impl->actor().invoke(&Impl::getRegionStatus, region.getID(), callback); +} + +void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const { + impl->actor().invoke(&Impl::setOfflineMapboxTileCountLimit, limit); +} + +void DefaultFileSource::pause() { + impl->pause(); +} + +void DefaultFileSource::resume() { + impl->resume(); +} + +void DefaultFileSource::put(const Resource& resource, const Response& response) { + impl->actor().invoke(&Impl::put, resource, response); +} + +void DefaultFileSource::resetDatabase(std::function<void (std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::resetDatabase, std::move(callback)); +} + +void DefaultFileSource::packDatabase(std::function<void(std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::packDatabase, std::move(callback)); +} + +void DefaultFileSource::runPackDatabaseAutomatically(bool autopack) { + impl->actor().invoke(&Impl::runPackDatabaseAutomatically, autopack); +} + +void DefaultFileSource::invalidateAmbientCache(std::function<void (std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::invalidateAmbientCache, std::move(callback)); +} + +void DefaultFileSource::clearAmbientCache(std::function<void(std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::clearAmbientCache, std::move(callback)); +} + +void DefaultFileSource::setMaximumAmbientCacheSize(uint64_t size, std::function<void (std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::setMaximumAmbientCacheSize, size, std::move(callback)); +} + +// For testing only: + +void DefaultFileSource::setOnlineStatus(const bool status) { + impl->actor().invoke(&Impl::setOnlineStatus, status); +} + +void DefaultFileSource::reopenDatabaseReadOnlyForTesting() { + impl->actor().invoke(&Impl::reopenDatabaseReadOnlyForTesting); +} + +void DefaultFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + impl->actor().invoke(&Impl::setMaximumConcurrentRequests, maximumConcurrentRequests_); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source.cpp b/platform/default/src/mbgl/storage/file_source.cpp new file mode 100644 index 000000000..4e800cc8f --- /dev/null +++ b/platform/default/src/mbgl/storage/file_source.cpp @@ -0,0 +1,15 @@ +#include <mbgl/storage/resource_options.hpp> +#include <mbgl/storage/default_file_source.hpp> + +#include <memory> + +namespace mbgl { + +std::shared_ptr<FileSource> FileSource::createPlatformFileSource(const ResourceOptions& options) { + auto fileSource = std::make_shared<DefaultFileSource>(options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests()); + fileSource->setAccessToken(options.accessToken()); + fileSource->setAPIBaseURL(options.baseURL()); + return fileSource; +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source_manager.cpp b/platform/default/src/mbgl/storage/file_source_manager.cpp deleted file mode 100644 index 2981096da..000000000 --- a/platform/default/src/mbgl/storage/file_source_manager.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include <mbgl/storage/asset_file_source.hpp> -#include <mbgl/storage/database_file_source.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/local_file_source.hpp> -#include <mbgl/storage/main_resource_loader.hpp> -#include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/resource_options.hpp> - -namespace mbgl { - -class DefaultFileSourceManagerImpl final : public FileSourceManager { -public: - DefaultFileSourceManagerImpl() { - registerFileSourceFactory(FileSourceType::ResourceLoader, [](const ResourceOptions& options) { - return std::make_unique<MainResourceLoader>(options); - }); - - registerFileSourceFactory(FileSourceType::Asset, [](const ResourceOptions& options) { - return std::make_unique<AssetFileSource>(options.assetPath()); - }); - - registerFileSourceFactory(FileSourceType::Database, [](const ResourceOptions& options) { - return std::make_unique<DatabaseFileSource>(options); - }); - - registerFileSourceFactory(FileSourceType::FileSystem, - [](const ResourceOptions&) { return std::make_unique<LocalFileSource>(); }); - - registerFileSourceFactory(FileSourceType::Network, [](const ResourceOptions& options) { - auto networkSource = std::make_unique<OnlineFileSource>(); - networkSource->setProperty(ACCESS_TOKEN_KEY, options.accessToken()); - networkSource->setProperty(API_BASE_URL_KEY, options.baseURL()); - return networkSource; - }); - } -}; - -FileSourceManager* FileSourceManager::get() noexcept { - static DefaultFileSourceManagerImpl instance; - return &instance; -} - -} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp index 54f12baf7..ca2eedc7b 100644 --- a/platform/default/src/mbgl/storage/local_file_source.cpp +++ b/platform/default/src/mbgl/storage/local_file_source.cpp @@ -1,17 +1,15 @@ +#include <mbgl/storage/local_file_source.hpp> #include <mbgl/storage/file_source_request.hpp> #include <mbgl/storage/local_file_request.hpp> -#include <mbgl/storage/local_file_source.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/storage/response.hpp> -#include <mbgl/util/constants.hpp> #include <mbgl/util/string.hpp> #include <mbgl/util/thread.hpp> #include <mbgl/util/url.hpp> namespace { -bool acceptsURL(const std::string& url) { - return 0 == url.rfind(mbgl::util::FILE_PROTOCOL, 0); -} + +const std::string fileProtocol = "file://"; + } // namespace namespace mbgl { @@ -30,7 +28,7 @@ public: } // Cut off the protocol and prefix with path. - const auto path = mbgl::util::percentDecode(url.substr(std::char_traits<char>::length(util::FILE_PROTOCOL))); + const auto path = mbgl::util::percentDecode(url.substr(fileProtocol.size())); requestLocalFile(path, std::move(req)); } }; @@ -49,16 +47,8 @@ std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, return std::move(req); } -bool LocalFileSource::canRequest(const Resource& resource) const { - return acceptsURL(resource.url); -} - -void LocalFileSource::pause() { - impl->pause(); -} - -void LocalFileSource::resume() { - impl->resume(); +bool LocalFileSource::acceptsURL(const std::string& url) { + return 0 == url.rfind(fileProtocol, 0); } } // namespace mbgl diff --git a/platform/default/src/mbgl/storage/main_resource_loader.cpp b/platform/default/src/mbgl/storage/main_resource_loader.cpp deleted file mode 100644 index 91ae0a1d7..000000000 --- a/platform/default/src/mbgl/storage/main_resource_loader.cpp +++ /dev/null @@ -1,196 +0,0 @@ -#include <mbgl/actor/actor.hpp> -#include <mbgl/actor/scheduler.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/file_source_request.hpp> -#include <mbgl/storage/main_resource_loader.hpp> -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/resource_options.hpp> -#include <mbgl/util/stopwatch.hpp> -#include <mbgl/util/thread.hpp> - -#include <cassert> - -namespace mbgl { - -class MainResourceLoaderThread { -public: - MainResourceLoaderThread(std::shared_ptr<FileSource> assetFileSource_, - std::shared_ptr<FileSource> databaseFileSource_, - std::shared_ptr<FileSource> localFileSource_, - std::shared_ptr<FileSource> onlineFileSource_) - : assetFileSource(std::move(assetFileSource_)), - databaseFileSource(std::move(databaseFileSource_)), - localFileSource(std::move(localFileSource_)), - onlineFileSource(std::move(onlineFileSource_)) {} - - void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) { - auto callback = [ref](const Response& res) { ref.invoke(&FileSourceRequest::setResponse, res); }; - - auto requestFromNetwork = [=](const Resource& res, - std::unique_ptr<AsyncRequest> parent) -> std::unique_ptr<AsyncRequest> { - if (!onlineFileSource || !onlineFileSource->canRequest(resource)) { - return parent; - } - - // Keep parent request alive while chained request is being processed. - std::shared_ptr<AsyncRequest> parentKeepAlive = std::move(parent); - - MBGL_TIMING_START(watch); - return onlineFileSource->request(res, [=, ptr = parentKeepAlive](Response response) { - if (databaseFileSource) { - databaseFileSource->forward(res, response); - } - if (res.kind == Resource::Kind::Tile) { - // onlineResponse.data will be null if data not modified - MBGL_TIMING_FINISH(watch, - " Action: " - << "Requesting," - << " URL: " << res.url.c_str() << " Size: " - << (response.data != nullptr ? response.data->size() : 0) << "B," - << " Time") - } - callback(response); - }); - }; - - // Initial tasksSize is used to check whether any of - // the sources were able to request a resource. - const std::size_t tasksSize = tasks.size(); - - // Waterfall resource request processing and return early once resource was requested. - if (assetFileSource && assetFileSource->canRequest(resource)) { - // Asset request - tasks[req] = assetFileSource->request(resource, callback); - } else if (localFileSource && localFileSource->canRequest(resource)) { - // Local file request - tasks[req] = localFileSource->request(resource, callback); - } else if (databaseFileSource && databaseFileSource->canRequest(resource)) { - // Try cache only request if needed. - if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) { - tasks[req] = databaseFileSource->request(resource, callback); - } else { - // Cache request with fallback to network with cache control - tasks[req] = databaseFileSource->request(resource, [=](Response response) { - Resource res = resource; - - // Resource is in the cache - if (!response.noContent) { - if (response.isUsable()) { - callback(response); - // Set the priority of existing resource to low if it's expired but usable. - res.setPriority(Resource::Priority::Low); - } - - // Copy response fields for cache control request - res.priorModified = response.modified; - res.priorExpires = response.expires; - res.priorEtag = response.etag; - res.priorData = response.data; - } - - tasks[req] = requestFromNetwork(res, std::move(tasks[req])); - }); - } - } else if (auto networkReq = requestFromNetwork(resource, nullptr)) { - // Get from the online file source - tasks[req] = std::move(networkReq); - } - - // If no new tasks were added, notify client that request cannot be processed. - if (tasks.size() == tasksSize) { - Response response; - response.noContent = true; - response.error = - std::make_unique<Response::Error>(Response::Error::Reason::Other, "Unsupported resource request."); - callback(response); - } - } - - void cancel(AsyncRequest* req) { - assert(req); - tasks.erase(req); - } - -private: - const std::shared_ptr<FileSource> assetFileSource; - const std::shared_ptr<FileSource> databaseFileSource; - const std::shared_ptr<FileSource> localFileSource; - const std::shared_ptr<FileSource> onlineFileSource; - std::unordered_map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks; -}; - -class MainResourceLoader::Impl { -public: - Impl(std::shared_ptr<FileSource> assetFileSource_, - std::shared_ptr<FileSource> databaseFileSource_, - std::shared_ptr<FileSource> localFileSource_, - std::shared_ptr<FileSource> onlineFileSource_) - : assetFileSource(std::move(assetFileSource_)), - databaseFileSource(std::move(databaseFileSource_)), - localFileSource(std::move(localFileSource_)), - onlineFileSource(std::move(onlineFileSource_)), - supportsCacheOnlyRequests_(bool(databaseFileSource)), - thread(std::make_unique<util::Thread<MainResourceLoaderThread>>( - "ResourceLoaderThread", assetFileSource, databaseFileSource, localFileSource, onlineFileSource)) {} - - std::unique_ptr<AsyncRequest> request(const Resource& resource, Callback callback) { - auto req = std::make_unique<FileSourceRequest>(std::move(callback)); - - req->onCancel([actorRef = thread->actor(), req = req.get()]() { - actorRef.invoke(&MainResourceLoaderThread::cancel, req); - }); - thread->actor().invoke(&MainResourceLoaderThread::request, req.get(), resource, req->actor()); - return std::move(req); - } - - bool canRequest(const Resource& resource) const { - return (assetFileSource && assetFileSource->canRequest(resource)) || - (localFileSource && localFileSource->canRequest(resource)) || - (databaseFileSource && databaseFileSource->canRequest(resource)) || - (onlineFileSource && onlineFileSource->canRequest(resource)); - } - - bool supportsCacheOnlyRequests() const { return supportsCacheOnlyRequests_; } - - void pause() { thread->pause(); } - - void resume() { thread->resume(); } - -private: - const std::shared_ptr<FileSource> assetFileSource; - const std::shared_ptr<FileSource> databaseFileSource; - const std::shared_ptr<FileSource> localFileSource; - const std::shared_ptr<FileSource> onlineFileSource; - const bool supportsCacheOnlyRequests_; - const std::unique_ptr<util::Thread<MainResourceLoaderThread>> thread; -}; - -MainResourceLoader::MainResourceLoader(const ResourceOptions& options) - : impl(std::make_unique<Impl>(FileSourceManager::get()->getFileSource(FileSourceType::Asset, options), - FileSourceManager::get()->getFileSource(FileSourceType::Database, options), - FileSourceManager::get()->getFileSource(FileSourceType::FileSystem, options), - FileSourceManager::get()->getFileSource(FileSourceType::Network, options))) {} - -MainResourceLoader::~MainResourceLoader() = default; - -bool MainResourceLoader::supportsCacheOnlyRequests() const { - return impl->supportsCacheOnlyRequests(); -} - -std::unique_ptr<AsyncRequest> MainResourceLoader::request(const Resource& resource, Callback callback) { - return impl->request(resource, std::move(callback)); -} - -bool MainResourceLoader::canRequest(const Resource& resource) const { - return impl->canRequest(resource); -} - -void MainResourceLoader::pause() { - impl->pause(); -} - -void MainResourceLoader::resume() { - impl->resume(); -} - -} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline_download.cpp b/platform/default/src/mbgl/storage/offline_download.cpp index 32fcb4f62..98eb1d388 100644 --- a/platform/default/src/mbgl/storage/offline_download.cpp +++ b/platform/default/src/mbgl/storage/offline_download.cpp @@ -1,3 +1,4 @@ +#include <mbgl/storage/online_file_source.hpp> #include <mbgl/storage/offline_database.hpp> #include <mbgl/storage/offline_download.hpp> #include <mbgl/storage/resource.hpp> @@ -89,11 +90,11 @@ uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType // OfflineDownload OfflineDownload::OfflineDownload(int64_t id_, - OfflineRegionDefinition definition_, + OfflineRegionDefinition&& definition_, OfflineDatabase& offlineDatabase_, - FileSource& onlineFileSource_) + OnlineFileSource& onlineFileSource_) : id(id_), - definition(std::move(definition_)), + definition(definition_), offlineDatabase(offlineDatabase_), onlineFileSource(onlineFileSource_) { setObserver(nullptr); @@ -368,13 +369,7 @@ void OfflineDownload::continueDownload() { if (resourcesToBeMarkedAsUsed.size() >= kMarkBatchSize) markPendingUsedResources(); - uint32_t maxConcurrentRequests = util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS; - auto value = onlineFileSource.getProperty("max-concurrent-requests"); - if (uint64_t* maxRequests = value.getUint()) { - maxConcurrentRequests = static_cast<uint32_t>(*maxRequests); - } - - while (!resourcesRemaining.empty() && requests.size() < maxConcurrentRequests) { + while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) { ensureResource(std::move(resourcesRemaining.front())); resourcesRemaining.pop_front(); } diff --git a/platform/default/src/mbgl/storage/online_file_source.cpp b/platform/default/src/mbgl/storage/online_file_source.cpp index 0f5b438e1..f4225bdf3 100644 --- a/platform/default/src/mbgl/storage/online_file_source.cpp +++ b/platform/default/src/mbgl/storage/online_file_source.cpp @@ -2,59 +2,52 @@ #include <mbgl/storage/http_file_source.hpp> #include <mbgl/storage/network_status.hpp> -#include <mbgl/storage/file_source_request.hpp> #include <mbgl/storage/resource_transform.hpp> #include <mbgl/storage/response.hpp> #include <mbgl/util/logging.hpp> #include <mbgl/actor/mailbox.hpp> -#include <mbgl/util/async_task.hpp> -#include <mbgl/util/chrono.hpp> #include <mbgl/util/constants.hpp> -#include <mbgl/util/exception.hpp> -#include <mbgl/util/http_timeout.hpp> #include <mbgl/util/mapbox.hpp> +#include <mbgl/util/exception.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/async_task.hpp> #include <mbgl/util/noncopyable.hpp> #include <mbgl/util/run_loop.hpp> -#include <mbgl/util/thread.hpp> #include <mbgl/util/timer.hpp> +#include <mbgl/util/http_timeout.hpp> #include <algorithm> #include <cassert> #include <list> -#include <map> -#include <unordered_map> #include <unordered_set> +#include <unordered_map> namespace mbgl { -// For testing only -constexpr const char* ONLINE_STATUS_KEY = "online-status"; - -class OnlineFileSourceThread; +static uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; -struct OnlineFileRequest { - using Callback = std::function<void(Response)>; +class OnlineFileRequest : public AsyncRequest { +public: + using Callback = std::function<void (Response)>; - OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSourceThread& impl_); - ~OnlineFileRequest(); + OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&); + ~OnlineFileRequest() override; void networkIsReachableAgain(); void schedule(); void schedule(optional<Timestamp> expires); void completed(Response); - void setTransformedURL(const std::string& url); + void setTransformedURL(const std::string&& url); ActorRef<OnlineFileRequest> actor(); - void onCancel(std::function<void()>); - OnlineFileSourceThread& impl; + OnlineFileSource::Impl& impl; Resource resource; std::unique_ptr<AsyncRequest> request; util::Timer timer; Callback callback; - std::function<void()> cancelCallback = nullptr; std::shared_ptr<Mailbox> mailbox; // Counts the number of times a response was already expired when received. We're using @@ -69,99 +62,101 @@ struct OnlineFileRequest { optional<Timestamp> retryAfter; }; -class OnlineFileSourceThread { +class OnlineFileSource::Impl { public: - OnlineFileSourceThread() { + Impl() { NetworkStatus::Subscribe(&reachability); - setMaximumConcurrentRequests(util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS); - } - - ~OnlineFileSourceThread() { NetworkStatus::Unsubscribe(&reachability); } - - void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) { - auto callback = [ref](const Response& res) { ref.invoke(&FileSourceRequest::setResponse, res); }; - tasks[req] = std::make_unique<OnlineFileRequest>(std::move(resource), std::move(callback), *this); + setMaximumConcurrentRequests(DEFAULT_MAXIMUM_CONCURRENT_REQUESTS); } - void cancel(AsyncRequest* req) { - auto it = tasks.find(req); - assert(it != tasks.end()); - remove(it->second.get()); - tasks.erase(it); + ~Impl() { + NetworkStatus::Unsubscribe(&reachability); } - void add(OnlineFileRequest* req) { - allRequests.insert(req); + void add(OnlineFileRequest* request) { + allRequests.insert(request); if (resourceTransform) { // Request the ResourceTransform actor a new url and replace the resource url with the // transformed one before proceeding to schedule the request. - resourceTransform.transform( - req->resource.kind, req->resource.url, [ref = req->actor()](const std::string& url) { - ref.invoke(&OnlineFileRequest::setTransformedURL, url); - }); + resourceTransform->invoke(&ResourceTransform::transform, + request->resource.kind, + std::move(request->resource.url), + [ref = request->actor()](const std::string&& url) { + ref.invoke(&OnlineFileRequest::setTransformedURL, url); + }); } else { - req->schedule(); + request->schedule(); } } - void remove(OnlineFileRequest* req) { - allRequests.erase(req); - if (activeRequests.erase(req)) { + void remove(OnlineFileRequest* request) { + allRequests.erase(request); + if (activeRequests.erase(request)) { activatePendingRequest(); } else { - pendingRequests.remove(req); + pendingRequests.remove(request); } } - void activateOrQueueRequest(OnlineFileRequest* req) { - assert(allRequests.find(req) != allRequests.end()); - assert(activeRequests.find(req) == activeRequests.end()); - assert(!req->request); + void activateOrQueueRequest(OnlineFileRequest* request) { + assert(allRequests.find(request) != allRequests.end()); + assert(activeRequests.find(request) == activeRequests.end()); + assert(!request->request); if (activeRequests.size() >= getMaximumConcurrentRequests()) { - queueRequest(req); + queueRequest(request); } else { - activateRequest(req); + activateRequest(request); } } - void queueRequest(OnlineFileRequest* req) { pendingRequests.insert(req); } + void queueRequest(OnlineFileRequest* request) { + pendingRequests.insert(request); + } - void activateRequest(OnlineFileRequest* req) { + void activateRequest(OnlineFileRequest* request) { auto callback = [=](Response response) { - activeRequests.erase(req); - req->request.reset(); - req->completed(response); + activeRequests.erase(request); + request->request.reset(); + request->completed(response); activatePendingRequest(); }; - activeRequests.insert(req); + activeRequests.insert(request); if (online) { - req->request = httpFileSource.request(req->resource, callback); + request->request = httpFileSource.request(request->resource, callback); } else { Response response; response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection, "Online connectivity is disabled."); callback(response); } + } void activatePendingRequest() { - auto req = pendingRequests.pop(); - if (req) { - activateRequest(*req); + auto request = pendingRequests.pop(); + + if (request) { + activateRequest(*request); } } - bool isPending(OnlineFileRequest* req) { return pendingRequests.contains(req); } + bool isPending(OnlineFileRequest* request) { + return pendingRequests.contains(request); + } - bool isActive(OnlineFileRequest* req) { return activeRequests.find(req) != activeRequests.end(); } + bool isActive(OnlineFileRequest* request) { + return activeRequests.find(request) != activeRequests.end(); + } - void setResourceTransform(ResourceTransform transform) { resourceTransform = std::move(transform); } + void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + resourceTransform = std::move(transform); + } - void setOnlineStatus(bool status) { + void setOnlineStatus(const bool status) { online = status; if (online) { networkIsReachableAgain(); @@ -176,27 +171,20 @@ public: maximumConcurrentRequests = maximumConcurrentRequests_; } - void setAPIBaseURL(const std::string& t) { apiBaseURL = t; } - std::string getAPIBaseURL() const { return apiBaseURL; } - - void setAccessToken(const std::string& t) { accessToken = t; } - std::string getAccessToken() const { return accessToken; } - private: - friend struct OnlineFileRequest; void networkIsReachableAgain() { // Notify regular priority requests. - for (auto& req : allRequests) { - if (req->resource.priority == Resource::Priority::Regular) { - req->networkIsReachableAgain(); + for (auto& request : allRequests) { + if (request->resource.priority == Resource::Priority::Regular) { + request->networkIsReachableAgain(); } } // Notify low priority requests. - for (auto& req : allRequests) { - if (req->resource.priority == Resource::Priority::Low) { - req->networkIsReachableAgain(); + for (auto& request : allRequests) { + if (request->resource.priority == Resource::Priority::Low) { + request->networkIsReachableAgain(); } } } @@ -243,6 +231,7 @@ private: } } + optional<OnlineFileRequest*> pop() { if (queue.empty()) { return optional<OnlineFileRequest*>(); @@ -263,7 +252,7 @@ private: }; - ResourceTransform resourceTransform; + optional<ActorRef<ResourceTransform>> resourceTransform; /** * The lifetime of a request is: @@ -285,99 +274,56 @@ private: bool online = true; uint32_t maximumConcurrentRequests; HTTPFileSource httpFileSource; - util::AsyncTask reachability{std::bind(&OnlineFileSourceThread::networkIsReachableAgain, this)}; - std::string accessToken; - std::string apiBaseURL = mbgl::util::API_BASE_URL; - std::map<AsyncRequest*, std::unique_ptr<OnlineFileRequest>> tasks; + util::AsyncTask reachability { std::bind(&Impl::networkIsReachableAgain, this) }; }; -class OnlineFileSource::Impl { -public: - Impl() : thread(std::make_unique<util::Thread<OnlineFileSourceThread>>("OnlineFileSource")) {} - - std::unique_ptr<AsyncRequest> request(Callback callback, Resource res) { - auto req = std::make_unique<FileSourceRequest>(std::move(callback)); - req->onCancel( - [actorRef = thread->actor(), req = req.get()]() { actorRef.invoke(&OnlineFileSourceThread::cancel, req); }); - thread->actor().invoke(&OnlineFileSourceThread::request, req.get(), std::move(res), req->actor()); - return std::move(req); - } - - void pause() { thread->pause(); } +OnlineFileSource::OnlineFileSource() + : impl(std::make_unique<Impl>()) { +} - void resume() { thread->resume(); } +OnlineFileSource::~OnlineFileSource() = default; - void setResourceTransform(ResourceTransform transform) { - thread->actor().invoke(&OnlineFileSourceThread::setResourceTransform, std::move(transform)); - } +std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) { + Resource res = resource; - void setOnlineStatus(bool status) { thread->actor().invoke(&OnlineFileSourceThread::setOnlineStatus, status); } + switch (resource.kind) { + case Resource::Kind::Unknown: + case Resource::Kind::Image: + break; - void setAPIBaseURL(const mapbox::base::Value& value) { - if (auto* baseURL = value.getString()) { - thread->actor().invoke(&OnlineFileSourceThread::setAPIBaseURL, *baseURL); - { - std::lock_guard<std::mutex> lock(cachedBaseURLMutex); - cachedBaseURL = *baseURL; - } - } else { - Log::Error(Event::General, "Invalid api-base-url property value type."); - } - } + case Resource::Kind::Style: + res.url = mbgl::util::mapbox::normalizeStyleURL(apiBaseURL, resource.url, accessToken); + break; - std::string getAPIBaseURL() const { - std::lock_guard<std::mutex> lock(cachedBaseURLMutex); - return cachedBaseURL; - } + case Resource::Kind::Source: + res.url = util::mapbox::normalizeSourceURL(apiBaseURL, resource.url, accessToken); + break; - void setMaximumConcurrentRequests(const mapbox::base::Value& value) { - if (auto* maximumConcurrentRequests = value.getUint()) { - assert(*maximumConcurrentRequests < std::numeric_limits<uint32_t>::max()); - const uint32_t maxConcurretnRequests = static_cast<uint32_t>(*maximumConcurrentRequests); - thread->actor().invoke(&OnlineFileSourceThread::setMaximumConcurrentRequests, maxConcurretnRequests); - { - std::lock_guard<std::mutex> lock(maximumConcurrentRequestsMutex); - cachedMaximumConcurrentRequests = maxConcurretnRequests; - } - } else { - Log::Error(Event::General, "Invalid max-concurrent-requests property value type."); - } - } + case Resource::Kind::Glyphs: + res.url = util::mapbox::normalizeGlyphsURL(apiBaseURL, resource.url, accessToken); + break; - uint32_t getMaximumConcurrentRequests() const { - std::lock_guard<std::mutex> lock(maximumConcurrentRequestsMutex); - return cachedMaximumConcurrentRequests; - } + case Resource::Kind::SpriteImage: + case Resource::Kind::SpriteJSON: + res.url = util::mapbox::normalizeSpriteURL(apiBaseURL, resource.url, accessToken); + break; - void setAccessToken(const mapbox::base::Value& value) { - if (auto* accessToken = value.getString()) { - thread->actor().invoke(&OnlineFileSourceThread::setAccessToken, *accessToken); - { - std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); - cachedAccessToken = *accessToken; - } - } else { - Log::Error(Event::General, "Invalid access-token property value type."); - } + case Resource::Kind::Tile: + res.url = util::mapbox::normalizeTileURL(apiBaseURL, resource.url, accessToken); + break; } - std::string getAccessToken() const { - std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); - return cachedAccessToken; - } + return std::make_unique<OnlineFileRequest>(std::move(res), std::move(callback), *impl); +} -private: - mutable std::mutex cachedAccessTokenMutex; - std::string cachedAccessToken; - mutable std::mutex cachedBaseURLMutex; - std::string cachedBaseURL = util::API_BASE_URL; - mutable std::mutex maximumConcurrentRequestsMutex; - uint32_t cachedMaximumConcurrentRequests = util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS; - const std::unique_ptr<util::Thread<OnlineFileSourceThread>> thread; -}; +void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + impl->setResourceTransform(std::move(transform)); +} -OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSourceThread& impl_) - : impl(impl_), resource(std::move(resource_)), callback(std::move(callback_)) { +OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_) + : impl(impl_), + resource(std::move(resource_)), + callback(std::move(callback_)) { impl.add(this); } @@ -391,12 +337,12 @@ void OnlineFileRequest::schedule() { } OnlineFileRequest::~OnlineFileRequest() { - if (mailbox) { - mailbox->close(); - } + impl.remove(this); } -Timestamp interpolateExpiration(const Timestamp& current, optional<Timestamp> prior, bool& expired) { +Timestamp interpolateExpiration(const Timestamp& current, + optional<Timestamp> prior, + bool& expired) { auto now = util::now(); if (current > now) { return current; @@ -437,8 +383,9 @@ void OnlineFileRequest::schedule(optional<Timestamp> expires) { // If we're not being asked for a forced refresh, calculate a timeout that depends on how many // consecutive errors we've encountered, and on the expiration time, if present. - Duration timeout = std::min(http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), - http::expirationTimeout(expires, expiredRequests)); + Duration timeout = std::min( + http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), + http::expirationTimeout(expires, expiredRequests)); if (timeout == Duration::max()) { return; @@ -522,7 +469,7 @@ void OnlineFileRequest::networkIsReachableAgain() { } } -void OnlineFileRequest::setTransformedURL(const std::string& url) { +void OnlineFileRequest::setTransformedURL(const std::string&& url) { resource.url = url; schedule(); } @@ -537,95 +484,19 @@ ActorRef<OnlineFileRequest> OnlineFileRequest::actor() { return ActorRef<OnlineFileRequest>(*this, mailbox); } -void OnlineFileRequest::onCancel(std::function<void()> callback_) { - cancelCallback = std::move(callback_); -} - -OnlineFileSource::OnlineFileSource() : impl(std::make_unique<Impl>()) {} - -OnlineFileSource::~OnlineFileSource() = default; - -std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) { - Resource res = resource; - - switch (resource.kind) { - case Resource::Kind::Unknown: - case Resource::Kind::Image: - break; - - case Resource::Kind::Style: - res.url = - mbgl::util::mapbox::normalizeStyleURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); - break; - - case Resource::Kind::Source: - res.url = util::mapbox::normalizeSourceURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); - break; - - case Resource::Kind::Glyphs: - res.url = util::mapbox::normalizeGlyphsURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); - break; - - case Resource::Kind::SpriteImage: - case Resource::Kind::SpriteJSON: - res.url = util::mapbox::normalizeSpriteURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); - break; - - case Resource::Kind::Tile: - res.url = util::mapbox::normalizeTileURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); - break; - } - - return impl->request(std::move(callback), std::move(res)); -} - -bool OnlineFileSource::canRequest(const Resource& resource) const { - return resource.hasLoadingMethod(Resource::LoadingMethod::Network) && - resource.url.rfind(mbgl::util::ASSET_PROTOCOL, 0) == std::string::npos && - resource.url.rfind(mbgl::util::FILE_PROTOCOL, 0) == std::string::npos; +void OnlineFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + impl->setMaximumConcurrentRequests(maximumConcurrentRequests_); } -void OnlineFileSource::pause() { - impl->pause(); +uint32_t OnlineFileSource::getMaximumConcurrentRequests() const { + return impl->getMaximumConcurrentRequests(); } -void OnlineFileSource::resume() { - impl->resume(); -} - -void OnlineFileSource::setProperty(const std::string& key, const mapbox::base::Value& value) { - if (key == ACCESS_TOKEN_KEY) { - impl->setAccessToken(value); - } else if (key == API_BASE_URL_KEY) { - impl->setAPIBaseURL(value); - } else if (key == MAX_CONCURRENT_REQUESTS_KEY) { - impl->setMaximumConcurrentRequests(value); - } else if (key == ONLINE_STATUS_KEY) { - // For testing only - if (auto* boolValue = value.getBool()) { - impl->setOnlineStatus(*boolValue); - } - } else { - std::string message = "Resource provider does not support property " + key; - Log::Error(Event::General, message.c_str()); - } -} -mapbox::base::Value OnlineFileSource::getProperty(const std::string& key) const { - if (key == ACCESS_TOKEN_KEY) { - return impl->getAccessToken(); - } else if (key == API_BASE_URL_KEY) { - return impl->getAPIBaseURL(); - } else if (key == MAX_CONCURRENT_REQUESTS_KEY) { - return impl->getMaximumConcurrentRequests(); - } - std::string message = "Resource provider does not support property " + key; - Log::Error(Event::General, message.c_str()); - return {}; -} +// For testing only: -void OnlineFileSource::setResourceTransform(ResourceTransform transform) { - impl->setResourceTransform(std::move(transform)); +void OnlineFileSource::setOnlineStatus(const bool status) { + impl->setOnlineStatus(status); } } // namespace mbgl diff --git a/platform/glfw/main.cpp b/platform/glfw/main.cpp index ded8ee3e1..8f134804f 100644 --- a/platform/glfw/main.cpp +++ b/platform/glfw/main.cpp @@ -3,14 +3,13 @@ #include "settings_json.hpp" #include <mbgl/gfx/backend.hpp> -#include <mbgl/renderer/renderer.hpp> -#include <mbgl/storage/database_file_source.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/style/style.hpp> #include <mbgl/util/default_styles.hpp> #include <mbgl/util/logging.hpp> #include <mbgl/util/platform.hpp> #include <mbgl/util/string.hpp> +#include <mbgl/storage/default_file_source.hpp> +#include <mbgl/style/style.hpp> +#include <mbgl/renderer/renderer.hpp> #include <args.hxx> @@ -107,16 +106,10 @@ int main(int argc, char *argv[]) { mbgl::ResourceOptions resourceOptions; resourceOptions.withCachePath(cacheDB).withAccessToken(token); - auto onlineFileSource = - mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::Network, resourceOptions); + auto fileSource = std::static_pointer_cast<mbgl::DefaultFileSource>(mbgl::FileSource::getSharedFileSource(resourceOptions)); if (!settings.online) { - if (onlineFileSource) { - onlineFileSource->setProperty("online-status", false); - mbgl::Log::Warning(mbgl::Event::Setup, "Application is offline. Press `O` to toggle online status."); - } else { - mbgl::Log::Warning(mbgl::Event::Setup, - "Network resource provider is not available, only local requests are supported."); - } + fileSource->setOnlineStatus(false); + mbgl::Log::Warning(mbgl::Event::Setup, "Application is offline. Press `O` to toggle online status."); } GLFWRendererFrontend rendererFrontend { std::make_unique<mbgl::Renderer>(view->getRendererBackend(), view->getPixelRatio()), *view }; @@ -139,14 +132,9 @@ int main(int argc, char *argv[]) { if (testDirValue) view->setTestDirectory(args::get(testDirValue)); - view->setOnlineStatusCallback([&settings, onlineFileSource]() { - if (!onlineFileSource) { - mbgl::Log::Warning(mbgl::Event::Setup, - "Cannot change online status. Network resource provider is not available."); - return; - } + view->setOnlineStatusCallback([&settings, fileSource]() { settings.online = !settings.online; - onlineFileSource->setProperty("online-status", settings.online); + fileSource->setOnlineStatus(settings.online); mbgl::Log::Info(mbgl::Event::Setup, "Application is %s. Press `O` to toggle online status.", settings.online ? "online" : "offline"); }); @@ -164,26 +152,20 @@ int main(int argc, char *argv[]) { mbgl::Log::Info(mbgl::Event::Setup, "Changed style to: %s", newStyle.name); }); - // Resource loader controls top-level request processing and can resume / pause all managed sources simultaneously. - auto resourceLoader = - mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::ResourceLoader, resourceOptions); - view->setPauseResumeCallback([resourceLoader]() { + view->setPauseResumeCallback([fileSource] () { static bool isPaused = false; if (isPaused) { - resourceLoader->resume(); + fileSource->resume(); } else { - resourceLoader->pause(); + fileSource->pause(); } isPaused = !isPaused; }); - // Database file source. - auto databaseFileSource = std::static_pointer_cast<mbgl::DatabaseFileSource>( - mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::Database, resourceOptions)); - view->setResetCacheCallback([databaseFileSource]() { - databaseFileSource->resetDatabase([](std::exception_ptr ex) { + view->setResetCacheCallback([fileSource] () { + fileSource->resetDatabase([](std::exception_ptr ex) { if (ex) { mbgl::Log::Error(mbgl::Event::Database, "Failed to reset cache:: %s", mbgl::util::toString(ex).c_str()); } diff --git a/platform/linux/filesource-files.json b/platform/linux/filesource-files.json index 669a4e612..448f5f861 100644 --- a/platform/linux/filesource-files.json +++ b/platform/linux/filesource-files.json @@ -1,6 +1,7 @@ { "//": "This file can be edited manually and is the canonical source.", "sources": [ + "platform/default/src/mbgl/storage/file_source.cpp", "platform/default/src/mbgl/storage/http_file_source.cpp", "platform/default/src/mbgl/storage/sqlite3.cpp" ], diff --git a/render-test/file_source.cpp b/render-test/file_source.cpp index f72bc08e3..4d6a800d1 100644 --- a/render-test/file_source.cpp +++ b/render-test/file_source.cpp @@ -1,11 +1,6 @@ -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/storage/resource_options.hpp> -#include <mbgl/util/async_request.hpp> #include <mbgl/util/logging.hpp> -#include <atomic> - #include "file_source.hpp" namespace mbgl { @@ -15,14 +10,8 @@ std::atomic_size_t transferredSize{0}; std::atomic_bool active{false}; std::atomic_bool offline{true}; -ProxyFileSource::ProxyFileSource(std::shared_ptr<FileSource> defaultResourceLoader_, const ResourceOptions& options) - : defaultResourceLoader(std::move(defaultResourceLoader_)) { - assert(defaultResourceLoader); - if (offline) { - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, options); - dbfs->setProperty("read-only-mode", true); - } -} +ProxyFileSource::ProxyFileSource(const std::string& cachePath, const std::string& assetPath) + : DefaultFileSource(cachePath, assetPath, false) {} ProxyFileSource::~ProxyFileSource() = default; @@ -54,7 +43,7 @@ std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, } } - return defaultResourceLoader->request(transformed, [=](Response response) { + return DefaultFileSource::request(transformed, [=](Response response) { if (transformed.loadingMethod == Resource::LoadingMethod::CacheOnly && response.noContent) { if (transformed.kind == Resource::Kind::Tile && transformed.tileData) { mbgl::Log::Info(mbgl::Event::Database, @@ -75,6 +64,19 @@ std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, }); } +std::shared_ptr<FileSource> FileSource::createPlatformFileSource(const ResourceOptions& options) { + auto fileSource = std::make_shared<ProxyFileSource>(options.cachePath(), options.assetPath()); + + fileSource->setAccessToken(options.accessToken()); + fileSource->setAPIBaseURL(options.baseURL()); + + if (offline) { + fileSource->reopenDatabaseReadOnlyForTesting(); + } + + return fileSource; +} + // static void ProxyFileSource::setOffline(bool status) { offline = status; diff --git a/render-test/file_source.hpp b/render-test/file_source.hpp index d0496ab8f..34ba739a2 100644 --- a/render-test/file_source.hpp +++ b/render-test/file_source.hpp @@ -1,18 +1,15 @@ #pragma once -#include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/default_file_source.hpp> namespace mbgl { -class ResourceOptions; - -class ProxyFileSource : public FileSource { +class ProxyFileSource : public DefaultFileSource { public: - ProxyFileSource(std::shared_ptr<FileSource>, const ResourceOptions&); + ProxyFileSource(const std::string& cachePath, const std::string& assetPath); ~ProxyFileSource(); - std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource&) const override { return true; } + std::unique_ptr<AsyncRequest> request(const Resource&, Callback); /** * @brief Flag to change the networking mode of the file source. @@ -42,9 +39,6 @@ public: * @return size_t */ static size_t getTransferredSize(); - -private: - std::shared_ptr<FileSource> defaultResourceLoader; }; } // namespace mbgl diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 8147852c4..484428976 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -8,7 +8,6 @@ #include <mbgl/map/map.hpp> #include <mbgl/renderer/renderer.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/style/conversion/filter.hpp> #include <mbgl/style/conversion/json.hpp> #include <mbgl/style/conversion/layer.hpp> diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 3f59b6df4..f67f1bfd7 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -2,7 +2,6 @@ #include <mbgl/map/map_observer.hpp> #include <mbgl/renderer/renderer.hpp> #include <mbgl/renderer/renderer_observer.hpp> -#include <mbgl/storage/file_source_manager.hpp> #include <mbgl/style/image.hpp> #include <mbgl/style/layer.hpp> #include <mbgl/style/light.hpp> @@ -87,26 +86,7 @@ std::string simpleDiff(const Value& result, const Value& expected) { } TestRunner::TestRunner(Manifest manifest_, UpdateResults updateResults_) - : manifest(std::move(manifest_)), updateResults(updateResults_) { - registerProxyFileSource(); -} - -void TestRunner::registerProxyFileSource() { - static std::once_flag registerProxyFlag; - std::call_once(registerProxyFlag, [] { - auto* fileSourceManager = mbgl::FileSourceManager::get(); - - auto resourceLoaderFactory = - fileSourceManager->unRegisterFileSourceFactory(mbgl::FileSourceType::ResourceLoader); - auto factory = [defaultFactory = std::move(resourceLoaderFactory)](const mbgl::ResourceOptions& options) { - assert(defaultFactory); - std::shared_ptr<FileSource> fileSource = defaultFactory(options); - return std::make_unique<ProxyFileSource>(std::move(fileSource), options); - }; - - fileSourceManager->registerFileSourceFactory(mbgl::FileSourceType::ResourceLoader, std::move(factory)); - }); -} + : manifest(std::move(manifest_)), updateResults(updateResults_) {} const Manifest& TestRunner::getManifest() const { return manifest; @@ -674,7 +654,7 @@ uint32_t getImageTileOffset(const std::set<uint32_t>& dims, uint32_t dim) { TestRunner::Impl::Impl(const TestMetadata& metadata, const mbgl::ResourceOptions& resourceOptions) : observer(std::make_unique<TestRunnerMapObserver>()), frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), - fileSource(mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::ResourceLoader, resourceOptions)), + fileSource(mbgl::FileSource::getSharedFileSource(resourceOptions)), map(frontend, *observer.get(), mbgl::MapOptions() diff --git a/render-test/runner.hpp b/render-test/runner.hpp index 72320dd34..e6027e335 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -53,7 +53,6 @@ private: TestMetadata&); void checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&); void checkProbingResults(TestMetadata&); - void registerProxyFileSource(); struct Impl { Impl(const TestMetadata&, const mbgl::ResourceOptions&); diff --git a/src/core-files.json b/src/core-files.json index fcc508fb3..58efba1ee 100644 --- a/src/core-files.json +++ b/src/core-files.json @@ -153,7 +153,7 @@ "src/mbgl/sprite/sprite_loader.cpp", "src/mbgl/sprite/sprite_loader_worker.cpp", "src/mbgl/sprite/sprite_parser.cpp", - "src/mbgl/storage/file_source_manager.cpp", + "src/mbgl/storage/file_source.cpp", "src/mbgl/storage/network_status.cpp", "src/mbgl/storage/resource.cpp", "src/mbgl/storage/resource_options.cpp", @@ -374,9 +374,8 @@ "mbgl/renderer/renderer_frontend.hpp": "include/mbgl/renderer/renderer_frontend.hpp", "mbgl/renderer/renderer_observer.hpp": "include/mbgl/renderer/renderer_observer.hpp", "mbgl/renderer/renderer_state.hpp": "include/mbgl/renderer/renderer_state.hpp", - "mbgl/storage/database_file_source.hpp": "include/mbgl/storage/database_file_source.hpp", + "mbgl/storage/default_file_source.hpp": "include/mbgl/storage/default_file_source.hpp", "mbgl/storage/file_source.hpp": "include/mbgl/storage/file_source.hpp", - "mbgl/storage/file_source_manager.hpp": "include/mbgl/storage/file_source_manager.hpp", "mbgl/storage/network_status.hpp": "include/mbgl/storage/network_status.hpp", "mbgl/storage/offline.hpp": "include/mbgl/storage/offline.hpp", "mbgl/storage/online_file_source.hpp": "include/mbgl/storage/online_file_source.hpp", @@ -696,7 +695,6 @@ "mbgl/storage/asset_file_source.hpp": "src/mbgl/storage/asset_file_source.hpp", "mbgl/storage/http_file_source.hpp": "src/mbgl/storage/http_file_source.hpp", "mbgl/storage/local_file_source.hpp": "src/mbgl/storage/local_file_source.hpp", - "mbgl/storage/main_resource_loader.hpp": "src/mbgl/storage/main_resource_loader.hpp", "mbgl/style/collection.hpp": "src/mbgl/style/collection.hpp", "mbgl/style/conversion/json.hpp": "src/mbgl/style/conversion/json.hpp", "mbgl/style/conversion/stringify.hpp": "src/mbgl/style/conversion/stringify.hpp", diff --git a/src/mbgl/map/map.cpp b/src/mbgl/map/map.cpp index 061669f56..a994af305 100644 --- a/src/mbgl/map/map.cpp +++ b/src/mbgl/map/map.cpp @@ -1,24 +1,24 @@ -#include <mbgl/annotation/annotation_manager.hpp> -#include <mbgl/layermanager/layer_manager.hpp> -#include <mbgl/map/camera.hpp> #include <mbgl/map/map.hpp> #include <mbgl/map/map_impl.hpp> +#include <mbgl/map/camera.hpp> #include <mbgl/map/transform.hpp> -#include <mbgl/math/log2.hpp> +#include <mbgl/annotation/annotation_manager.hpp> +#include <mbgl/layermanager/layer_manager.hpp> +#include <mbgl/style/style_impl.hpp> +#include <mbgl/style/observer.hpp> +#include <mbgl/renderer/update_parameters.hpp> #include <mbgl/renderer/renderer_frontend.hpp> #include <mbgl/renderer/renderer_observer.hpp> -#include <mbgl/renderer/update_parameters.hpp> -#include <mbgl/storage/file_source_manager.hpp> +#include <mbgl/storage/file_source.hpp> #include <mbgl/storage/resource.hpp> #include <mbgl/storage/response.hpp> -#include <mbgl/style/observer.hpp> -#include <mbgl/style/style_impl.hpp> #include <mbgl/util/constants.hpp> +#include <mbgl/util/math.hpp> #include <mbgl/util/exception.hpp> -#include <mbgl/util/logging.hpp> #include <mbgl/util/mapbox.hpp> -#include <mbgl/util/math.hpp> #include <mbgl/util/tile_coordinate.hpp> +#include <mbgl/util/logging.hpp> +#include <mbgl/math/log2.hpp> #include <utility> @@ -30,11 +30,9 @@ Map::Map(RendererFrontend& frontend, MapObserver& observer, const MapOptions& mapOptions, const ResourceOptions& resourceOptions) - : impl(std::make_unique<Impl>( - frontend, - observer, - FileSourceManager::get() ? FileSourceManager::get()->getFileSource(ResourceLoader, resourceOptions) : nullptr, - mapOptions)) {} + : impl(std::make_unique<Impl>(frontend, observer, + FileSource::getSharedFileSource(resourceOptions), + mapOptions)) {} Map::Map(std::unique_ptr<Impl> impl_) : impl(std::move(impl_)) {} diff --git a/src/mbgl/map/map_impl.cpp b/src/mbgl/map/map_impl.cpp index bc2a37fe0..69c3de978 100644 --- a/src/mbgl/map/map_impl.cpp +++ b/src/mbgl/map/map_impl.cpp @@ -11,15 +11,15 @@ Map::Impl::Impl(RendererFrontend& frontend_, MapObserver& observer_, std::shared_ptr<FileSource> fileSource_, const MapOptions& mapOptions) - : observer(observer_), - rendererFrontend(frontend_), - transform(observer, mapOptions.constrainMode(), mapOptions.viewportMode()), - mode(mapOptions.mapMode()), - pixelRatio(mapOptions.pixelRatio()), - crossSourceCollisions(mapOptions.crossSourceCollisions()), - fileSource(std::move(fileSource_)), - style(std::make_unique<style::Style>(fileSource, pixelRatio)), - annotationManager(*style) { + : observer(observer_), + rendererFrontend(frontend_), + transform(observer, mapOptions.constrainMode(), mapOptions.viewportMode()), + mode(mapOptions.mapMode()), + pixelRatio(mapOptions.pixelRatio()), + crossSourceCollisions(mapOptions.crossSourceCollisions()), + fileSource(std::move(fileSource_)), + style(std::make_unique<style::Style>(*fileSource, pixelRatio)), + annotationManager(*style) { transform.setNorthOrientation(mapOptions.northOrientation()); style->impl->setObserver(this); rendererFrontend.setObserver(*this); diff --git a/src/mbgl/sprite/sprite_loader.cpp b/src/mbgl/sprite/sprite_loader.cpp index d4b1cade1..bfb0c570d 100644 --- a/src/mbgl/sprite/sprite_loader.cpp +++ b/src/mbgl/sprite/sprite_loader.cpp @@ -1,18 +1,17 @@ -#include <mbgl/actor/actor.hpp> -#include <mbgl/actor/scheduler.hpp> #include <mbgl/sprite/sprite_loader.hpp> -#include <mbgl/sprite/sprite_loader_observer.hpp> #include <mbgl/sprite/sprite_loader_worker.hpp> +#include <mbgl/sprite/sprite_loader_observer.hpp> #include <mbgl/sprite/sprite_parser.hpp> -#include <mbgl/storage/file_source.hpp> -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/response.hpp> -#include <mbgl/util/async_request.hpp> -#include <mbgl/util/constants.hpp> -#include <mbgl/util/exception.hpp> #include <mbgl/util/logging.hpp> #include <mbgl/util/platform.hpp> #include <mbgl/util/std.hpp> +#include <mbgl/util/constants.hpp> +#include <mbgl/util/exception.hpp> +#include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/actor/actor.hpp> +#include <mbgl/actor/scheduler.hpp> #include <cassert> diff --git a/src/mbgl/storage/asset_file_source.hpp b/src/mbgl/storage/asset_file_source.hpp index 6dfd3ce4a..cc15dbb60 100644 --- a/src/mbgl/storage/asset_file_source.hpp +++ b/src/mbgl/storage/asset_file_source.hpp @@ -14,9 +14,8 @@ public: ~AssetFileSource() override; std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource&) const override; - void pause() override; - void resume() override; + + static bool acceptsURL(const std::string& url); private: class Impl; diff --git a/src/mbgl/storage/file_source.cpp b/src/mbgl/storage/file_source.cpp new file mode 100644 index 000000000..5f60a0527 --- /dev/null +++ b/src/mbgl/storage/file_source.cpp @@ -0,0 +1,37 @@ +#include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/resource_options.hpp> +#include <mbgl/util/string.hpp> + +#include <mutex> +#include <map> + +namespace mbgl { + +std::shared_ptr<FileSource> FileSource::getSharedFileSource(const ResourceOptions& options) { + static std::mutex mutex; + static std::map<std::string, std::weak_ptr<mbgl::FileSource>> fileSources; + + std::lock_guard<std::mutex> lock(mutex); + + // Purge entries no longer in use. + for (auto it = fileSources.begin(); it != fileSources.end();) { + it = it->second.expired() ? fileSources.erase(it) : ++it; + } + + const auto context = reinterpret_cast<uint64_t>(options.platformContext()); + const std::string key = options.baseURL() + '|' + options.accessToken() + '|' + options.cachePath() + '|' + util::toString(context); + + std::shared_ptr<mbgl::FileSource> fileSource; + auto tuple = fileSources.find(key); + if (tuple != fileSources.end()) { + fileSource = tuple->second.lock(); + } + + if (!fileSource) { + fileSources[key] = fileSource = createPlatformFileSource(options); + } + + return fileSource; +} + +} // namespace mbgl diff --git a/src/mbgl/storage/file_source_manager.cpp b/src/mbgl/storage/file_source_manager.cpp deleted file mode 100644 index 6817717f1..000000000 --- a/src/mbgl/storage/file_source_manager.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/resource_options.hpp> -#include <mbgl/util/string.hpp> - -#include <map> -#include <mutex> -#include <tuple> - -namespace mbgl { - -class FileSourceManager::Impl { -public: - using Key = std::tuple<FileSourceType, std::string>; - std::map<Key, std::weak_ptr<FileSource>> fileSources; - std::map<FileSourceType, FileSourceFactory> fileSourceFactories; - std::recursive_mutex mutex; -}; - -FileSourceManager::FileSourceManager() : impl(std::make_unique<Impl>()) {} - -FileSourceManager::~FileSourceManager() = default; - -std::shared_ptr<FileSource> FileSourceManager::getFileSource(FileSourceType type, - const ResourceOptions& options) noexcept { - std::lock_guard<std::recursive_mutex> lock(impl->mutex); - - // Remove released file sources. - for (auto it = impl->fileSources.begin(); it != impl->fileSources.end();) { - it = it->second.expired() ? impl->fileSources.erase(it) : ++it; - } - - const auto context = reinterpret_cast<uint64_t>(options.platformContext()); - const std::string optionsKey = - options.baseURL() + '|' + options.accessToken() + '|' + options.cachePath() + '|' + util::toString(context); - const auto key = std::tie(type, optionsKey); - - std::shared_ptr<FileSource> fileSource; - auto tuple = impl->fileSources.find(key); - if (tuple != impl->fileSources.end()) { - fileSource = tuple->second.lock(); - } - - if (!fileSource) { - auto it = impl->fileSourceFactories.find(type); - if (it != impl->fileSourceFactories.end()) { - assert(it->second); - impl->fileSources[key] = fileSource = it->second(options); - } - } - - return fileSource; -} - -void FileSourceManager::registerFileSourceFactory(FileSourceType type, FileSourceFactory&& factory) noexcept { - assert(factory); - std::lock_guard<std::recursive_mutex> lock(impl->mutex); - impl->fileSourceFactories[type] = std::move(factory); -} - -FileSourceManager::FileSourceFactory FileSourceManager::unRegisterFileSourceFactory(FileSourceType type) noexcept { - std::lock_guard<std::recursive_mutex> lock(impl->mutex); - auto it = impl->fileSourceFactories.find(type); - FileSourceFactory factory; - if (it != impl->fileSourceFactories.end()) { - factory = std::move(it->second); - impl->fileSourceFactories.erase(it); - } - return factory; -} - -} // namespace mbgl diff --git a/src/mbgl/storage/http_file_source.hpp b/src/mbgl/storage/http_file_source.hpp index 693ea3414..09834aa4d 100644 --- a/src/mbgl/storage/http_file_source.hpp +++ b/src/mbgl/storage/http_file_source.hpp @@ -1,7 +1,6 @@ #pragma once #include <mbgl/storage/file_source.hpp> -#include <mbgl/storage/resource.hpp> namespace mbgl { @@ -11,9 +10,6 @@ public: ~HTTPFileSource() override; std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource& resource) const override { - return resource.hasLoadingMethod(Resource::LoadingMethod::Network); - } class Impl; diff --git a/src/mbgl/storage/local_file_source.hpp b/src/mbgl/storage/local_file_source.hpp index 39ebc8c4b..0f065e0b5 100644 --- a/src/mbgl/storage/local_file_source.hpp +++ b/src/mbgl/storage/local_file_source.hpp @@ -14,12 +14,12 @@ public: ~LocalFileSource() override; std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource&) const override; - void pause() override; - void resume() override; + + static bool acceptsURL(const std::string& url); private: class Impl; + std::unique_ptr<util::Thread<Impl>> impl; }; diff --git a/src/mbgl/storage/main_resource_loader.hpp b/src/mbgl/storage/main_resource_loader.hpp deleted file mode 100644 index f5603d7fe..000000000 --- a/src/mbgl/storage/main_resource_loader.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include <mbgl/storage/file_source.hpp> - -namespace mbgl { - -class ResourceTransform; -class ResourceOptions; - -class MainResourceLoader final : public FileSource { -public: - explicit MainResourceLoader(const ResourceOptions& options); - ~MainResourceLoader() override; - - bool supportsCacheOnlyRequests() const override; - std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource&) const override; - void pause() override; - void resume() override; - -private: - class Impl; - const std::unique_ptr<Impl> impl; -}; - -} // namespace mbgl diff --git a/src/mbgl/storage/resource_options.cpp b/src/mbgl/storage/resource_options.cpp index c56a22540..21ecca979 100644 --- a/src/mbgl/storage/resource_options.cpp +++ b/src/mbgl/storage/resource_options.cpp @@ -10,6 +10,7 @@ public: std::string cachePath = ":memory:"; std::string assetPath = "."; uint64_t maximumSize = mbgl::util::DEFAULT_MAX_CACHE_SIZE; + bool supportCacheOnlyRequests = true; void* platformContext = nullptr; }; @@ -68,6 +69,15 @@ uint64_t ResourceOptions::maximumCacheSize() const { return impl_->maximumSize; } +ResourceOptions& ResourceOptions::withCacheOnlyRequestsSupport(bool supportCacheOnlyRequests) { + impl_->supportCacheOnlyRequests = supportCacheOnlyRequests; + return *this; +} + +bool ResourceOptions::supportsCacheOnlyRequests() const { + return impl_->supportCacheOnlyRequests; +} + ResourceOptions& ResourceOptions::withPlatformContext(void* context) { impl_->platformContext = context; return *this; diff --git a/src/mbgl/storage/resource_transform.cpp b/src/mbgl/storage/resource_transform.cpp index eaf10c93f..6596551e6 100644 --- a/src/mbgl/storage/resource_transform.cpp +++ b/src/mbgl/storage/resource_transform.cpp @@ -2,12 +2,12 @@ namespace mbgl { -ResourceTransform::ResourceTransform(TransformCallback callback) : transformCallback(std::move(callback)) {} +ResourceTransform::ResourceTransform(ActorRef<ResourceTransform>, TransformCallback&& callback) + : transformCallback(std::move(callback)) { +} -void ResourceTransform::transform(Resource::Kind kind, const std::string& url, FinishedCallback finished) { - assert(finished); - assert(transformCallback); - transformCallback(kind, url, std::move(finished)); +void ResourceTransform::transform(Resource::Kind kind, const std::string& url, FinishedCallback&& finished) { + finished(transformCallback(kind, url)); } } // namespace mbgl diff --git a/src/mbgl/style/sources/geojson_source.cpp b/src/mbgl/style/sources/geojson_source.cpp index 3afccf07f..e1191705a 100644 --- a/src/mbgl/style/sources/geojson_source.cpp +++ b/src/mbgl/style/sources/geojson_source.cpp @@ -6,7 +6,6 @@ #include <mbgl/style/sources/geojson_source.hpp> #include <mbgl/style/sources/geojson_source_impl.hpp> #include <mbgl/tile/tile.hpp> -#include <mbgl/util/async_request.hpp> #include <mbgl/util/logging.hpp> #include <mbgl/util/thread_pool.hpp> diff --git a/src/mbgl/style/sources/image_source.cpp b/src/mbgl/style/sources/image_source.cpp index d55f7c9f0..4c18ae581 100644 --- a/src/mbgl/style/sources/image_source.cpp +++ b/src/mbgl/style/sources/image_source.cpp @@ -4,7 +4,6 @@ #include <mbgl/style/sources/image_source.hpp> #include <mbgl/style/sources/image_source_impl.hpp> #include <mbgl/tile/tile.hpp> -#include <mbgl/util/async_request.hpp> #include <mbgl/util/geo.hpp> #include <mbgl/util/premultiply.hpp> diff --git a/src/mbgl/style/sources/raster_source.cpp b/src/mbgl/style/sources/raster_source.cpp index f90306945..851f32573 100644 --- a/src/mbgl/style/sources/raster_source.cpp +++ b/src/mbgl/style/sources/raster_source.cpp @@ -6,7 +6,6 @@ #include <mbgl/style/sources/raster_source.hpp> #include <mbgl/style/sources/raster_source_impl.hpp> #include <mbgl/tile/tile.hpp> -#include <mbgl/util/async_request.hpp> #include <mbgl/util/exception.hpp> #include <mbgl/util/mapbox.hpp> diff --git a/src/mbgl/style/sources/vector_source.cpp b/src/mbgl/style/sources/vector_source.cpp index 510106adb..dc1a45fdf 100644 --- a/src/mbgl/style/sources/vector_source.cpp +++ b/src/mbgl/style/sources/vector_source.cpp @@ -6,7 +6,6 @@ #include <mbgl/style/sources/vector_source.hpp> #include <mbgl/style/sources/vector_source_impl.hpp> #include <mbgl/tile/tile.hpp> -#include <mbgl/util/async_request.hpp> #include <mbgl/util/constants.hpp> #include <mbgl/util/exception.hpp> #include <mbgl/util/mapbox.hpp> diff --git a/src/mbgl/style/style.cpp b/src/mbgl/style/style.cpp index 8a821e5a5..783c85009 100644 --- a/src/mbgl/style/style.cpp +++ b/src/mbgl/style/style.cpp @@ -8,8 +8,9 @@ namespace mbgl { namespace style { -Style::Style(std::shared_ptr<FileSource> fileSource, float pixelRatio) - : impl(std::make_unique<Impl>(std::move(fileSource), pixelRatio)) {} +Style::Style(FileSource& fileSource, float pixelRatio) + : impl(std::make_unique<Impl>(fileSource, pixelRatio)) { +} Style::~Style() = default; diff --git a/src/mbgl/style/style_impl.cpp b/src/mbgl/style/style_impl.cpp index d5961b590..ef1f8436f 100644 --- a/src/mbgl/style/style_impl.cpp +++ b/src/mbgl/style/style_impl.cpp @@ -19,7 +19,6 @@ #include <mbgl/style/source_impl.hpp> #include <mbgl/style/style_impl.hpp> #include <mbgl/style/transition_options.hpp> -#include <mbgl/util/async_request.hpp> #include <mbgl/util/exception.hpp> #include <mbgl/util/logging.hpp> #include <mbgl/util/string.hpp> @@ -30,8 +29,8 @@ namespace style { static Observer nullObserver; -Style::Impl::Impl(std::shared_ptr<FileSource> fileSource_, float pixelRatio) - : fileSource(std::move(fileSource_)), +Style::Impl::Impl(FileSource& fileSource_, float pixelRatio) + : fileSource(fileSource_), spriteLoader(std::make_unique<SpriteLoader>(pixelRatio)), light(std::make_unique<Light>()), observer(&nullObserver) { @@ -50,19 +49,13 @@ void Style::Impl::loadJSON(const std::string& json_) { } void Style::Impl::loadURL(const std::string& url_) { - if (!fileSource) { - observer->onStyleError( - std::make_exception_ptr(util::StyleLoadException("Unable to find resource provider for style url."))); - return; - } - lastError = nullptr; observer->onStyleLoading(); loaded = false; url = url_; - styleRequest = fileSource->request(Resource::style(url), [this](Response res) { + styleRequest = fileSource.request(Resource::style(url), [this](Response res) { // Don't allow a loaded, mutated style to be overwritten with a new version. if (mutated && loaded) { return; @@ -119,11 +112,7 @@ void Style::Impl::parse(const std::string& json_) { setLight(std::make_unique<Light>(parser.light)); spriteLoaded = false; - if (fileSource) { - spriteLoader->load(parser.spriteURL, *fileSource); - } else { - onSpriteError(std::make_exception_ptr(std::runtime_error("Unable to find resource provider for sprite url."))); - } + spriteLoader->load(parser.spriteURL, fileSource); glyphURL = parser.glyphURL; loaded = true; @@ -154,9 +143,7 @@ void Style::Impl::addSource(std::unique_ptr<Source> source) { source->setObserver(this); auto item = sources.add(std::move(source)); - if (fileSource) { - item->loadDescription(*fileSource); - } + item->loadDescription(fileSource); } std::unique_ptr<Source> Style::Impl::removeSource(const std::string& id) { @@ -314,8 +301,8 @@ void Style::Impl::onSourceError(Source& source, std::exception_ptr error) { void Style::Impl::onSourceDescriptionChanged(Source& source) { sources.update(source); observer->onSourceDescriptionChanged(source); - if (!source.loaded && fileSource) { - source.loadDescription(*fileSource); + if (!source.loaded) { + source.loadDescription(fileSource); } } diff --git a/src/mbgl/style/style_impl.hpp b/src/mbgl/style/style_impl.hpp index ca165e24f..c4c0a9a41 100644 --- a/src/mbgl/style/style_impl.hpp +++ b/src/mbgl/style/style_impl.hpp @@ -37,7 +37,7 @@ class Style::Impl : public SpriteLoaderObserver, public LightObserver, public util::noncopyable { public: - Impl(std::shared_ptr<FileSource>, float pixelRatio); + Impl(FileSource&, float pixelRatio); ~Impl() override; void loadJSON(const std::string&); @@ -97,7 +97,7 @@ public: private: void parse(const std::string&); - std::shared_ptr<FileSource> fileSource; + FileSource& fileSource; std::string url; std::string json; diff --git a/src/mbgl/text/glyph_manager.cpp b/src/mbgl/text/glyph_manager.cpp index 8caac1be3..35ea1031d 100644 --- a/src/mbgl/text/glyph_manager.cpp +++ b/src/mbgl/text/glyph_manager.cpp @@ -1,12 +1,11 @@ -#include <mbgl/storage/file_source.hpp> -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/response.hpp> #include <mbgl/text/glyph_manager.hpp> #include <mbgl/text/glyph_manager_observer.hpp> #include <mbgl/text/glyph_pbf.hpp> -#include <mbgl/util/async_request.hpp> -#include <mbgl/util/std.hpp> +#include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> #include <mbgl/util/tiny_sdf.hpp> +#include <mbgl/util/std.hpp> namespace mbgl { diff --git a/src/mbgl/tile/geometry_tile.cpp b/src/mbgl/tile/geometry_tile.cpp index 04ee5f9df..4ee15f136 100644 --- a/src/mbgl/tile/geometry_tile.cpp +++ b/src/mbgl/tile/geometry_tile.cpp @@ -265,9 +265,7 @@ void GeometryTile::onGlyphsAvailable(GlyphMap glyphs) { } void GeometryTile::getGlyphs(GlyphDependencies glyphDependencies) { - if (fileSource) { - glyphManager.getGlyphs(*this, std::move(glyphDependencies), *fileSource); - } + glyphManager.getGlyphs(*this, std::move(glyphDependencies), *fileSource); } void GeometryTile::onImagesAvailable(ImageMap images, ImageMap patterns, ImageVersionMap versionMap, uint64_t imageCorrelationID) { diff --git a/src/mbgl/tile/tile_loader_impl.hpp b/src/mbgl/tile/tile_loader_impl.hpp index 51efbb99e..7c0202108 100644 --- a/src/mbgl/tile/tile_loader_impl.hpp +++ b/src/mbgl/tile/tile_loader_impl.hpp @@ -1,19 +1,12 @@ #pragma once -#include <mbgl/renderer/tile_parameters.hpp> -#include <mbgl/storage/file_source.hpp> #include <mbgl/tile/tile_loader.hpp> -#include <mbgl/util/async_request.hpp> +#include <mbgl/storage/file_source.hpp> +#include <mbgl/renderer/tile_parameters.hpp> #include <mbgl/util/tileset.hpp> #include <cassert> -namespace { -inline std::exception_ptr getCantLoadTileError() { - return std::make_exception_ptr(std::runtime_error("Can't load tile.")); -} -} // namespace - namespace mbgl { template <typename T> @@ -33,11 +26,6 @@ TileLoader<T>::TileLoader(T& tile_, Resource::LoadingMethod::CacheOnly)), fileSource(parameters.fileSource) { assert(!request); - if (!fileSource) { - tile.setError(getCantLoadTileError()); - return; - } - if (fileSource->supportsCacheOnlyRequests()) { // When supported, the first request is always optional, even if the TileLoader // is marked as required. That way, we can let the first optional request continue @@ -61,10 +49,6 @@ TileLoader<T>::~TileLoader() = default; template <typename T> void TileLoader<T>::loadFromCache() { assert(!request); - if (!fileSource) { - tile.setError(getCantLoadTileError()); - return; - } resource.loadingMethod = Resource::LoadingMethod::CacheOnly; request = fileSource->request(resource, [this](Response res) { @@ -129,10 +113,6 @@ void TileLoader<T>::loadedData(const Response& res) { template <typename T> void TileLoader<T>::loadFromNetwork() { assert(!request); - if (!fileSource) { - tile.setError(getCantLoadTileError()); - return; - } // Instead of using Resource::LoadingMethod::All, we're first doing a CacheOnly, and then a // NetworkOnly request. diff --git a/test/map/map.test.cpp b/test/map/map.test.cpp index 89be4ad73..3eb01a738 100644 --- a/test/map/map.test.cpp +++ b/test/map/map.test.cpp @@ -10,8 +10,7 @@ #include <mbgl/gl/context.hpp> #include <mbgl/map/map_options.hpp> #include <mbgl/math/log2.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/main_resource_loader.hpp> +#include <mbgl/storage/default_file_source.hpp> #include <mbgl/storage/network_status.hpp> #include <mbgl/storage/online_file_source.hpp> #include <mbgl/storage/resource_options.hpp> @@ -48,17 +47,13 @@ public: MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} template <typename T = FileSource> - MapTest(const std::string& cachePath, - const std::string& assetPath, - float pixelRatio = 1, - MapMode mode = MapMode::Static, - typename std::enable_if<std::is_same<T, MainResourceLoader>::value>::type* = nullptr) - : fileSource(std::make_shared<T>(ResourceOptions().withCachePath(cachePath).withAssetPath(assetPath))), - frontend(pixelRatio), - map(frontend, - observer, - fileSource, - MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} + MapTest(const std::string& cachePath, const std::string& assetPath, + float pixelRatio = 1, MapMode mode = MapMode::Static, + typename std::enable_if<std::is_same<T, DefaultFileSource>::value>::type* = nullptr) + : fileSource(std::make_shared<T>(cachePath, assetPath)) + , frontend(pixelRatio) + , map(frontend, observer, fileSource, + MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} }; TEST(Map, RendererState) { @@ -300,7 +295,7 @@ TEST(Map, CameraToLatLngBoundsUnwrappedCrossDateLine) { } TEST(Map, Offline) { - MapTest<MainResourceLoader> test{":memory:", "."}; + MapTest<DefaultFileSource> test {":memory:", "."}; auto expiredItem = [] (const std::string& path) { Response response; @@ -309,21 +304,19 @@ TEST(Map, Offline) { return response; }; - NetworkStatus::Set(NetworkStatus::Status::Offline); const std::string prefix = "http://127.0.0.1:3000/"; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(Resource::style(prefix + "style.json"), expiredItem("style.json")); - dbfs->forward(Resource::source(prefix + "streets.json"), expiredItem("streets.json")); - dbfs->forward(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json")); - dbfs->forward(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png")); - dbfs->forward(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), - expiredItem("0-0-0.vector.pbf")); - dbfs->forward(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), - expiredItem("glyph.pbf"), - [&] { test.map.getStyle().loadURL(prefix + "style.json"); }); + test.fileSource->put(Resource::style(prefix + "style.json"), expiredItem("style.json")); + test.fileSource->put(Resource::source(prefix + "streets.json"), expiredItem("streets.json")); + test.fileSource->put(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json")); + test.fileSource->put(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png")); + test.fileSource->put(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), expiredItem("0-0-0.vector.pbf")); + test.fileSource->put(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), expiredItem("glyph.pbf")); + NetworkStatus::Set(NetworkStatus::Status::Offline); + + test.map.getStyle().loadURL(prefix + "style.json"); #if ANDROID - test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0046, 0.1); + test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0045, 0.1); #else test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0015, 0.1); #endif @@ -679,7 +672,7 @@ TEST(Map, WithoutVAOExtension) { return; } - MapTest<MainResourceLoader> test{":memory:", "test/fixtures/api/assets"}; + MapTest<DefaultFileSource> test { ":memory:", "test/fixtures/api/assets" }; gfx::BackendScope scope { *test.frontend.getBackend() }; static_cast<gl::Context&>(test.frontend.getBackend()->getContext()).disableVAOExtension = true; @@ -843,7 +836,7 @@ TEST(Map, TEST_DISABLED_ON_CI(ContinuousRendering)) { } TEST(Map, NoContentTiles) { - MapTest<MainResourceLoader> test{":memory:", "."}; + MapTest<DefaultFileSource> test {":memory:", "."}; using namespace std::chrono_literals; @@ -851,32 +844,33 @@ TEST(Map, NoContentTiles) { Response response; response.noContent = true; response.expires = util::now() + 1h; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward( - Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), response, [&] { - test.map.getStyle().loadJSON(R"STYLE({ - "version": 8, - "name": "Water", - "sources": { - "mapbox": { - "type": "vector", - "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"] - } - }, - "layers": [{ - "id": "background", - "type": "background", - "paint": { - "background-color": "red" - } - }, { - "id": "water", - "type": "fill", - "source": "mapbox", - "source-layer": "water" - }] - })STYLE"); - }); + test.fileSource->put(Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, + Tileset::Scheme::XYZ), + response); + + test.map.getStyle().loadJSON(R"STYLE({ + "version": 8, + "name": "Water", + "sources": { + "mapbox": { + "type": "vector", + "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"] + } + }, + "layers": [{ + "id": "background", + "type": "background", + "paint": { + "background-color": "red" + } + }, { + "id": "water", + "type": "fill", + "source": "mapbox", + "source-layer": "water" + }] + })STYLE"); + test::checkImage("test/fixtures/map/nocontent", test.frontend.render(test.map).image, 0.0015, 0.1); } diff --git a/test/src/mbgl/test/fake_file_source.hpp b/test/src/mbgl/test/fake_file_source.hpp index 1faf4b7a1..8803e9576 100644 --- a/test/src/mbgl/test/fake_file_source.hpp +++ b/test/src/mbgl/test/fake_file_source.hpp @@ -2,8 +2,6 @@ #include <mbgl/storage/file_source.hpp> #include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/resource.hpp> -#include <mbgl/util/async_request.hpp> #include <algorithm> #include <list> @@ -44,8 +42,6 @@ public: return std::make_unique<FakeFileRequest>(resource, callback, requests); } - bool canRequest(const Resource&) const override { return true; } - bool respond(Resource::Kind kind, const Response& response) { auto it = std::find_if(requests.begin(), requests.end(), [&] (FakeFileRequest* fakeRequest) { return fakeRequest->resource.kind == kind; @@ -64,7 +60,7 @@ public: }; -class FakeOnlineFileSource : public FakeFileSource { +class FakeOnlineFileSource : public OnlineFileSource, public FakeFileSource { public: std::unique_ptr<AsyncRequest> request(const Resource& resource, Callback callback) override { return FakeFileSource::request(resource, callback); @@ -73,12 +69,7 @@ public: bool respond(Resource::Kind kind, const Response& response) { return FakeFileSource::respond(kind, response); } - - mapbox::base::Value getProperty(const std::string& property) const override { - return onlineFs.getProperty(property); - } - - OnlineFileSource onlineFs; }; + } // namespace mbgl diff --git a/test/src/mbgl/test/stub_file_source.cpp b/test/src/mbgl/test/stub_file_source.cpp index 8870a45bd..0bbff84ff 100644 --- a/test/src/mbgl/test/stub_file_source.cpp +++ b/test/src/mbgl/test/stub_file_source.cpp @@ -1,5 +1,4 @@ #include <mbgl/test/stub_file_source.hpp> -#include <mbgl/util/async_request.hpp> namespace mbgl { diff --git a/test/src/mbgl/test/stub_file_source.hpp b/test/src/mbgl/test/stub_file_source.hpp index 46bb33d5e..1135fa9a8 100644 --- a/test/src/mbgl/test/stub_file_source.hpp +++ b/test/src/mbgl/test/stub_file_source.hpp @@ -2,7 +2,6 @@ #include <mbgl/storage/file_source.hpp> #include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/util/timer.hpp> #include <unordered_map> @@ -20,7 +19,6 @@ public: ~StubFileSource() override; std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override; - bool canRequest(const Resource&) const override { return true; } void remove(AsyncRequest*); using ResponseFunction = std::function<optional<Response> (const Resource&)>; @@ -50,4 +48,15 @@ private: util::Timer timer; }; +class StubOnlineFileSource : public StubFileSource, public OnlineFileSource { +public: + + StubOnlineFileSource(ResponseType t = ResponseType::Asynchronous) : StubFileSource(t) {}; + ~StubOnlineFileSource() override = default; + + std::unique_ptr<AsyncRequest> request(const Resource& r, Callback c) override { return StubFileSource::request(r, c); }; + void remove(AsyncRequest* r) { StubFileSource::remove(r); }; +}; + + } // namespace mbgl diff --git a/test/storage/asset_file_source.test.cpp b/test/storage/asset_file_source.test.cpp index ac04bc7dc..978a41a30 100644 --- a/test/storage/asset_file_source.test.cpp +++ b/test/storage/asset_file_source.test.cpp @@ -1,10 +1,9 @@ -#include <mbgl/actor/actor_ref.hpp> #include <mbgl/storage/asset_file_source.hpp> -#include <mbgl/storage/resource.hpp> -#include <mbgl/util/chrono.hpp> #include <mbgl/util/platform.hpp> +#include <mbgl/util/chrono.hpp> #include <mbgl/util/run_loop.hpp> #include <mbgl/util/thread.hpp> +#include <mbgl/actor/actor_ref.hpp> #include <gtest/gtest.h> #include <atomic> @@ -71,13 +70,12 @@ TEST(AssetFileSource, Load) { } TEST(AssetFileSource, AcceptsURL) { - AssetFileSource fs("test/fixtures/storage/assets"); - EXPECT_TRUE(fs.canRequest(Resource::style("asset://empty"))); - EXPECT_TRUE(fs.canRequest(Resource::style("asset:///test"))); - EXPECT_FALSE(fs.canRequest(Resource::style("assds://foo"))); - EXPECT_FALSE(fs.canRequest(Resource::style("asset:"))); - EXPECT_FALSE(fs.canRequest(Resource::style("style.json"))); - EXPECT_FALSE(fs.canRequest(Resource::style(""))); + EXPECT_TRUE(AssetFileSource::acceptsURL("asset://empty")); + EXPECT_TRUE(AssetFileSource::acceptsURL("asset:///test")); + EXPECT_FALSE(AssetFileSource::acceptsURL("assds://foo")); + EXPECT_FALSE(AssetFileSource::acceptsURL("asset:")); + EXPECT_FALSE(AssetFileSource::acceptsURL("style.json")); + EXPECT_FALSE(AssetFileSource::acceptsURL("")); } TEST(AssetFileSource, EmptyFile) { diff --git a/test/storage/main_resource_loader.test.cpp b/test/storage/default_file_source.test.cpp index c5f1a9c70..52051ac83 100644 --- a/test/storage/main_resource_loader.test.cpp +++ b/test/storage/default_file_source.test.cpp @@ -1,21 +1,17 @@ #include <mbgl/actor/actor.hpp> -#include <mbgl/storage/database_file_source.hpp> -#include <mbgl/storage/file_source_manager.hpp> -#include <mbgl/storage/main_resource_loader.hpp> +#include <mbgl/storage/default_file_source.hpp> #include <mbgl/storage/network_status.hpp> -#include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/resource_options.hpp> #include <mbgl/storage/resource_transform.hpp> #include <mbgl/test/util.hpp> #include <mbgl/util/run_loop.hpp> using namespace mbgl; -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheResponse)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheResponse)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource resource{Resource::Unknown, "http://127.0.0.1:3000/cache"}; + const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" }; Response response; std::unique_ptr<AsyncRequest> req1; @@ -51,11 +47,11 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheResponse)) { loop.run(); } -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource revalidateSame{Resource::Unknown, "http://127.0.0.1:3000/revalidate-same"}; + const Resource revalidateSame { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; std::unique_ptr<AsyncRequest> req1; std::unique_ptr<AsyncRequest> req2; bool gotResponse = false; @@ -113,11 +109,12 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { loop.run(); } -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource revalidateModified{Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified"}; + const Resource revalidateModified{ Resource::Unknown, + "http://127.0.0.1:3000/revalidate-modified" }; std::unique_ptr<AsyncRequest> req1; std::unique_ptr<AsyncRequest> req2; bool gotResponse = false; @@ -132,7 +129,7 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { EXPECT_EQ("Response", *res.data); EXPECT_FALSE(bool(res.expires)); EXPECT_TRUE(res.mustRevalidate); - EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); + EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); EXPECT_FALSE(res.etag); // The first response is stored in the cache, but it has 'must-revalidate' set. This means @@ -152,7 +149,7 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { EXPECT_EQ("Response", *res2.data); EXPECT_TRUE(bool(res2.expires)); EXPECT_TRUE(res2.mustRevalidate); - EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res2.modified); + EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res2.modified); EXPECT_FALSE(res2.etag); } else { // The test server sends a Cache-Control header with a max-age of 1 second. This @@ -165,7 +162,7 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { EXPECT_FALSE(res2.data.get()); EXPECT_TRUE(bool(res2.expires)); EXPECT_TRUE(res2.mustRevalidate); - EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res2.modified); + EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res2.modified); EXPECT_FALSE(res2.etag); loop.stop(); } @@ -175,11 +172,11 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { loop.run(); } -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource revalidateEtag{Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag"}; + const Resource revalidateEtag { Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag" }; std::unique_ptr<AsyncRequest> req1; std::unique_ptr<AsyncRequest> req2; @@ -225,13 +222,16 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { // will notify as expected, the second one will have bound a DefaultFileRequest* in the lambda that // gets invalidated by the first notify's pending.erase, and when it gets notified, the crash // occurs. -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(HTTPIssue1369)) { + +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(HTTPIssue1369)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource resource{Resource::Unknown, "http://127.0.0.1:3000/test"}; + const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/test" }; - auto req = fs.request(resource, [&](Response) { ADD_FAILURE() << "Callback should not be called"; }); + auto req = fs.request(resource, [&](Response) { + ADD_FAILURE() << "Callback should not be called"; + }); req.reset(); req = fs.request(resource, [&](Response res) { req.reset(); @@ -248,77 +248,87 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(HTTPIssue1369)) { loop.run(); } -TEST(MainResourceLoader, OptionalNonExpired) { +TEST(DefaultFileSource, OptionalNonExpired) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource optionalResource{ - Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; + const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; using namespace std::chrono_literals; Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() + 1h; + fs.put(optionalResource, response); std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(optionalResource, response, [&] { - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(bool(res.expires)); - EXPECT_EQ(*response.expires, *res.expires); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(bool(res.expires)); + EXPECT_EQ(*response.expires, *res.expires); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); }); loop.run(); } -TEST(MainResourceLoader, OptionalExpired) { +TEST(DefaultFileSource, OptionalExpired) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource optionalResource{ - Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; + const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; using namespace std::chrono_literals; Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() - 1h; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + fs.put(optionalResource, response); + std::unique_ptr<AsyncRequest> req; - dbfs->forward(optionalResource, response, [&] { - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(bool(res.expires)); - EXPECT_EQ(*response.expires, *res.expires); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(bool(res.expires)); + EXPECT_EQ(*response.expires, *res.expires); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); }); loop.run(); } -TEST(MainResourceLoader, OptionalNotFound) { +TEST(DefaultFileSource, GetBaseURLAndAccessTokenWhilePaused) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); - const Resource optionalResource{ - Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; + fs.pause(); + + auto baseURL = "http://url"; + auto accessToken = "access_token"; + + fs.setAPIBaseURL(baseURL); + fs.setAccessToken(accessToken); + + EXPECT_EQ(fs.getAPIBaseURL(), baseURL); + EXPECT_EQ(fs.getAccessToken(), accessToken); +} + +TEST(DefaultFileSource, OptionalNotFound) { + util::RunLoop loop; + DefaultFileSource fs(":memory:", "."); + + const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; using namespace std::chrono_literals; @@ -340,9 +350,9 @@ TEST(MainResourceLoader, OptionalNotFound) { } // Test that a network only request doesn't attempt to load data from the cache. -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -354,31 +364,30 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() + 1h; + fs.put(resource, response); + std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(resource, response, [&] { - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_TRUE(res.notModified); - EXPECT_FALSE(res.data.get()); - ASSERT_TRUE(bool(res.expires)); - EXPECT_LT(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_TRUE(res.notModified); + EXPECT_FALSE(res.data.get()); + ASSERT_TRUE(bool(res.expires)); + EXPECT_LT(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); }); loop.run(); } // Test that a network only request doesn't attempt to load data from the cache. -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -390,31 +399,30 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() + 1h; + fs.put(resource, response); + std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(resource, response, [&] { - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); }); loop.run(); } // Test that a network only request doesn't attempt to load data from the cache. -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheFull)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheFull)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -425,22 +433,21 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheFull)) { Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() + 1h; + fs.put(resource, response); + std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(resource, response, [&] { - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); }); loop.run(); @@ -448,9 +455,9 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheFull)) { // Test that we can make a request with a Modified field that doesn't first try to load // from cache like a regular request -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -462,22 +469,21 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified) Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() + 1h; + fs.put(resource, response); + std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(resource, response, [&] { - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_TRUE(res.notModified); - EXPECT_FALSE(res.data.get()); - ASSERT_TRUE(bool(res.expires)); - EXPECT_LT(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(bool(res.modified)); - EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_TRUE(res.notModified); + EXPECT_FALSE(res.data.get()); + ASSERT_TRUE(bool(res.expires)); + EXPECT_LT(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(bool(res.modified)); + EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); }); loop.run(); @@ -485,9 +491,9 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified) // Test that we can make a request with a Modified field that doesn't first try to load // from cache like a regular request -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -499,49 +505,40 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { Response response; response.data = std::make_shared<std::string>("Cached value"); response.expires = util::now() + 1h; + fs.put(resource, response); + std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(resource, response, [&] { - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); - EXPECT_FALSE(res.etag); - loop.stop(); - }); + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); + EXPECT_FALSE(res.etag); + loop.stop(); }); loop.run(); } -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(SetResourceTransform)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(SetResourceTransform)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); - - auto onlinefs = std::static_pointer_cast<OnlineFileSource>( - FileSourceManager::get()->getFileSource(FileSourceType::Network, ResourceOptions{})); + DefaultFileSource fs(":memory:", "."); // Translates the URL "localhost://test to http://127.0.0.1:3000/test - Actor<ResourceTransform::TransformCallback> transform( - loop, [](Resource::Kind, const std::string& url, ResourceTransform::FinishedCallback cb) { - if (url == "localhost://test") { - cb("http://127.0.0.1:3000/test"); - } else { - cb(url); - } - }); + Actor<ResourceTransform> transform(loop, [](Resource::Kind, const std::string& url) -> std::string { + if (url == "localhost://test") { + return "http://127.0.0.1:3000/test"; + } else { + return url; + } + }); - onlinefs->setResourceTransform( - {[actorRef = transform.self()]( - Resource::Kind kind, const std::string& url, ResourceTransform::FinishedCallback cb) { - actorRef.invoke(&ResourceTransform::TransformCallback::operator(), kind, url, std::move(cb)); - }}); - const Resource resource1{Resource::Unknown, "localhost://test"}; + fs.setResourceTransform(transform.self()); + const Resource resource1 { Resource::Unknown, "localhost://test" }; std::unique_ptr<AsyncRequest> req; req = fs.request(resource1, [&](Response res) { @@ -558,8 +555,8 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(SetResourceTransform)) { loop.run(); - onlinefs->setResourceTransform({}); - const Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/test"}; + fs.setResourceTransform({}); + const Resource resource2 { Resource::Unknown, "http://127.0.0.1:3000/test" }; req = fs.request(resource2, [&](Response res) { req.reset(); @@ -576,19 +573,22 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(SetResourceTransform)) { loop.run(); } -TEST(MainResourceLoader, SetResourceCachePath) { +TEST(DefaultFileSource, SetResourceCachePath) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); - auto dbfs = std::static_pointer_cast<DatabaseFileSource>( - FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{})); - dbfs->setDatabasePath("./new_offline.db", [&loop] { loop.stop(); }); + DefaultFileSource fs(":memory:", "."); + + Actor<PathChangeCallback> callback(loop, [&]() -> void { + loop.stop(); + }); + + fs.setResourceCachePath("./new_offline.db", callback.self()); loop.run(); } // Test that a stale cache file that has must-revalidate set will trigger a response. -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::CacheOnly; @@ -602,38 +602,37 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { response.expires = Timestamp(Seconds(1417392000)); response.mustRevalidate = true; response.etag.emplace("snowfall"); + fs.put(resource, response); + std::unique_ptr<AsyncRequest> req; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - dbfs->forward(resource, response, [&] { - req = fs.request(resource, [&](Response res) { - req.reset(); - ASSERT_TRUE(res.error.get()); - EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); - EXPECT_EQ("Cached resource is unusable", res.error->message); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(res.expires); - EXPECT_EQ(Timestamp{Seconds(1417392000)}, *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(res.modified); - EXPECT_EQ(Timestamp{Seconds(1417392000)}, *res.modified); - ASSERT_TRUE(res.etag); - EXPECT_EQ("snowfall", *res.etag); - - resource.priorEtag = res.etag; - resource.priorModified = res.modified; - resource.priorExpires = res.expires; - resource.priorData = res.data; + req = fs.request(resource, [&](Response res) { + req.reset(); + ASSERT_TRUE(res.error.get()); + EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); + EXPECT_EQ("Cached resource is unusable", res.error->message); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(res.expires); + EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(res.modified); + EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.modified); + ASSERT_TRUE(res.etag); + EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); + resource.priorEtag = res.etag; + resource.priorModified = res.modified; + resource.priorExpires = res.expires; + resource.priorData = res.data; + + loop.stop(); }); loop.run(); // Now run this request again, with the data we gathered from the previous stale/unusable - // request. We're replacing the data so that we can check that the MainResourceLoader doesn't + // request. We're replacing the data so that we can check that the DefaultFileSource doesn't // attempt another database access if we already have the value. resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; resource.priorData = std::make_shared<std::string>("Prior value"); @@ -665,9 +664,9 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { } // Test that requests for expired resources have lower priority than requests for new resources -TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { +TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { util::RunLoop loop; - MainResourceLoader fs(ResourceOptions{}); + DefaultFileSource fs(":memory:", "."); Response response; std::size_t online_response_counter = 0; @@ -675,20 +674,16 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { using namespace std::chrono_literals; response.expires = util::now() - 1h; - auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); - auto onlineFs = FileSourceManager::get()->getFileSource(FileSourceType::Network, ResourceOptions{}); - // Put existing values into the cache. Resource resource1{Resource::Unknown, "http://127.0.0.1:3000/load/3", {}, Resource::LoadingMethod::All}; response.data = std::make_shared<std::string>("Cached Request 3"); - dbfs->forward(resource1, response); + fs.put(resource1, response); Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/load/4", {}, Resource::LoadingMethod::All}; response.data = std::make_shared<std::string>("Cached Request 4"); - dbfs->forward(resource2, response); + fs.put(resource2, response); - onlineFs->setProperty("max-concurrent-requests", 1u); - fs.pause(); + fs.setMaximumConcurrentRequests(1); NetworkStatus::Set(NetworkStatus::Status::Offline); // Ensure that the online requests for new resources are processed first. @@ -700,6 +695,14 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { EXPECT_EQ("Request 1", *res.data); }); + Resource nonCached2{Resource::Unknown, "http://127.0.0.1:3000/load/2", {}, Resource::LoadingMethod::All}; + std::unique_ptr<AsyncRequest> req2 = fs.request(nonCached2, [&](Response res) { + online_response_counter++; + req2.reset(); + EXPECT_EQ(online_response_counter, 2); // make sure this is responded second + EXPECT_EQ("Request 2", *res.data); + }); + bool req3CachedResponseReceived = false; std::unique_ptr<AsyncRequest> req3 = fs.request(resource1, [&](Response res) { // Offline callback is received first @@ -729,15 +732,6 @@ TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { } }); - Resource nonCached2{Resource::Unknown, "http://127.0.0.1:3000/load/2", {}, Resource::LoadingMethod::All}; - std::unique_ptr<AsyncRequest> req2 = fs.request(nonCached2, [&](Response res) { - online_response_counter++; - req2.reset(); - EXPECT_EQ(online_response_counter, 2); // make sure this is responded second - EXPECT_EQ("Request 2", *res.data); - }); - - fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); loop.run(); diff --git a/test/storage/http_file_source.test.cpp b/test/storage/http_file_source.test.cpp index 37476c8e7..42b4174e6 100644 --- a/test/storage/http_file_source.test.cpp +++ b/test/storage/http_file_source.test.cpp @@ -1,10 +1,9 @@ -#include <mbgl/storage/http_file_source.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/test/util.hpp> -#include <mbgl/util/chrono.hpp> +#include <mbgl/storage/http_file_source.hpp> #include <mbgl/util/exception.hpp> -#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/chrono.hpp> #include <mbgl/util/string.hpp> +#include <mbgl/util/run_loop.hpp> using namespace mbgl; diff --git a/test/storage/local_file_source.test.cpp b/test/storage/local_file_source.test.cpp index 45c8c54d9..e1756f8e7 100644 --- a/test/storage/local_file_source.test.cpp +++ b/test/storage/local_file_source.test.cpp @@ -1,5 +1,4 @@ #include <mbgl/storage/local_file_source.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/util/platform.hpp> #include <mbgl/util/run_loop.hpp> @@ -22,13 +21,12 @@ std::string toAbsoluteURL(const std::string& fileName) { using namespace mbgl; TEST(LocalFileSource, AcceptsURL) { - LocalFileSource fs; - EXPECT_TRUE(fs.canRequest(Resource::style("file://empty"))); - EXPECT_TRUE(fs.canRequest(Resource::style("file:///test"))); - EXPECT_FALSE(fs.canRequest(Resource::style("flie://foo"))); - EXPECT_FALSE(fs.canRequest(Resource::style("file:"))); - EXPECT_FALSE(fs.canRequest(Resource::style("style.json"))); - EXPECT_FALSE(fs.canRequest(Resource::style(""))); + EXPECT_TRUE(LocalFileSource::acceptsURL("file://empty")); + EXPECT_TRUE(LocalFileSource::acceptsURL("file:///test")); + EXPECT_FALSE(LocalFileSource::acceptsURL("flie://foo")); + EXPECT_FALSE(LocalFileSource::acceptsURL("file:")); + EXPECT_FALSE(LocalFileSource::acceptsURL("style.json")); + EXPECT_FALSE(LocalFileSource::acceptsURL("")); } TEST(LocalFileSource, EmptyFile) { diff --git a/test/storage/offline_download.test.cpp b/test/storage/offline_download.test.cpp index a15c96d39..c1a9bb73f 100644 --- a/test/storage/offline_download.test.cpp +++ b/test/storage/offline_download.test.cpp @@ -6,6 +6,7 @@ #include <mbgl/test/sqlite3_test_fs.hpp> #include <mbgl/gfx/headless_frontend.hpp> +#include <mbgl/storage/default_file_source.hpp> #include <mbgl/storage/offline.hpp> #include <mbgl/storage/offline_database.hpp> #include <mbgl/storage/offline_download.hpp> @@ -63,7 +64,7 @@ public: } util::RunLoop loop; - StubFileSource fileSource; + StubOnlineFileSource fileSource; OfflineDatabase db; std::size_t size = 0; @@ -385,7 +386,7 @@ TEST(OfflineDownload, DoesNotFloodTheFileSourceWithRequests) { fileSource.respond(Resource::Kind::Style, test.response("style.json")); test.loop.runOnce(); - EXPECT_EQ(*fileSource.getProperty("max-concurrent-requests").getUint(), fileSource.requests.size()); + EXPECT_EQ(fileSource.getMaximumConcurrentRequests(), fileSource.requests.size()); } TEST(OfflineDownload, GetStatusNoResources) { @@ -821,6 +822,7 @@ TEST(OfflineDownload, AllOfflineRequestsHaveLowPriorityAndOfflineUsage) { test.loop.run(); } + #ifndef __QT__ // Qt doesn't expose the ability to register virtual file system handlers. TEST(OfflineDownload, DiskFull) { FixtureLog log; diff --git a/test/storage/online_file_source.test.cpp b/test/storage/online_file_source.test.cpp index 88dbf519f..3e697a99e 100644 --- a/test/storage/online_file_source.test.cpp +++ b/test/storage/online_file_source.test.cpp @@ -1,12 +1,11 @@ -#include <mbgl/storage/network_status.hpp> -#include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/test/util.hpp> +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/network_status.hpp> #include <mbgl/util/chrono.hpp> -#include <mbgl/util/constants.hpp> #include <mbgl/util/run_loop.hpp> -#include <mbgl/util/string.hpp> #include <mbgl/util/timer.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/constants.hpp> #include <gtest/gtest.h> @@ -53,9 +52,9 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(TemporaryError)) { OnlineFileSource fs; const auto start = Clock::now(); - int counter = 0; auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/temporary-error" }, [&](Response res) { + static int counter = 0; switch (counter++) { case 0: { const auto duration = std::chrono::duration<const double>(Clock::now() - start).count(); @@ -93,10 +92,10 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(ConnectionError)) { OnlineFileSource fs; const auto start = Clock::now(); - int counter = 0; - int wait = 0; std::unique_ptr<AsyncRequest> req = fs.request({ Resource::Unknown, "http://127.0.0.1:3001/" }, [&](Response res) { + static int counter = 0; + static int wait = 0; const auto duration = std::chrono::duration<const double>(Clock::now() - start).count(); EXPECT_LT(wait - 0.01, duration) << "Backoff timer didn't wait 1 second"; EXPECT_GT(wait + 0.3, duration) << "Backoff timer fired too late"; @@ -158,6 +157,10 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RetryDelayOnExpiredTile)) { EXPECT_EQ(nullptr, res.error); EXPECT_GT(util::now(), *res.expires); EXPECT_FALSE(res.mustRevalidate); + }); + + util::Timer timer; + timer.start(Milliseconds(500), Duration::zero(), [&] () { loop.stop(); }); @@ -303,13 +306,12 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChange)) { TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChangePreempt)) { util::RunLoop loop; OnlineFileSource fs; - fs.pause(); const auto start = Clock::now(); - int counter = 0; const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001/test" }; std::unique_ptr<AsyncRequest> req = fs.request(resource, [&](Response res) { + static int counter = 0; const auto duration = std::chrono::duration<const double>(Clock::now() - start).count(); if (counter == 0) { EXPECT_GT(0.2, duration) << "Response came in too late"; @@ -339,7 +341,6 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChangePreempt)) { mbgl::NetworkStatus::Reachable(); }); - fs.resume(); loop.run(); } @@ -415,30 +416,14 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitDefault)) { loop.run(); } -TEST(OnlineFileSource, GetBaseURLAndAccessTokenWhilePaused) { - util::RunLoop loop; - OnlineFileSource fs; - - fs.pause(); - - auto baseURL = "http://url"; - auto accessToken = "access_token"; - - fs.setProperty(API_BASE_URL_KEY, baseURL); - fs.setProperty(ACCESS_TOKEN_KEY, accessToken); - - EXPECT_EQ(*fs.getProperty(API_BASE_URL_KEY).getString(), baseURL); - EXPECT_EQ(*fs.getProperty(ACCESS_TOKEN_KEY).getString(), accessToken); -} - TEST(OnlineFileSource, ChangeAPIBaseURL){ util::RunLoop loop; OnlineFileSource fs; - EXPECT_EQ(mbgl::util::API_BASE_URL, *fs.getProperty(API_BASE_URL_KEY).getString()); + EXPECT_EQ(mbgl::util::API_BASE_URL, fs.getAPIBaseURL()); const std::string customURL = "test.domain"; - fs.setProperty(API_BASE_URL_KEY, customURL); - EXPECT_EQ(customURL, *fs.getProperty(API_BASE_URL_KEY).getString()); + fs.setAPIBaseURL(customURL); + EXPECT_EQ(customURL, fs.getAPIBaseURL()); } @@ -448,38 +433,34 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequests)) { std::size_t response_counter = 0; const std::size_t NUM_REQUESTS = 3; + fs.setMaximumConcurrentRequests(1); + NetworkStatus::Set(NetworkStatus::Status::Offline); - fs.setProperty("max-concurrent-requests", 1u); - // After DefaultFileSource was split, OnlineFileSource lives on a separate - // thread. Pause OnlineFileSource, so that messages are queued for processing. - fs.pause(); - - // First regular request. - Resource regular1{Resource::Unknown, "http://127.0.0.1:3000/load/1"}; - std::unique_ptr<AsyncRequest> req_0 = fs.request(regular1, [&](Response) { - response_counter++; - req_0.reset(); - }); - // Low priority request that will be queued and should be requested last. - Resource low_prio{Resource::Unknown, "http://127.0.0.1:3000/load/2"}; + // requesting a low priority resource + Resource low_prio{ Resource::Unknown, "http://127.0.0.1:3000/load/1" }; low_prio.setPriority(Resource::Priority::Low); - std::unique_ptr<AsyncRequest> req_1 = fs.request(low_prio, [&](Response) { + std::unique_ptr<AsyncRequest> req_0 = fs.request(low_prio, [&](Response) { response_counter++; - req_1.reset(); + req_0.reset(); EXPECT_EQ(response_counter, NUM_REQUESTS); // make sure this is responded last loop.stop(); }); - // Second regular priority request that should de-preoritize low priority request. + // requesting two "regular" resources + Resource regular1{ Resource::Unknown, "http://127.0.0.1:3000/load/2" }; + std::unique_ptr<AsyncRequest> req_1 = fs.request(regular1, [&](Response) { + response_counter++; + req_1.reset(); + }); Resource regular2{ Resource::Unknown, "http://127.0.0.1:3000/load/3" }; std::unique_ptr<AsyncRequest> req_2 = fs.request(regular2, [&](Response) { response_counter++; req_2.reset(); }); - fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); + loop.run(); } @@ -491,9 +472,10 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequestsMany)) { int correct_low = 0; int correct_regular = 0; + + fs.setMaximumConcurrentRequests(1); + NetworkStatus::Set(NetworkStatus::Status::Offline); - fs.setProperty("max-concurrent-requests", 1u); - fs.pause(); std::vector<std::unique_ptr<AsyncRequest>> collector; @@ -533,8 +515,8 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequestsMany)) { } } - fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); + loop.run(); } @@ -542,12 +524,11 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(MaximumConcurrentRequests)) { util::RunLoop loop; OnlineFileSource fs; - ASSERT_EQ(*fs.getProperty("max-concurrent-requests").getUint(), 20u); + ASSERT_EQ(fs.getMaximumConcurrentRequests(), 20u); - fs.setProperty("max-concurrent-requests", 10u); - ASSERT_EQ(*fs.getProperty("max-concurrent-requests").getUint(), 10u); + fs.setMaximumConcurrentRequests(10); + ASSERT_EQ(fs.getMaximumConcurrentRequests(), 10u); } - TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RequestSameUrlMultipleTimes)) { util::RunLoop loop; OnlineFileSource fs; diff --git a/test/storage/sync_file_source.test.cpp b/test/storage/sync_file_source.test.cpp index 3cd6cd9f6..4bd964199 100644 --- a/test/storage/sync_file_source.test.cpp +++ b/test/storage/sync_file_source.test.cpp @@ -1,12 +1,11 @@ +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/storage/file_source.hpp> #include <mbgl/gfx/headless_frontend.hpp> #include <mbgl/map/map.hpp> #include <mbgl/map/map_impl.hpp> -#include <mbgl/storage/file_source.hpp> -#include <mbgl/storage/resource.hpp> #include <mbgl/style/style.hpp> #include <mbgl/test/map_adapter.hpp> -#include <mbgl/util/io.hpp> -#include <mbgl/util/run_loop.hpp> #include <unordered_map> #include <gtest/gtest.h> @@ -15,7 +14,7 @@ using namespace mbgl; class SyncFileSource : public FileSource { public: - std::unique_ptr<AsyncRequest> request(const Resource& resource, FileSource::Callback callback) override { + std::unique_ptr<AsyncRequest> request(const Resource& resource, FileSource::Callback callback) { Response response; auto it = assets.find(resource.url); if (it == assets.end()) { @@ -28,8 +27,6 @@ public: return nullptr; } - bool canRequest(const Resource&) const override { return true; } - void add(std::string const& key, std::string const& data) { assets.emplace(key, std::make_shared<std::string>(data)); }; diff --git a/test/style/source.test.cpp b/test/style/source.test.cpp index 0286aaaec..5eb837d92 100644 --- a/test/style/source.test.cpp +++ b/test/style/source.test.cpp @@ -58,7 +58,7 @@ public: StubRenderSourceObserver renderSourceObserver; Transform transform; TransformState transformState; - Style style{fileSource, 1}; + Style style { *fileSource, 1 }; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/style/style.test.cpp b/test/style/style.test.cpp index c866431ac..b9e19d5a8 100644 --- a/test/style/style.test.cpp +++ b/test/style/style.test.cpp @@ -18,7 +18,7 @@ using namespace mbgl::style; TEST(Style, Properties) { util::RunLoop loop; - auto fileSource = std::make_shared<StubFileSource>(); + StubFileSource fileSource; Style::Impl style { fileSource, 1.0 }; style.loadJSON(R"STYLE({"name": "Test"})STYLE"); @@ -60,7 +60,7 @@ TEST(Style, Properties) { TEST(Style, DuplicateSource) { util::RunLoop loop; - auto fileSource = std::make_shared<StubFileSource>(); + StubFileSource fileSource; Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -81,7 +81,7 @@ TEST(Style, RemoveSourceInUse) { auto log = new FixtureLogObserver(); Log::setObserver(std::unique_ptr<Log::Observer>(log)); - auto fileSource = std::make_shared<StubFileSource>(); + StubFileSource fileSource; Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -106,7 +106,7 @@ TEST(Style, RemoveSourceInUse) { TEST(Style, SourceImplsOrder) { util::RunLoop loop; - auto fileSource = std::make_shared<StubFileSource>(); + StubFileSource fileSource; Style::Impl style{fileSource, 1.0}; style.addSource(std::make_unique<VectorSource>("c", "mapbox://mapbox.mapbox-terrain-v2")); diff --git a/test/style/style_layer.test.cpp b/test/style/style_layer.test.cpp index 77e936ff3..1d60197c2 100644 --- a/test/style/style_layer.test.cpp +++ b/test/style/style_layer.test.cpp @@ -283,7 +283,7 @@ TEST(Layer, DuplicateLayer) { util::RunLoop loop; // Setup style - auto fileSource = std::make_shared<StubFileSource>(); + StubFileSource fileSource; Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -304,7 +304,7 @@ TEST(Layer, IncompatibleLayer) { util::RunLoop loop; // Setup style - auto fileSource = std::make_shared<StubFileSource>(); + StubFileSource fileSource; Style::Impl style{fileSource, 1.0}; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); diff --git a/test/test-files.json b/test/test-files.json index 3a4ad9572..b99750387 100644 --- a/test/test-files.json +++ b/test/test-files.json @@ -36,11 +36,10 @@ "test/src/mbgl/test/test.cpp", "test/src/mbgl/test/util.cpp", "test/storage/asset_file_source.test.cpp", - "test/storage/database_file_source.test.cpp", + "test/storage/default_file_source.test.cpp", "test/storage/headers.test.cpp", "test/storage/http_file_source.test.cpp", "test/storage/local_file_source.test.cpp", - "test/storage/main_resource_loader.test.cpp", "test/storage/offline.test.cpp", "test/storage/offline_database.test.cpp", "test/storage/offline_download.test.cpp", diff --git a/test/tile/custom_geometry_tile.test.cpp b/test/tile/custom_geometry_tile.test.cpp index f3d11ab89..fb905ac07 100644 --- a/test/tile/custom_geometry_tile.test.cpp +++ b/test/tile/custom_geometry_tile.test.cpp @@ -25,7 +25,7 @@ public: std::shared_ptr<FileSource> fileSource = std::make_shared<FakeFileSource>(); TransformState transformState; util::RunLoop loop; - style::Style style{fileSource, 1}; + style::Style style { *fileSource, 1 }; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/geojson_tile.test.cpp b/test/tile/geojson_tile.test.cpp index 25fd268dc..d4bf1e075 100644 --- a/test/tile/geojson_tile.test.cpp +++ b/test/tile/geojson_tile.test.cpp @@ -25,7 +25,7 @@ public: std::shared_ptr<FileSource> fileSource = std::make_shared<FakeFileSource>(); TransformState transformState; util::RunLoop loop; - style::Style style{fileSource, 1}; + style::Style style { *fileSource, 1 }; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/raster_dem_tile.test.cpp b/test/tile/raster_dem_tile.test.cpp index f5f761009..42e759472 100644 --- a/test/tile/raster_dem_tile.test.cpp +++ b/test/tile/raster_dem_tile.test.cpp @@ -19,7 +19,7 @@ public: std::shared_ptr<FileSource> fileSource = std::make_shared<FakeFileSource>(); TransformState transformState; util::RunLoop loop; - style::Style style{fileSource, 1}; + style::Style style { *fileSource, 1 }; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/raster_tile.test.cpp b/test/tile/raster_tile.test.cpp index a5a2875f2..f19bd2626 100644 --- a/test/tile/raster_tile.test.cpp +++ b/test/tile/raster_tile.test.cpp @@ -19,7 +19,7 @@ public: std::shared_ptr<FileSource> fileSource = std::make_shared<FakeFileSource>(); TransformState transformState; util::RunLoop loop; - style::Style style{fileSource, 1}; + style::Style style { *fileSource, 1 }; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/tile_cache.test.cpp b/test/tile/tile_cache.test.cpp index 43b409ae8..7a89ece75 100644 --- a/test/tile/tile_cache.test.cpp +++ b/test/tile/tile_cache.test.cpp @@ -28,7 +28,7 @@ public: std::shared_ptr<FileSource> fileSource = std::make_shared<FakeFileSource>(); TransformState transformState; util::RunLoop loop; - style::Style style{fileSource, 1}; + style::Style style{*fileSource, 1}; AnnotationManager annotationManager{style}; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/vector_tile.test.cpp b/test/tile/vector_tile.test.cpp index d282c874e..940c0272d 100644 --- a/test/tile/vector_tile.test.cpp +++ b/test/tile/vector_tile.test.cpp @@ -25,7 +25,7 @@ public: std::shared_ptr<FileSource> fileSource = std::make_shared<FakeFileSource>(); TransformState transformState; util::RunLoop loop; - style::Style style{fileSource, 1}; + style::Style style { *fileSource, 1 }; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; |