diff options
author | Nikolay Zamotaev <nzamotaev@luxoft.com> | 2019-03-13 16:13:25 +0300 |
---|---|---|
committer | Nikolay Zamotaev <nzamotaev@luxoft.com> | 2019-03-13 16:13:25 +0300 |
commit | 75256bfc7a74f781fc5dbf4d15a771c96466f1be (patch) | |
tree | 0ca523a92fd3879dcf4aeb2b4f5d680e34ea38f7 | |
parent | b0dd327dfa5b3cb0aa3157119f306bfab5515edc (diff) | |
parent | af559c314224584ca656ff5bc1136fdff4ad8400 (diff) |
Merge branch '5.12' of ssh://codereview.qt-project.org:29418/qt-apps/qtauto-deployment-server into dev
Change-Id: I4980db7c3b143d0127a249d51639ef3ed485602e
29 files changed, 1088 insertions, 45 deletions
diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5b4275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.tag export-subst diff --git a/.qmake.conf b/.qmake.conf new file mode 100644 index 0000000..9bdff44 --- /dev/null +++ b/.qmake.conf @@ -0,0 +1,3 @@ +MODULE_VERSION = 5.12 + +CONFIG += prepare_docs qt_docs_targets @@ -0,0 +1 @@ +$Format:%H$ @@ -1,5 +1,5 @@ This is a PoC deployment server, which can be used together with -the Neptune IVI UI and the Pelagicore Application Manager. +the Neptune IVI UI and the Luxoft Application Manager. **This is a development server only - do NOT use in production.** diff --git a/appstore/settings.py b/appstore/settings.py index 377843a..a405491 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/appstore/urls.py b/appstore/urls.py index 327c28b..4d0467f 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/appstore/wsgi.py b/appstore/wsgi.py index 9e80014..3f007d4 100644 --- a/appstore/wsgi.py +++ b/appstore/wsgi.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/doc/online/qtautodeploymentserver.qdocconf b/doc/online/qtautodeploymentserver.qdocconf new file mode 100644 index 0000000..95a4f9e --- /dev/null +++ b/doc/online/qtautodeploymentserver.qdocconf @@ -0,0 +1,11 @@ +include($QT_INSTALL_DOCS/global/qt-module-defaults-online.qdocconf) + +# Add an .html file with sidebar content, used in the online style +# HTML.stylesheets += style/qt5-sidebar.html + +# override defaults for Qt module documentation +HTML.nosubdirs = "false" +HTML.outputsubdir = "qtauto-deployment-server" + +include(../qtautodeploymentserver-project.qdocconf) + diff --git a/doc/qtautodeploymentserver-project.qdocconf b/doc/qtautodeploymentserver-project.qdocconf new file mode 100644 index 0000000..8625f13 --- /dev/null +++ b/doc/qtautodeploymentserver-project.qdocconf @@ -0,0 +1,26 @@ +project = QtAutoDeploymentServer +description = Qt Automotive Suite Deployment Server Documentation +version = $QT_VERSION +url = https://doc.qt.io/QtAutoDeploymentServer + +sourcedirs += src +imagedirs += src/images + +qhp.projects = QtAutoDeploymentServer + +qhp.QtAutoDeploymentServer.file = qtautodeploymentserver.qhp +qhp.QtAutoDeploymentServer.namespace = org.qt-project.qtautodeploymentserver.$QT_VERSION_TAG +qhp.QtAutoDeploymentServer.virtualFolder = qtautodeploymentserver +qhp.QtAutoDeploymentServer.indexTitle = Qt Automotive Suite Deployment Server +qhp.QtAutoDeploymentServer.indexRoot = + +qhp.QtAutoDeploymentServer.filterAttributes = qtautodeploymentserver $QT_VERSION qtrefdoc +qhp.QtAutoDeploymentServer.customFilters.Qt.name = QtAutoDeploymentServer $QT_VERSION +qhp.QtAutoDeploymentServer.customFilters.Qt.filterAttributes = qtautodeploymentserver $QT_VERSION + +tagfile = qtautodeploymentserver.tags + +depends = qtautomotivesuite + +buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" +navigation.homepage = "Qt Automotive Suite" diff --git a/doc/qtautodeploymentserver.qdocconf b/doc/qtautodeploymentserver.qdocconf new file mode 100644 index 0000000..fa27088 --- /dev/null +++ b/doc/qtautodeploymentserver.qdocconf @@ -0,0 +1,2 @@ +include($QT_INSTALL_DOCS/global/qt-module-defaults-offline.qdocconf) +include(qtautodeploymentserver-project.qdocconf) diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc new file mode 100644 index 0000000..c49c3d2 --- /dev/null +++ b/doc/src/deployment-server-reference.qdoc @@ -0,0 +1,447 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-reference.html + \previouspage Qt Automotive Suite Deployment Server + \contentspage Qt Automotive Suite + + \title Qt Automotive Suite Deployment Server API Reference + + \section1 API Reference + + The following tables describe the requests, their parameters, as well as the corresponding responses. + + \section2 hello + + Checks whether you are using the correct Platform and the right API to communicate with the deployment + server. + \table + \header + \li Parameter + \li Description + \row + \li platform + \li The platform on which the client is running. This parameter sets the architecture + of the packages you get. For more information, refer to the + \c{settings.APPSTORE_PLATFORM} parameter in the \c{appstore/settings.py} file. + \row + \li version + \li The Deployment Server's HTTP API version that you are using to communicate with the + server. For more information, see \c{settings.APPSTORE_VERSION}. + \row + \li require_tag + \li An optional parameter used to filter packages by tags. Receives a comma-separated + list of tags; these tags must be alphanumeric. Only applications that contain any + of the specified tags should be listed. + \row + \li conflicts_tag + \li An optional parameter used to filter packages by tags. Receives a comma-separated + list of tags; these tags must be alphanumeric. Applications that contain any of the + speicifed tags should be excluded. + \row + \li architecture + \li An optional parameter used to filter packages by architecture. Receives the CPU + architecture. If the architecture is not speciifed, only packages showing \e{All} + architecture are listed. + \endtable + + Returns a JSON object with the following fields and values. + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,5} status + \li ok + \li Successful + \row + \li maintenance + \li The server is in maintenance mode and cannot be used at the moment. + \row + \li incompatible-platform + \li The platform you are using is not compatible. + \row + \li incompatible-version + \li The API version you are using is not compatible. + \row + \li malformed-tag + \li The tag format is incorrect, may not be alphanumeric, or could + not be parsed. + \endtable + + \section2 login + + Logs onto the deployment server with the given username and password. + Either an IMEI or a unique hardware identifier, such as a MAC address, + must be provided. This call is necessary to be able to download apps. + \table + \header + \li Parameter + \li Description + \row + \li username + \li The username. + \row + \li password + \li The password for the specified username. + \endtable + + Returns a JSON object with the following fields and values. + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,4} status + \li ok + \li Login successful. + \row + \li missing-credentials + \li No username or password was provided. + \row + \li account-disabled + \li The account has been disabled in the Admin panel. + \row + \li authentication-failed + \li The user name and/or password may be wrong; or another authentication error + occurred. + \endtable + + \section2 logout + Logs out the currently logged-in user from the deployment server. + + Returns a JSON object with the following fields and values. + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,2} status + \li ok + \li The user has logged out successfully. + \row + \li failed + \li The user was not logged in. + \endtable + + \section2 app/list + Lists all apps. The returned list can be filtered by using the \c{category_id} and the + \c{filter} parameters. + \table + \header + \li Parameter + \li Description + \row + \li category_id + \li Limits the app to those with this category ID only. + \row + \li filter + \li Lists apps with names that match this filter only. + \endtable + + Returns an array of JSON objects (\b{not an object itself!}). + \table + \header + \li JSON Field + \li Description + \row + \li id + \li A unique app ID, in reverse domain name notation. + \row + \li name + \li The app's name. + \row + \li vendor + \li The vendor name for the app; not the vendor ID. + \row + \li category + \li A category name for the app. + \row + \li tags + \li JSON array of app tags + \row + \li version + \li The app's version, returned as a string. If the there is no version number, the + default version, "0.0.0" is returned. + \row + \li architecture + \li The app's architecture, returned as detected in the app's library components. + + If the application is not native, contains \e{All}. + Otherwise it is formed like this: mips-little_endian-32-elf + + Where it is a combination of: + \list 1 + \li CPU architecture, as returned by \l{https://doc.qt.io/qt-5/qsysinfo.html#buildCpuArchitecture}{QsysInfo::buildCpuArchitecture()} + \li CPU endianness, either \c{little_endian} or \c{big_endian}) + \li ABI bitness + \li Binary format, either \c{elf}, \c{mach_o} or \c{pe32} + \endlist + \row + \li briefDescription + \li A short text that describes the app, limited to 1 line, approximately 80-130 + characters. + \row + \li category_id + \li Numeric category ID that matches the app's category. + \endtable + + \section2 app/icon + Returns an icon for the given application id. + \table + \header + \li Parameter + \li Description + \row + \li id + \li The app ID. + \endtable + + Returns a PNG image if the app exists; an HTTP 404 error otherwise. + + \section2 app/description + + Returns a description for the given app ID. + + \table + \header + \li Parameter + \li Description + \row + \li id + \li app ID + \endtable + + Returns a description text for the app, either HTML or plain text. + + \section2 app/purchase + + Returns a URL that you can use to download the requested app for a certain period of time + only; configurable in the settings. + + \note This request is a legacy from the AppStore. Changing the name of this API would + involve changes in Neptune 3 UI. + + \table + \header + \li Parameter + \li Description + \row + \li device_id + \li A unique device ID for the client hardware; currently not used. + \row + \li id + \li The app ID. + \endtable + Returns a JSON object: + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,2} status + \li ok + \li Successful + \row + \li failed + \li + \li An error has occurred, dheck the error field for more information. + \row + \li error + \li Text. + \li If the status is equal to \c{failed}, contains an error description. + \row + \li url + \li A URL. + \li The URL from where to download the app. Expires according to the value specified + in \c{expiresIn}. + \row + \li expiresIn + \li An integer value. + \li Time in seconds during which the download URL is valid. + \endtable + + \section2 category/list + + Lists all of the available categories. Also returns the \e{All} metacategory, that is used + to hold all available applications. + + Returns an array of JSON objects (\b{not an object itself!}). + + \table + \header + \li JSON field + \li Description + \row + \li id + \li Unique category id. + \row + \li name + \li Category name. + \endtable + + \section2 category/icon + Returns an icon for the given category ID. + + \table + \header + \li Parameter + \li Description + \row + \li id + \li The category ID. + \endtable + + Returns an image in PNG format or an empty 1x1 PNG file. + + \note Currently takes the icon of the first app in the category, if it exists. + + \section2 upload + Accepts remote package upload requests. + The user must be in the \e{staff} group to use this API. Also requires either basic authentication + or a previous call to the \c{login} method. This is a POST request to the server due to the parameters used. + \table + \header + \li Parameter + \li Description + \row + \li description + \li Package description, long version. Can be text or HTML. + \row + \li short-description + \li One line package description. + \row + \li category + \li Category name for the category where the package will be put. + \row + \li vendor + \li Vendor name for the package. + \row + \li package + \li Package itself. This is uploaded as a file parameter. + \endtable + + Returns JSON object: + \table + \header + \li Parameter + \li Value + \li Description + \row + \li {1,9} status + \li ok + \li Success + \row + \li no description + \li The description parameter is missing. + \row + \li no short description + \li The short-description parameter is missing. + \row + \li no category + \li The category parameter is missing. + \row + \li no vendor + \li The vendor parameter is missing. + \row + \li Package validation failed + \li Package did not pass format or sanity validation + \row + \li Non-existing category + \li The specified category does not match the parameter passed. + \row + \li Non-existing vendor + \li The specified vendor does not match the parameter passed. + \row + \li no package to upload + \li There was no \c{package} parameter in the request, or it was not a POST request. + \endtable + + \section2 API Usage Examples + + The Deployment Server exposes an HTTP API. Arguments to these requests need to be provided + using the HTTP GET or POST syntax. The data is returned in JSON, PNG, or text format, depending + on the request. + + \section3 Workflow + + \list 1 + \li Send a \c{hello} request to the server to get the current status and to check + whether your platform is compatible with this server instance: + + \tt http://<server>/hello?platform=AM&version=1 + + Returns: + + \tt { { "status": "ok" } } + + \li Login as \c{user} with password, \c{pass}: + + \tt http://<server>/login?username=user&password=pass + + Returns: + + \tt { { "status": "ok" } } + + \li List all applications + + \tt http://<server>/app/list + + Returns: + + \tt { [{ "category": "Entertainment", + "name": "Nice App", + "vendor": "Luxoft", + "briefDescription": "Nice App is a really nice app.", + "category_id": 4, + "id": "com.luxoft.niceapp"}, + .... + ] } + + + \li Request a download for an app: + + \tt http://<server>/app/purchase?device_id=12345&id=com.luxoft.niceapp + + Returns: + + \tt { { "status": "ok", + "url": "http://<server>/app/download/com.luxoft.niceapp.2.npkg", + "expiresIn": 600 + } } + + \li Use the \c{url} provided in step 4 to download the application within \c{expiresIn} + seconds. + + \endlist +*/
\ No newline at end of file diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc new file mode 100644 index 0000000..c6b2c45 --- /dev/null +++ b/doc/src/deployment-server.qdoc @@ -0,0 +1,161 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-index.html + \contentspage {Qt Automotive Suite} + \nextpage Qt Automotive Suite Deployment Server API Reference + + \title Qt Automotive Suite Deployment Server + + The Qt Automotive Suite Deployment Server is a new component in the Qt Automotive Suite 5.12. + Previously, it was known as the Neptune Appstore and used for demonstrations purposes. + + This component is a lightweight HTTP server that provides network feeds to application packages + that are available for installation by a Qt Application Manager instance, running on a target + device connected to a network. The UI counterpart for the Deployment Server is the Downloads app + in the Neptune 3 UI. Together, the Deployment Server and the Downloads app enable you to install + different apps that are available in the server via network. + + The key scenario for these components is to install apps during the integration testing + process. Additionally, the code can also be used as a reference implementation for a + fully-featured server, and a new Downloads app on the target device for production. + + The Deployment Server works with the Application Installer in the Qt Application Manager and + acts as an installation source for \c{http://} and \c{https://} schemes. In addition to + application packages, the Deployment Server also hosts meta information about each package, + that is used by the Downloads app to visualize the choices available for the user to select. + This meta information is stored in the form of tags and other information in the package + header, according to the Qt Application Manager’s package format. When a package is uploaded + to the server, the package header is parsed, associated with that package, and then sent to a + Downloads app, that queries for a list of available apps. Using this information, a Downloads + app can inform users about apps that are available and even hide those that are not compatible + with the target installation. The figure below illustrates this installation scenario. + + \image deployment-server-installation-handshake.png "Install an App via the Deployment Server" + + The Deployment Server is implemented in Python, using Django, based on the following assumptions. + + \section1 Assumptions + + \list + \li Applications are identified with a group of: Application ID, version, architecture, and tags; + these groups are unique. + \li Architecture is specified as a group of: CPU architecture, endianness, bitness, and OS. + If a package does not contain architecture specific parts, the architecture is specified as \e{All}. + \li CPU architecture is based on the return value from QsysInfo::buildCpuArchitecture(). + \li The installation target is automatically determined by parsing binary files. For example, + detecting an ELF binary means that it's a Linux package; the CPU architecture, such as armv8, + further defines the installation target. + See \l {https://doc.qt.io/qt-5/qsysinfo.html#kernelType} {QSysInfo::kernelType()} for more details. + \li If both native and non-native applications match the selection criteria, then the native application + is preferred. + \li Applications can be further filtered by tags, both as positive (inlucde) and negative (exclude) + filters. + \li Tags are considered alphanumeric and can contain lowercase Latin letters, numbers, and + the underscore symbol. All tags passed to the server are converted to lowercase. + \li Tags can also have an optional version. The version number is separated from tag using a colon (:). + Similar to tags, the version can contain lowercase Latin letters, numbers, and the underscore symbol. + \li Tags are matched according to versions. For example, if you request for "version 5.12", this matches + with "5.12.0", but not vice versa. If you request for a non-versioned tag, any version matches your request. + \li Tag lists in requests and packages are simplified. For example, "qt:5.12,qt:5.12.0" is reduced to "qt:5.12". + \li Although the \l {https://doc.qt.io/QtApplicationManager/manifest.html#basic-manifest} {Application manifest} + allows for any number of categories to be assigned to an application, currently, the Deployment Server + requires manual assignment of only one category to the application. Categories in the application manifest + are ignored. + \li Tag information is parsed from the package header's \b{extra} and \b{extraSigned} parts, + from \b{tags} array. All elements of that array are added to package’s tag list. + \li Each package has a version number. If the manifest does not contain a version field, a default + version "0.0.0" is assigned. + \endlist + + \section1 Installation + + \section2 Set up the Server in a Virtual Environment + + Before you install the dependencies in the Python virtual environment, you must install the + \c{libffi-dev} package. Next, prepare the virtual environment: + + \code + virtualenv ./venv + ./venv/bin/pip install -r requirements.txt + \endcode + + Make sure to adapt the \c{APPSTORE_*} settings in \c{appstore/settings.py} to your environment, + before you run the server. + + Since package downloads are done via temporary files, you need to setup a cron-job to remove + these temporary files periodically. The cron-job should be triggered every + \c{settings.APPSTORE_DOWNLOAD_EXPIRY/2} minutes; it needs to run: + + \code + ./manage.py expire-downloads + \endcode + + \section2 Start the Server + + To start the server, run the following command in your terminal: + + \code + ./manage.py runserver 0.0.0.0:8080 + \endcode + + This command starts the server on port 8080, and is reachable by anyone. You can modify the + listening address to another address that suits your use case. + + \section2 Maintain the Server + + \list + \li Clean up the downloads directory: + \code + ./manage.py expire-downloads + \endcode + + This command removes all files from the downloads directory, that are older than + \c{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. Ideally, this command should be run via a cron-job. + + \li Manually verify a package for upload: + + \code + ./manage.py verify-upload-package <pkg.appkg> + \endcode + + This command verifies if \c{<pkg.appkg>} is a valid package that can be uploaded to the Downloads + app. + + \li Manually add a store signature to a package: + + \code + ./manage.py store-sign-package <in.appkg> <out.appkg> [device id] + \endcode + + This command first verifies the \c{<in.appkg>}. If this verification succeeds, \c{<in.appkg>} + is copied over to \c{<out.appkg>} and a store signature is added. The optional \c{[device id]} + parameter locks the generated package to the device with this ID. + \endlist +*/ diff --git a/doc/src/images/deployment-server-installation-handshake.png b/doc/src/images/deployment-server-installation-handshake.png Binary files differnew file mode 100644 index 0000000..08cc244 --- /dev/null +++ b/doc/src/images/deployment-server-installation-handshake.png diff --git a/header.FDL-QTAS b/header.FDL-QTAS new file mode 100644 index 0000000..6cac945 --- /dev/null +++ b/header.FDL-QTAS @@ -0,0 +1,27 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ diff --git a/header.GPL-QTAS b/header.GPL-QTAS index 0bd19ca..eb625be 100644 --- a/header.GPL-QTAS +++ b/header.GPL-QTAS @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/qtauto-deployment-server.pro b/qtauto-deployment-server.pro new file mode 100644 index 0000000..f7f9002 --- /dev/null +++ b/qtauto-deployment-server.pro @@ -0,0 +1,13 @@ +TEMPLATE = aux + +build_online_docs: { + QMAKE_DOCS = $$PWD/doc/online/qtautodeploymentserver.qdocconf +} else { + QMAKE_DOCS = $$PWD/doc/qtautodeploymentserver.qdocconf +} + +OTHER_FILES += \ + $$PWD/doc/*.qdocconf \ + $$PWD/doc/src/*.qdoc + +load(qt_docs) diff --git a/store/admin.py b/store/admin.py index 1cc06da..eff2550 100644 --- a/store/admin.py +++ b/store/admin.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server @@ -174,7 +175,7 @@ class AppAdminForm(forms.ModelForm): class AppAdmin(admin.ModelAdmin): form = AppAdminForm - list_display = ('name', 'appid', 'architecture', 'version') + list_display = ('name', 'appid', 'architecture', 'version', 'tags') def save_model(self, request, obj, form, change): obj.save() diff --git a/store/api.py b/store/api.py index 6e951a4..88ceef4 100644 --- a/store/api.py +++ b/store/api.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server @@ -42,8 +43,9 @@ from authdecorators import logged_in_or_basicauth, is_staff_member from models import App, Category, Vendor, savePackageFile from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage -from utilities import packagePath, iconPath, downloadPath, validateTag +from utilities import packagePath, iconPath, downloadPath from osandarch import normalizeArch +from tags import SoftwareTagList def hello(request): @@ -58,12 +60,11 @@ def hello(request): for j in ("require_tag", "conflicts_tag",): if j in request.REQUEST: #Tags are coma-separated, - taglist = [i.lower() for i in request.REQUEST[j].split(',') if i] - for i in taglist: - if not validateTag(i): #Tags must be alphanumeric (or, even better - limited to ASCII alphanumeric) - status = 'malformed-tag' - break - request.session[j] = taglist + versionmap = SoftwareTagList() + if not versionmap.parse(request.REQUEST[j]): + status = 'malformed-tag' + break + request.session[j] = str(versionmap) if 'architecture' in request.REQUEST: request.session['architecture'] = normalizeArch(request.REQUEST['architecture']) else: @@ -169,13 +170,15 @@ def appList(request): #"require_tag", "conflicts_tag" # Tags are combined by logical AND (for require) and logical OR for conflicts if 'require_tag' in request.session: - for i in request.session['require_tag']: - regex = '(^|,)%s(,|$)' % (i,) - apps = apps.filter(Q(tags__regex = regex)) + require_tags = SoftwareTagList() + require_tags.parse(request.session['require_tag']) + for i in require_tags.make_regex(): + apps = apps.filter(Q(tags__regex = i)) if 'conflicts_tag' in request.session: - for i in request.session['conflicts_tag']: - regex = '(^|,)%s(,|$)' % (i,) - apps = apps.filter(~Q(tags__regex = regex)) + conflict_tags = SoftwareTagList() + conflict_tags.parse(request.session['conflicts_tag']) + for i in conflict_tags.make_regex(): + apps = apps.filter(~Q(tags__regex=i)) # Here goes the logic of listing packages when multiple architectures are available # in /hello request, the target architecture is stored in the session. By definition target machine can support diff --git a/store/authdecorators.py b/store/authdecorators.py index fbbc01d..2a4119c 100644 --- a/store/authdecorators.py +++ b/store/authdecorators.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2018 Luxoft +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/expire-downloads.py b/store/management/commands/expire-downloads.py index edc5235..94b0d24 100644 --- a/store/management/commands/expire-downloads.py +++ b/store/management/commands/expire-downloads.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/store-sign-package.py b/store/management/commands/store-sign-package.py index 479b4c4..b1a42d0 100644 --- a/store/management/commands/store-sign-package.py +++ b/store/management/commands/store-sign-package.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 81f96fa..77286bc 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2018 Luxoft +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/verify-upload-package.py b/store/management/commands/verify-upload-package.py index 87ac08a..59df116 100644 --- a/store/management/commands/verify-upload-package.py +++ b/store/management/commands/verify-upload-package.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index 772b1d2..51b0677 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server @@ -99,6 +100,6 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='app', - unique_together=set([('appid', 'architecture')]), + unique_together=set([('appid', 'architecture', 'tags')]), ), ] diff --git a/store/models.py b/store/models.py index 63b1d89..eb3ede9 100644 --- a/store/models.py +++ b/store/models.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server @@ -128,10 +129,10 @@ class App(models.Model): class Meta: """Makes the group of id and arch - a unique identifier""" - unique_together = (('appid', 'architecture', ),) + unique_together = (('appid', 'architecture', 'tags'),) def __unicode__(self): - return self.name + " [" + " ".join([self.appid,self.version,self.architecture]) + "]" + return self.name + " [" + " ".join([self.appid,self.version,self.architecture,self.tags]) + "]" def save(self, *args, **kwargs): try: diff --git a/store/osandarch.py b/store/osandarch.py index c7d15a6..9a2db12 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -1,5 +1,6 @@ ############################################################################# ## +## Copyright (C) 2019 Luxoft Sweden AB ## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## diff --git a/store/tags.py b/store/tags.py new file mode 100644 index 0000000..aa73006 --- /dev/null +++ b/store/tags.py @@ -0,0 +1,340 @@ +# vim: set fileencoding=utf-8 : +############################################################################# +## +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the Neptune Deployment Server +## +## $QT_BEGIN_LICENSE:GPL-QTAS$ +## Commercial License Usage +## Licensees holding valid commercial Qt Automotive Suite licenses may use +## this file in accordance with the commercial license agreement provided +## with the Software or, alternatively, in accordance with the terms +## contained in a written agreement between you and The Qt Company. For +## licensing terms and conditions see https://www.qt.io/terms-conditions. +## For further information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 or (at your option) any later version +## approved by the KDE Free Qt Foundation. The licenses are as published by +## the Free Software Foundation and appearing in the file LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +## SPDX-License-Identifier: GPL-3.0 +## +############################################################################# + +import hashlib +import unittest +import re + + +def validateTagVersion(version): + for i in version: + if not i.isalnum(): + if not ((i == "_") or (i == ".")): + return False + return True + + +def validateTag(tag): + if len(tag) == 0: + return False + lst = tag.split(':') + if len(lst) > 2: + return False # More than one version component is not allowed + for i in lst[0]: + if not i.isalnum(): + if i != "_": + return False + if len(lst) > 1: + return validateTagVersion(lst[1]) + return True + + +class SoftwareTag: + def __init__(self, tag): + """ Takes tag and parses it. If it can't parse - raises exception of invalid value + :type tag: str + """ + if not ((type(tag) == str) or (type(tag) == unicode)): + raise (BaseException("Invalid input data-type")) + if not validateTag(tag): + raise (BaseException("Malformed tag")) + tag_version = tag.split(':') + self.tag = tag_version[0].lower() # No, this should be lowercase + self.version = None if len(tag_version) == 1 else tag_version[1] + + def __repr__(self): + return "SoftwareTag()" + + def __str__(self): + if self.version: + return "%s:%s" % (self.tag, self.version) + else: + return self.tag + + def has_version(self): + return self.version is not None + + def match(self, tag): # self is "on server", tag is "request" + if self.tag == tag.tag: + # Names are the same, that is good, matching versions now. + if self.version == tag.version: + return True + else: + if tag.version is None: + return True # qt in request, anything else on server - True + if self.version is not None and self.version.startswith(tag.version + "."): + return True + return False + return False + + def make_regex(self): + if self.version is None: + # versionless tag + temp_string = re.escape(self.tag) + regex = "(%s:[a-z0-9_.]*)|(%s)" % (temp_string, temp_string,) + else: + # tag with versions + temp_string = re.escape("%s:%s" % (self.tag, self.version)) + regex = "(%s\.[a-z0-9_.]*)|(%s)" % (temp_string, temp_string) + return regex + + +class SoftwareTagList: + def __init__(self): + # dictionary of tags, key is - tag name + self.taglist = dict() + + def __str__(self): + lst = list() + for key, value in self.taglist.items(): + lst += [str(i) for i in value] + lst.sort() + return ",".join(lst) + + def __repr__(self): + return "SoftwareTagList()" + + def __getitem__(self, item): + return self.taglist[item] + + def parse(self, tag_string): + self.taglist = dict() + try: + return all(self.append(SoftwareTag(i)) for i in tag_string.split(',')) + except: + return False + + def has_version(self, tag_name): + if tag_name in self.taglist: + # This check is possible, because, when there is tag without version - it is the only tag in the list + if self.taglist[tag_name][0].has_version(): + return True + return False + + def append(self, tag): + # tag should be SoftwareTag, return false or raise exception in case it is not so + if tag.has_version(): + if tag.tag in self.taglist: + # Tag in list - need to check version + if self.has_version(tag.tag) and not any(tag.match(i) for i in self.taglist[tag.tag]): + self.taglist[tag.tag].append(tag) + self.taglist[tag.tag].sort() # this is slow, I guess + else: + # Tag not in list - just add it. + self.taglist[tag.tag] = [tag, ] + else: + # tag without version tag + self.taglist[tag.tag] = [tag, ] + return True + + def is_empty(self): + return len(self.taglist) == 0 + + def make_regex(self): + lst = list() + for key, value in self.taglist.items(): + regex = "(^|,)%s(,|$)" % "|".join([i.make_regex() for i in value]) + lst.append(regex) + return lst + + def match_positive(self, taglist): + # checks that everything from tag list matches current tags + # Start checking with checking if all requested tags in taglist are present in self.taglist + for i in taglist.taglist: + if i not in self.taglist: + return False + # Now we need to check if versions are matching + for tag in taglist.taglist: + if not self.has_version(tag): + # If package tag accepts anything - it already matches, next please + continue + if taglist.has_version(tag) and not any(v1.match(v) for v in taglist[tag] for v1 in self[tag]): + return False + return True + + def match_negative(self, taglist): + # checks that nothing from taglist matches current tags + for i in taglist.taglist: + if i in self.taglist: + if (not taglist.has_version(i)) or (not self.has_version(i)): + return False + # Tag found, version list is present. check if it matches, if it does - check is failed + for version in taglist[i]: + for version1 in self[i]: + if version1.match(version): + return False + return True + + def hash(self): + # Looks like the list is sorted, but well... + return hashlib.md5(str(self)).hexdigest() + + +class TestSoftwareTagMethods(unittest.TestCase): + def test_tag_creation(self): + tag = SoftwareTag('qt') + self.assertFalse(tag.has_version()) + tag = SoftwareTag('qt:5.01') + self.assertTrue(tag.has_version()) + tag = SoftwareTag('qt:5.01_asdf_the_version') + self.assertTrue(tag.has_version()) + tag = SoftwareTag('Qt') + self.assertFalse(tag.has_version()) + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + SoftwareTag('фыва') + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + SoftwareTag('фыва:5.1') + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + SoftwareTag('qt.1:5.1') + with self.assertRaisesRegexp(BaseException, "Invalid input data-type"): + SoftwareTag(1) + + def test_tag_match(self): + tag_13 = SoftwareTag('qt:1.3') + tag_12 = SoftwareTag('qt:1.2') + tag_121 = SoftwareTag('qt:1.2.1') + tag_122 = SoftwareTag('qt:1.2.2') + tag = SoftwareTag('qt') + tag2 = SoftwareTag('neptune') + self.assertFalse(tag_12.match(tag_13)) + self.assertFalse(tag_13.match(tag_12)) + self.assertTrue(tag_121.match(tag_12)) + self.assertTrue(tag_122.match(tag_12)) + self.assertTrue(tag_121.match(tag_121)) + self.assertFalse(tag_12.match(tag_121)) + self.assertFalse(tag.match(tag2)) + self.assertTrue(tag_13.match(tag)) + self.assertFalse(tag.match(tag_13)) + + +class TestSoftwareTagListMethods(unittest.TestCase): + def test_empty(self): + lst = SoftwareTagList() + self.assertTrue(lst.is_empty()) + + def test_not_empty_after_append(self): + lst = SoftwareTagList() + lst.append(SoftwareTag('qt')) + self.assertFalse(lst.is_empty()) + + def test_empty_matches_everything(self): + empty_list = SoftwareTagList() + test_list = SoftwareTagList() + test_list.append(SoftwareTag('qt')) + self.assertTrue(test_list.match_positive(empty_list)) + self.assertTrue(test_list.match_negative(empty_list)) + + def test_match_positive(self): + list_to_test = SoftwareTagList() + list_to_test.parse("qt:5.1,neptune,test:1,second_test") + matching_list = SoftwareTagList() + matching_list.parse("qt") + self.assertTrue(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1") + self.assertTrue(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1,qt:5.2,neptune:1") + self.assertTrue(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1,test:2") + self.assertFalse(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1.1") + self.assertFalse(list_to_test.match_positive(matching_list)) + + def test_match_negative(self): + list_to_test = SoftwareTagList() + list_to_test.parse("qt:5.1,neptune") + matching_list = SoftwareTagList() + matching_list.parse("qt") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("qt:5.1") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("qt:5.1,qt:5.2,neptune:1") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("qt:5.1,qt:5.2") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("test") + self.assertTrue(list_to_test.match_negative(matching_list)) + + def test_append_invalid(self): + lst = SoftwareTagList() + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + self.assertFalse(lst.append(SoftwareTag('qt:1:1'))) # Invalid version + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + self.assertFalse(lst.append(SoftwareTag('фыва'))) # Non-ascii + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + self.assertFalse(lst.append(SoftwareTag(''))) # empty tag is not valid + + def test_append_valid(self): + lst = SoftwareTagList() + # capital letters should be treated as lowercase + self.assertTrue(lst.append(SoftwareTag('QT'))) + # underscore is allowed, capital letters should be treated as lowercase + self.assertTrue(lst.append(SoftwareTag('QT_something'))) + # Version is valid, tag is valid too + self.assertTrue(lst.append(SoftwareTag('qt:1.1.1'))) + + def test_parsing_positive(self): + lst = SoftwareTagList() + self.assertTrue(lst.parse('qt')) + self.assertTrue(lst.parse('qt:5')) + self.assertTrue(lst.parse('qt:5.1')) + self.assertTrue(lst.parse('qt:5.1,qt:5.2')) + self.assertTrue(lst.parse('qt:5.1,qt:5.2,neptune')) + self.assertTrue(lst.parse('qt:5.1,qt:5.2,neptune:5.1,neptune:5.2')) + # This should equal to qt:5.1,qt:5.2,neptune:5.1,neptune:5.2 - due to matching + self.assertTrue(lst.parse('qt:5.1,qt:5.2,qt:5.2,qt:5.2.1,neptune:5.1,neptune:5.2')) + # This equals to: qt, neptune, due to matching + self.assertTrue(lst.parse('qt,qt:5.2,neptune:5.1,neptune')) + + def test_parsing_negative(self): + lst = SoftwareTagList() + self.assertFalse(lst.parse(',,')) # empty tags + self.assertFalse(lst.parse('фыва')) # non-ascii + self.assertFalse(lst.parse('qt:5.1:5.2,qt')) # multiple versions + + def test_hashes_does_not_depend_on_order(self): + lst1 = SoftwareTagList() + lst2 = SoftwareTagList() + self.assertTrue(lst1.parse('qt:5,qt:4,neptune:1')) + self.assertTrue(lst2.parse('neptune:1,qt:4,qt:5')) + self.assertEqual(lst1.hash(), lst2.hash()) + + def test_different_list_different_hash(self): + lst1 = SoftwareTagList() + lst2 = SoftwareTagList() + self.assertTrue(lst1.parse('qt:5,neptune:2')) + self.assertTrue(lst2.parse('neptune:1,qt:5')) + self.assertNotEqual(lst1.hash(), lst2.hash()) + + +if __name__ == '__main__': + unittest.main() diff --git a/store/utilities.py b/store/utilities.py index bce9a7b..42f78f7 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server @@ -44,24 +45,17 @@ from M2Crypto import SMIME, BIO, X509 from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate from django.conf import settings +from tags import SoftwareTagList, SoftwareTag import osandarch -def validateTag(tag): - for i in tag: - if not i.isalnum(): - if i != "_": - return False - return True - def makeTagList(pkgdata): - taglist = set() + taglist = SoftwareTagList() for fields in ('extra', 'extraSigned'): if fields in pkgdata['header']: if 'tags' in pkgdata['header'][fields]: - tags = set(pkgdata['header'][fields]['tags']) # Fill tags list then add them - taglist = taglist.union(tags) - tags = ','.join(taglist) - return tags + for i in list(pkgdata['header'][fields]['tags']): # Fill tags list then add them + taglist.append(SoftwareTag(i)) + return str(taglist) def packagePath(appId = None, architecture = None): path = settings.MEDIA_ROOT + 'packages/' |