Fix for Qt AppMan 5.14 support

Qt 5.14 brough new package format. This change brings backward-compatible support for new and old
packages in the same deployment server. Supported formats are differentiated by version parameter
in /hello API call (version 1 - only old packages, version 2 - new and old packages)

Task-number: AUTOSUITE-1356
Change-Id: Ifcd65f162dbadf069f2bb4f506482bbda6a2984e
Reviewed-by: Egor Nemtsev <enemtsev@luxoft.com>

8 files changed, 105 insertions, 53 deletions

diff --git a/.gitignore b/.gitignore
index 306e8fe..cd45596 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 *.pyc
 db.sqlite3
 media/
+static/
 certificates/
 .idea/*
 venv/*
diff --git a/appstore/settings.py b/appstore/settings.py
index 5babd8d..2224284 100644
--- a/appstore/settings.py
+++ b/appstore/settings.py
@@ -42,7 +42,9 @@ https://docs.djangoproject.com/en/1.7/ref/settings/
 
 APPSTORE_MAINTENANCE = False
 APPSTORE_PLATFORM_ID = 'NEPTUNE3'
-APPSTORE_PLATFORM_VERSION = 1
+APPSTORE_PLATFORM_VERSION = 2     # Maximum supported platform version:
+                                  # version 1 - only old package format
+                                  # version 2 - old and new package formats
 APPSTORE_DOWNLOAD_EXPIRY = 10     # in minutes
 APPSTORE_BIND_TO_DEVICE_ID = True # unique downloads for each device
 APPSTORE_NO_SECURITY = True       # ignore developer signatures and do not generate store signatures
diff --git a/store/admin.py b/store/admin.py
index 491ed8f..2252af1 100644
--- a/store/admin.py
+++ b/store/admin.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2019 Luxoft Sweden AB
+## Copyright (C) 2020 Luxoft Sweden AB
 ## Copyright (C) 2018 Pelagicore AG
 ## Contact: https://www.qt.io/licensing/
 ##
@@ -31,17 +31,18 @@
 #############################################################################
 
 import os
+import StringIO
+from PIL import Image, ImageChops
 
 from django import forms
 from django.contrib import admin
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext_lazy
-from ordered_model.admin import OrderedModelAdmin
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from ordered_model.admin import OrderedModelAdmin
 
 from store.models import *
 from utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList
-import StringIO
 
 class CategoryAdminForm(forms.ModelForm):
     class Meta:
@@ -65,7 +66,7 @@ class CategoryAdminForm(forms.ModelForm):
                 im = im.convert('LA')
             else:
                 # No conversion, icons are uploaded as-is, only scaling is used.
-                im = Image.open(cleared_data['icon'])
+                im = Image.open(cleaned_data['icon'])
             size = (settings.ICON_SIZE_X,settings.ICON_SIZE_Y,)
             im.thumbnail(size, Image.ANTIALIAS)
             imagefile = StringIO.StringIO()
@@ -99,7 +100,7 @@ class CategoryAdmin(OrderedModelAdmin):
 
 class AppAdminForm(forms.ModelForm):
     class Meta:
-        exclude = ["appid", "name", "tags", "architecture", 'version']
+        exclude = ["appid", "name", "tags", "architecture", 'version', 'pkgformat']
 
     appId = ""
     name = ""
@@ -109,7 +110,6 @@ class AppAdminForm(forms.ModelForm):
         file = cleaned_data.get('file')
 
         # validate package
-        pkgdata = None
         try:
             pkgdata = parseAndValidatePackageMetadata(file)
         except Exception as error:
@@ -149,12 +149,13 @@ class AppAdminForm(forms.ModelForm):
         m.file.seek(0)
         pkgdata = parseAndValidatePackageMetadata(m.file)
         m.tags = makeTagList(pkgdata)
+        m.pkgformat = pkgdata['packageFormat']['formatVersion']
         return m
 
 
 class AppAdmin(admin.ModelAdmin):
     form = AppAdminForm
-    list_display = ('name', 'appid', 'architecture', 'version', 'tags')
+    list_display = ('name', 'appid', 'architecture', 'version', 'pkgformat', 'tags')
 
     def save_model(self, request, obj, form, change):
         obj.save()
diff --git a/store/api.py b/store/api.py
index 3384df1..fe8e25d 100644
--- a/store/api.py
+++ b/store/api.py
@@ -51,27 +51,38 @@ from tags import SoftwareTagList
 
 def hello(request):
     status = 'ok'
+    dictionary = getRequestDictionary(request)
+    try:
+        version = int(dictionary.get("version", "-1"))
+        if version > 256: #Sanity check against DoS attack (memory exhaustion)
+            version = -1
+    except:
+        version = -1
+
     if settings.APPSTORE_MAINTENANCE:
         status = 'maintenance'
-    elif getRequestDictionary(request).get("platform", "") != str(settings.APPSTORE_PLATFORM_ID):
+    elif dictionary.get("platform", "") != str(settings.APPSTORE_PLATFORM_ID):
         status = 'incompatible-platform'
-    elif getRequestDictionary(request).get("version", "") != str(settings.APPSTORE_PLATFORM_VERSION):
+    elif not ((version) > 0 and (version <= settings.APPSTORE_PLATFORM_VERSION)):
         status = 'incompatible-version'
 
     for j in ("require_tag", "conflicts_tag",):
-        if j in getRequestDictionary(request): #Tags are coma-separated,
+        if j in dictionary: #Tags are coma-separated,
             versionmap = SoftwareTagList()
             if not versionmap.parse(getRequestDictionary(request)[j]):
                 status = 'malformed-tag'
                 break
             request.session[j] = str(versionmap)
-    if 'architecture' in getRequestDictionary(request):
+
+    if 'architecture' in dictionary:
         arch = normalizeArch(getRequestDictionary(request)['architecture'])
         if arch == "":
             status = 'incompatible-architecture'
         request.session['architecture'] = arch
     else:
         request.session['architecture'] = ''
+
+    request.session['pkgversions'] = range(1, version + 1)
     return JsonResponse({'status': status})
 
 
@@ -161,10 +172,11 @@ def upload(request):
 
 def appList(request):
     apps = App.objects.all()
-    if 'filter' in getRequestDictionary(request):
-        apps = apps.filter(name__contains = getRequestDictionary(request)['filter'])
-    if 'category_id' in getRequestDictionary(request):
-        catId = getRequestDictionary(request)['category_id']
+    dictionary = getRequestDictionary(request)
+    if 'filter' in dictionary:
+        apps = apps.filter(name__contains = dictionary['filter'])
+    if 'category_id' in dictionary:
+        catId = dictionary['category_id']
         if catId != -1: # All metacategory
             apps = apps.filter(category__exact = catId)
 
@@ -189,7 +201,13 @@ def appList(request):
     archlist = ['All', ]
     if 'architecture' in request.session:
         archlist.append(request.session['architecture'])
+
+    versionlist = [1]
+    if 'pkgversions' in request.session:
+        versionlist = request.session['pkgversions']
+
     apps = apps.filter(architecture__in = archlist)
+    apps = apps.filter(pkgformat__in = versionlist)
 
     # After filtering, there are potential duplicates in list. And we should prefer native applications to pure qml ones
     # due to speedups offered.
@@ -284,7 +302,8 @@ def appPurchase(request):
         if not settings.APPSTORE_NO_SECURITY:
             with open(fromFilePath, 'rb') as package:
                 pkgdata = parsePackageMetadata(package)
-                addSignatureToPackage(fromFilePath, toPath + toFile, pkgdata['rawDigest'], deviceId)
+                addSignatureToPackage(fromFilePath, toPath + toFile, pkgdata['rawDigest'], deviceId,
+                                      pkgdata['packageFormat']['formatVersion'])
         else:
             try:
                 shutil.copyfile(fromFilePath, toPath + toFile)
diff --git a/store/management/commands/store-sign-package.py b/store/management/commands/store-sign-package.py
index b1a42d0..cf51670 100644
--- a/store/management/commands/store-sign-package.py
+++ b/store/management/commands/store-sign-package.py
@@ -30,8 +30,6 @@
 ##
 #############################################################################
 
-import sys
-
 from django.core.management.base import BaseCommand, CommandError
 from django.conf import settings
 
@@ -55,7 +53,8 @@ class Command(BaseCommand):
             self.stdout.write('  -> passed validation (internal name: %s)\n' % pkgdata['storeName'])
 
             self.stdout.write('Adding signature to package %s' % destinationPackage)
-            addSignatureToPackage(sourcePackage, destinationPackage, pkgdata['rawDigest'], deviceId)
+            addSignatureToPackage(sourcePackage, destinationPackage, pkgdata['rawDigest'],
+                                  deviceId, pkgdata['packageFormat']['formatVersion'])
             self.stdout.write('  -> finished')
 
         except Exception as error:
diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py
index dca1599..d0e2c04 100644
--- a/store/migrations/0001_initial.py
+++ b/store/migrations/0001_initial.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #############################################################################
 ##
-## Copyright (C) 2019 Luxoft Sweden AB
+## Copyright (C) 2020 Luxoft Sweden AB
 ## Copyright (C) 2018 Pelagicore AG
 ## Contact: https://www.qt.io/licensing/
 ##
@@ -31,7 +31,7 @@
 ##
 #############################################################################
 
-# Generated by Django 1.11 on 2019-03-25 14:51
+# Generated by Django 1.11.27 on 2020-02-07 16:50
 from __future__ import unicode_literals
 
 from django.conf import settings
@@ -63,6 +63,7 @@ class Migration(migrations.Migration):
                 ('tags', models.TextField(blank=True)),
                 ('architecture', models.CharField(default=b'All', max_length=20)),
                 ('version', models.CharField(default=b'0.0.0', max_length=20)),
+                ('pkgformat', models.IntegerField()),
             ],
         ),
         migrations.CreateModel(
diff --git a/store/models.py b/store/models.py
index aa8bf25..93136af 100644
--- a/store/models.py
+++ b/store/models.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2019 Luxoft Sweden AB
+## Copyright (C) 2020 Luxoft Sweden AB
 ## Copyright (C) 2018 Pelagicore AG
 ## Contact: https://www.qt.io/licensing/
 ##
@@ -31,14 +31,12 @@
 #############################################################################
 
 import os
-from PIL import Image, ImageChops
 
 from django.db import models
 from ordered_model.models import OrderedModel
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.files.storage import FileSystemStorage
-from django.db.models.fields.files import ImageFieldFile
 
 from utilities import packagePath, writeTempIcon, makeTagList
 
@@ -101,13 +99,14 @@ class App(models.Model):
     tags = models.TextField(blank=True)
     architecture = models.CharField(max_length=20, default='All')
     version = models.CharField(max_length=20, default='0.0.0')
+    pkgformat = models.IntegerField()
 
     class Meta:
         """Makes the group of id and arch - a unique identifier"""
         unique_together = (('appid', 'architecture', 'tags'),)
 
     def __unicode__(self):
-        return self.name + " [" + " ".join([self.appid,self.version,self.architecture,self.tags]) + "]"
+        return self.name + " [" + " ".join([self.appid, self.version, self.architecture, self.tags]) + "]"
 
     def save(self, *args, **kwargs):
         try:
@@ -123,6 +122,7 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri
     appId = pkgdata['info']['id']
     name = pkgdata['storeName']
     architecture = pkgdata['architecture']
+    pkgformat = pkgdata['packageFormat']['formatVersion']
     tags = makeTagList(pkgdata)
     success, error = writeTempIcon(appId, architecture, tags, pkgdata['icon'])
     if not success:
@@ -145,12 +145,13 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri
         app.description = description
         app.briefDescription = shortdescription
         app.architecture = architecture
+        app.pkgformat = pkgformat
         app.file.save(packagePath(appId, architecture, tags), pkgfile)
         app.save()
     else:
-        app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor,
-                                                 category=category, appid=appId,
-                                                 briefDescription=shortdescription, description=description,
-                                                 architecture=architecture)
+        app, _ = App.objects.get_or_create(name=name, tags=tags, vendor=vendor,
+                                           category=category, appid=appId,
+                                           briefDescription=shortdescription, description=description,
+                                           pkgformat = pkgformat, architecture=architecture)
         app.file.save(packagePath(appId, architecture, tags), pkgfile)
         app.save()
diff --git a/store/utilities.py b/store/utilities.py
index 02faf4a..1205f1b 100644
--- a/store/utilities.py
+++ b/store/utilities.py
@@ -31,20 +31,18 @@
 #############################################################################
 
 import tarfile
-import hashlib
-import hmac
-import yaml
-import sys
-import tarfile
 import tempfile
 import base64
 import os
+import hashlib
+import hmac
+import yaml
 import magic
 
-from M2Crypto import SMIME, BIO, X509
+from django.conf import settings
 from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate
+from M2Crypto import SMIME, BIO, X509
 
-from django.conf import settings
 from tags import SoftwareTagList, SoftwareTag
 import osandarch
 
@@ -196,12 +194,14 @@ def parsePackageMetadata(packageFile):
     foundInfo = False
     foundIcon = False
     digest = hashlib.new('sha256')
-    #Init magic sequnce checker
+    #Init magic sequence checker
     ms = magic.Magic()
     osset = set()
     archset = set()
     pkgfmt = set()
 
+    packageHeaders = ['am-application', 'am-package']
+
     for entry in pkg:
         fileCount = fileCount + 1
 
@@ -234,9 +234,12 @@ def parsePackageMetadata(packageFile):
 
             if len(docs) != 2:
                 raise Exception('file --PACKAGE-HEADER-- does not consist of 2 YAML documents')
-            if docs[0]['formatVersion'] != 1 or docs[0]['formatType'] != 'am-package-header':
+            if not (docs[0]['formatVersion'] in [1, 2] and docs[0]['formatType'] == 'am-package-header'):
                 raise Exception('file --PACKAGE-HEADER-- has an invalid document type')
 
+            # Set initial package format version from --PACKAGE-HEADER--
+            # it must be consistent with info.yaml file
+            pkgdata['packageFormat'] = docs[0]
             pkgdata['header'] = docs[1]
         elif fileCount == 1:
             raise Exception('the first file in the package is not --PACKAGE-HEADER--, but %s' % entry.name)
@@ -271,10 +274,13 @@ def parsePackageMetadata(packageFile):
 
             if len(docs) != 2:
                 raise Exception('file %s does not consist of 2 YAML documents' % entry.name)
-            if docs[0]['formatVersion'] != 1 or docs[0]['formatType'] != 'am-application':
+            if docs[0]['formatVersion'] != 1 or not docs[0]['formatType'] in packageHeaders:
                 raise Exception('file %s has an invalid document type' % entry.name)
+            if (packageHeaders.index(docs[0]['formatType']) + 1) > pkgdata['packageFormat']['formatVersion']:
+                raise Exception('inconsistent package version between --PACKAGE-HEADER-- and info.yaml files.')
 
             pkgdata['info'] = docs[1]
+            pkgdata['info.type'] = docs[0]['formatType']
             foundInfo = True
 
         elif entry.name == 'icon.png':
@@ -310,8 +316,10 @@ def parsePackageMetadata(packageFile):
 
     if len(docs) < 2:
         raise Exception('file --PACKAGE-FOOTER-- does not consist of at least 2 YAML documents')
-    if docs[0]['formatVersion'] != 1 or docs[0]['formatType'] != 'am-package-footer':
+    if not (docs[0]['formatVersion'] in [1, 2]) or docs[0]['formatType'] != 'am-package-footer':
         raise Exception('file --PACKAGE-FOOTER-- has an invalid document type')
+    if docs[0]['formatVersion'] != pkgdata['packageFormat']['formatVersion']:
+        raise Exception('inconsistent package version between --PACKAGE-HEADER-- and --PACKAGE-FOOTER-- files.')
 
     pkgdata['footer'] = docs[1]
     for doc in docs[2:]:
@@ -326,7 +334,7 @@ def parsePackageMetadata(packageFile):
         raise Exception('Multiple binary architectures detected in package')
     if len(pkgfmt) > 1:
         raise Exception('Multiple binary formats detected in package')
-    if (len(osset) == 0) and (len(archset) == 0) and (len(pkgfmt) == 0):
+    if (not osset) and (not archset) and (not pkgfmt):
         pkgdata['architecture'] = 'All'
     else:
         pkgdata['architecture'] = list(archset)[0]
@@ -339,8 +347,24 @@ def parsePackageMetadata(packageFile):
 def parseAndValidatePackageMetadata(packageFile, certificates = []):
     pkgdata = parsePackageMetadata(packageFile)
 
-    partFields = { 'header': [ 'applicationId', 'diskSpaceUsed' ],
-                   'info':   [ 'id', 'name', 'icon', 'runtime', 'code' ],
+    if pkgdata['packageFormat']['formatVersion'] == 1:
+        packageIdKey = 'applicationId'
+    elif pkgdata['packageFormat']['formatVersion'] == 2:
+        packageIdKey = 'packageId'
+    else:
+        raise Exception('Unknown package formatVersion %s' % pkgdata['packageFormat']['formatVersion'])
+
+    if pkgdata['info.type'] == 'am-package':
+        infoList = ['id', 'name', 'icon', 'applications']
+    elif pkgdata['info.type'] == 'am-application':
+        infoList = ['id', 'name', 'icon', 'runtime', 'code']
+    else:
+        raise Exception('Unknown info.yaml formatType %s' % pkgdata['info.type'])
+
+
+
+    partFields = { 'header': [ packageIdKey, 'diskSpaceUsed' ],
+                   'info':   infoList,
                    'footer': [ 'digest' ],
                    'icon':   [],
                    'digest': [] }
@@ -354,8 +378,8 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []):
             if field not in data:
                 raise Exception('metadata %s is missing in the %s part' % (field, part))
 
-    if pkgdata['header']['applicationId'] != pkgdata['info']['id']:
-        raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header']['applicationId'], pkgdata['info']['id']))
+    if pkgdata['header'][packageIdKey] != pkgdata['info']['id']:
+        raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header'][packageIdKey], pkgdata['info']['id']))
 
     error = ['']
     if not isValidDnsName(pkgdata['info']['id'], error):
@@ -375,13 +399,13 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []):
     elif len(pkgdata['info']['name']) > 0:
         name = pkgdata['info']['name'].values()[0]
 
-    if len(name) == 0:
+    if not name:
         raise Exception('could not deduce a suitable package name from the info part')
 
     pkgdata['storeName'] = name
 
     if pkgdata['digest'] != pkgdata['footer']['digest']:
-            raise Exception('digest does not match, is: %s, but should be %s' % (pkgdata['digest'], pkgdata['footer']['digest']))
+        raise Exception('digest does not match, is: %s, but should be %s' % (pkgdata['digest'], pkgdata['footer']['digest']))
     if 'storeSignature' in pkgdata['footer']:
         raise Exception('cannot upload a package with an existing storeSignature field')
 
@@ -422,14 +446,18 @@ def addFileToPackage(sourcePackageFile, destinationPackageFile, fileName, fileCo
     dst.close()
     src.close()
 
-def addSignatureToPackage(sourcePackageFile, destinationPackageFile, digest, deviceId):
+def addSignatureToPackage(sourcePackageFile, destinationPackageFile, digest, deviceId, version=1):
     signingCertificate = ''
     with open(settings.APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE) as cert:
         signingCertificate = cert.read()
 
-    digestPlusId = hmac.new(deviceId, digest, hashlib.sha256).digest();
-    signature = createSignature(digestPlusId, signingCertificate, settings.APPSTORE_STORE_SIGN_PKCS12_PASSWORD)
+    digestPlusId = hmac.new(deviceId, digest, hashlib.sha256).digest()
+    signature = createSignature(digestPlusId, signingCertificate,
+                                settings.APPSTORE_STORE_SIGN_PKCS12_PASSWORD)
 
-    yamlContent = yaml.dump_all([{ 'formatVersion': 1, 'formatType': 'am-package-footer'}, { 'storeSignature': base64.encodestring(signature) }], explicit_start=True)
+    yamlContent = yaml.dump_all([{'formatVersion': version, 'formatType': 'am-package-footer'},
+                                 {'storeSignature': base64.encodestring(signature)}],
+                                explicit_start=True)
 
-    addFileToPackage(sourcePackageFile, destinationPackageFile, '--PACKAGE-FOOTER--store-signature', yamlContent)
+    addFileToPackage(sourcePackageFile, destinationPackageFile,
+                     '--PACKAGE-FOOTER--store-signature', yamlContent)