[deployment-server] Added checking architecture for native packages

Added checking architecture for native package and architecture filtering

Change-Id: Icd1472078151c11659d40ede1a2a61d5e4cc2aaa
Reviewed-by: Dominik Holland <dominik.holland@pelagicore.com>

8 files changed, 131 insertions, 13 deletions

diff --git a/README.md b/README.md
index 2b0aec9..ebd0327 100644
--- a/README.md
+++ b/README.md
@@ -141,6 +141,7 @@ Checks whether you are using the right Platform and the right API to communicate
 | `version`  | The Deployment Server HTTP API version you are using to communicate with the server. (see `settings.APPSTORE_VERSION`) |
 | `require_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. Only applications containing any of specified tags should be listed. Tags must be alphanumeric. |
 | `conflicts_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. No application containing any of the tags should be listed. Tags must be alphanumeric. |
+| `architecture` | Optional parameter for filtering packages by architecture. Receives cpu architecture. If architecture is not specified, only packages showing 'All' architecture are listed. |
 
 Returns a JSON object:
 
diff --git a/requirements.txt b/requirements.txt
index 6f033fe..d1eb087 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,4 +9,4 @@ ipaddress
 cffi
 paramiko
 cryptography
-
+python-magic==0.4.15
diff --git a/store/admin.py b/store/admin.py
index 0a34951..d7712fd 100644
--- a/store/admin.py
+++ b/store/admin.py
@@ -121,7 +121,7 @@ class CategoryAdmin(admin.ModelAdmin):
 
 class AppAdminForm(forms.ModelForm):
     class Meta:
-        exclude = ["id", "name", "tags"]
+        exclude = ["id", "name", "tags", "architecture"]
 
     appId = ""
     name = ""
@@ -137,8 +137,9 @@ class AppAdminForm(forms.ModelForm):
         except Exception as error:
             raise forms.ValidationError(_('Validation error: %s' % str(error)))
 
-        self.appId = pkgdata['info']['id'];
-        self.name = pkgdata['storeName'];
+        self.appId = pkgdata['info']['id']
+        self.name = pkgdata['storeName']
+        self.architecture = pkgdata['architecture']
 
         try:
             a = App.objects.get(name__exact = self.name)
@@ -176,10 +177,13 @@ class AppAdminForm(forms.ModelForm):
         m = super(AppAdminForm, self).save(commit);
         m.id = self.appId
         m.name = self.name
+        m.architecture = self.architecture
 
         m.file.seek(0)
         pkgdata = parseAndValidatePackageMetadata(m.file)
         taglist = set()
+        if 'binfmt' in pkgdata:
+            taglist.add(pkgdata['binfmt'])
         for fields in ('extra','extraSigned'):
             if fields in pkgdata['header']:
                 if 'tags' in pkgdata['header'][fields]:
@@ -191,7 +195,7 @@ class AppAdminForm(forms.ModelForm):
 
 class AppAdmin(admin.ModelAdmin):
     form = AppAdminForm
-    list_display = ('name', 'id')
+    list_display = ('name', 'id', 'architecture')
 
     def save_model(self, request, obj, form, change):
         obj.save()
diff --git a/store/api.py b/store/api.py
index 9bb8dc2..fc3a18e 100644
--- a/store/api.py
+++ b/store/api.py
@@ -63,6 +63,10 @@ def hello(request):
                     status = 'malformed-tag'
                     break
             request.session[j] = taglist
+    if 'architecture' in request.REQUEST:
+        request.session['architecture'] = request.REQUEST['architecture']
+    else:
+        request.session['architecture'] = ''
     return JsonResponse({'status': status})
 
 
@@ -121,8 +125,13 @@ def appList(request):
         for i in request.session['conflicts_tag']:
             regex = '(^|,)%s(,|$)' % (i,)
             apps = apps.filter(~Q(tags__regex = regex))
+    if 'architecture' in request.session:
+        apps = apps.filter(Q(architecture__exact = request.session['architecture'])|Q(architecture__exact = 'All'))
+    else:
+        apps = apps.filter(Q(architecture__exact = 'All'))
+
+    appList = list(apps.values('id', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture').order_by('id'))
 
-    appList = list(apps.values('id', 'name', 'vendor__name', 'briefDescription', 'category', 'tags').order_by('id'))
     for app in appList:
         app['category_id'] = app['category']
         app['category'] = Category.objects.all().filter(id__exact = app['category_id'])[0].name
diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py
index 474d3af..fc6dabe 100644
--- a/store/migrations/0001_initial.py
+++ b/store/migrations/0001_initial.py
@@ -55,6 +55,7 @@ class Migration(migrations.Migration):
                 ('dateAdded', models.DateField(auto_now_add=True)),
                 ('dateModified', models.DateField(auto_now=True)),
                 ('tags', models.TextField(blank=True)),
+                ('architecture', models.CharField(default=b'All', max_length=20)),
             ],
             options={
             },
diff --git a/store/models.py b/store/models.py
index 7335ab2..02de85d 100644
--- a/store/models.py
+++ b/store/models.py
@@ -123,6 +123,7 @@ class App(models.Model):
     dateAdded = models.DateField(auto_now_add = True)
     dateModified = models.DateField(auto_now = True)
     tags = models.TextField(blank=True)
+    architecture = models.CharField(max_length=20, default='All')
 
     def __unicode__(self):
         return self.name + " [" + self.id + "]"
diff --git a/store/osandarch.py b/store/osandarch.py
new file mode 100644
index 0000000..5e7c3a1
--- /dev/null
+++ b/store/osandarch.py
@@ -0,0 +1,76 @@
+#############################################################################
+##
+## Copyright (C) 2018 Pelagicore AG
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the Neptune Deployment Server
+##
+## $QT_BEGIN_LICENSE:GPL-QTAS$
+## Commercial License Usage
+## Licensees holding valid commercial Qt Automotive Suite licenses may use
+## this file in accordance with the commercial license agreement provided
+## with the Software or, alternatively, in accordance with the terms
+## contained in a written agreement between you and The Qt Company.  For
+## licensing terms and conditions see https://www.qt.io/terms-conditions.
+## For further information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 or (at your option) any later version
+## approved by the KDE Free Qt Foundation. The licenses are as published by
+## the Free Software Foundation and appearing in the file LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+## SPDX-License-Identifier: GPL-3.0
+##
+#############################################################################
+
+# check for file type here.
+# those are expected types
+# ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.18, not stripped
+# ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, not stripped
+# ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped
+# Mach-O 64-bit x86_64 dynamically linked shared library
+# Mach-O 64-bit x86_64 executable
+# Mach-O universal binary with 2 architectures: [x86_64: Mach-O 64-bit x86_64 bundle] [i386: Mach-O i386 bundle] []
+# PE32+ executable (console) x86-64, for MS Windows
+# PE32+ executable (DLL) (console) x86-64, for MS Windows
+# PE32+ executable (DLL) (GUI) x86-64, for MS Windows
+# PE32+ executable (GUI) x86-64, for MS Windows
+
+def getOsArch(str):
+    os = None
+    arch = None
+    fmt = None
+    if str.startswith("ELF "):
+        os = "Linux"
+        arch = str.split(',')
+        arch = arch[1]
+        fmt = "elf"
+    elif str.startswith("Mach-O "):
+        os = "macOS"
+        if " universal " in str:
+            # Universal binary - not supported
+            raise Exception("Universal binaries are not supported in packages")
+        else:
+            arch = str.split(' ')
+            arch = arch[2]
+            fmt = "mach_o"
+    elif str.startswith("PE32+ ") or str.startswith("PE32 "):
+        os = "Windows"
+        arch = str.split(',')
+        arch = arch[0]  # Take first part
+        arch = arch.split(' ')
+        arch = arch[-1]  # Take last element
+        fmt = "pe32"
+    if arch:
+        arch = arch.replace('_', '-')
+    result = {'os': os, 'arch': arch, 'format': fmt }
+    if os:
+        return result
+    else:
+        return None
diff --git a/store/utilities.py b/store/utilities.py
index a464507..b9fa3a4 100644
--- a/store/utilities.py
+++ b/store/utilities.py
@@ -38,12 +38,13 @@ import tarfile
 import tempfile
 import base64
 import os
+import magic
 
 from M2Crypto import SMIME, BIO, X509
 from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate
 
 from django.conf import settings
-
+import osandarch
 
 def validateTag(tag):
     for i in tag:
@@ -161,7 +162,7 @@ def createSignature(hash, signingCertificatePkcs12, signingCertificatePassword):
 def parsePackageMetadata(packageFile):
     pkgdata = { }
 
-    pkg = tarfile.open(fileobj=packageFile, mode='r:*', encoding='utf-8');
+    pkg = tarfile.open(fileobj=packageFile, mode='r:*', encoding='utf-8')
 
     fileCount = 0
     foundFooter = False
@@ -169,6 +170,11 @@ def parsePackageMetadata(packageFile):
     foundInfo = False
     foundIcon = False
     digest = hashlib.new('sha256')
+    #Init magic sequnce checker
+    ms = magic.Magic()
+    osset = set()
+    archset = set()
+    pkgfmt = set()
 
     for entry in pkg:
         fileCount = fileCount + 1
@@ -223,9 +229,6 @@ def parsePackageMetadata(packageFile):
                 entryName = entryName[:-1]
             addToDigest2 = unicode(entryName, 'utf-8').encode('utf-8')
 
-            ## print >>sys.stderr, addToDigest1
-            ## print >>sys.stderr, addToDigest2
-
             if entry.isfile():
                 digest.update(contents)
             digest.update(addToDigest1)
@@ -254,9 +257,20 @@ def parsePackageMetadata(packageFile):
             pkgdata['icon'] = contents
             foundIcon = True
 
-        elif not foundInfo and not foundInfo and fileCount >= 2:
+        elif not foundInfo and not foundIcon and fileCount >= 2:
             raise Exception('package does not start with info.yaml and icon.png - found %s' % entry.name)
 
+        if fileCount > 2:
+            if contents and entry.isfile():
+                # check for file type here.
+                filemagic = ms.from_buffer(contents)
+                osarch = osandarch.getOsArch(filemagic)
+                if osarch:
+                    osset.add(osarch['os'])
+                    archset.add(osarch['arch'])
+                    pkgfmt.add(osarch['format'])
+                print(entry.name, osarch)
+
     # finished enumerating all files
     try:
         docs = list(yaml.safe_load_all(footerContents))
@@ -275,6 +289,18 @@ def parsePackageMetadata(packageFile):
     pkgdata['digest'] = digest.hexdigest()
     pkgdata['rawDigest'] = digest.digest()
 
+    if len(osset) > 1:
+        raise Exception('Package can not contain binaries for more than one OS')
+    if len(archset) > 1:
+        raise Exception('Multiple binary architectures detected in package')
+    if len(pkgfmt) > 1:
+        raise Exception('Multiple binary formats detected in package')
+    if (len(osset) == 0) and (len(archset) == 0) and (len(pkgfmt) == 0):
+        pkgdata['architecture'] = 'All'
+    else:
+        pkgdata['architecture'] = list(archset)[0]
+        pkgdata['binfmt'] = 'binfmt_' + list(pkgfmt)[0]
+
     pkg.close()
     return pkgdata
 
@@ -283,7 +309,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []):
     pkgdata = parsePackageMetadata(packageFile)
 
     partFields = { 'header': [ 'applicationId', 'diskSpaceUsed' ],
-                   'info':   [ 'id', 'name', 'icon' ],
+                   'info':   [ 'id', 'name', 'icon', 'runtime', 'code' ],
                    'footer': [ 'digest' ],
                    'icon':   [],
                    'digest': [] }