diff options
author | Nikolay Zamotaev <nzamotaev@luxoft.com> | 2018-06-14 21:17:43 +0300 |
---|---|---|
committer | Nikolay Zamotaev <nzamotaev@luxoft.com> | 2018-07-02 12:39:25 +0000 |
commit | 91e213c05f314e9267ca0fe0b6d283badca450f7 (patch) | |
tree | a01e226ea842c319eb0befa568ccfce70cb9b963 | |
parent | e03e50b4e4d1a48d05ab48fb01769c35fe92d722 (diff) |
[deployment-server] Added checking architecture for native packages
Added checking architecture for native package and architecture filtering
Change-Id: Icd1472078151c11659d40ede1a2a61d5e4cc2aaa
Reviewed-by: Dominik Holland <dominik.holland@pelagicore.com>
-rw-r--r-- | README.md | 1 | ||||
-rw-r--r-- | requirements.txt | 2 | ||||
-rw-r--r-- | store/admin.py | 12 | ||||
-rw-r--r-- | store/api.py | 11 | ||||
-rw-r--r-- | store/migrations/0001_initial.py | 1 | ||||
-rw-r--r-- | store/models.py | 1 | ||||
-rw-r--r-- | store/osandarch.py | 76 | ||||
-rw-r--r-- | store/utilities.py | 40 |
8 files changed, 131 insertions, 13 deletions
@@ -141,6 +141,7 @@ Checks whether you are using the right Platform and the right API to communicate | `version` | The Deployment Server HTTP API version you are using to communicate with the server. (see `settings.APPSTORE_VERSION`) | | `require_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. Only applications containing any of specified tags should be listed. Tags must be alphanumeric. | | `conflicts_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. No application containing any of the tags should be listed. Tags must be alphanumeric. | +| `architecture` | Optional parameter for filtering packages by architecture. Receives cpu architecture. If architecture is not specified, only packages showing 'All' architecture are listed. | Returns a JSON object: diff --git a/requirements.txt b/requirements.txt index 6f033fe..d1eb087 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ ipaddress cffi paramiko cryptography - +python-magic==0.4.15 diff --git a/store/admin.py b/store/admin.py index 0a34951..d7712fd 100644 --- a/store/admin.py +++ b/store/admin.py @@ -121,7 +121,7 @@ class CategoryAdmin(admin.ModelAdmin): class AppAdminForm(forms.ModelForm): class Meta: - exclude = ["id", "name", "tags"] + exclude = ["id", "name", "tags", "architecture"] appId = "" name = "" @@ -137,8 +137,9 @@ class AppAdminForm(forms.ModelForm): except Exception as error: raise forms.ValidationError(_('Validation error: %s' % str(error))) - self.appId = pkgdata['info']['id']; - self.name = pkgdata['storeName']; + self.appId = pkgdata['info']['id'] + self.name = pkgdata['storeName'] + self.architecture = pkgdata['architecture'] try: a = App.objects.get(name__exact = self.name) @@ -176,10 +177,13 @@ class AppAdminForm(forms.ModelForm): m = super(AppAdminForm, self).save(commit); m.id = self.appId m.name = self.name + m.architecture = self.architecture m.file.seek(0) pkgdata = parseAndValidatePackageMetadata(m.file) taglist = set() + if 'binfmt' in pkgdata: + taglist.add(pkgdata['binfmt']) for fields in ('extra','extraSigned'): if fields in pkgdata['header']: if 'tags' in pkgdata['header'][fields]: @@ -191,7 +195,7 @@ class AppAdminForm(forms.ModelForm): class AppAdmin(admin.ModelAdmin): form = AppAdminForm - list_display = ('name', 'id') + list_display = ('name', 'id', 'architecture') def save_model(self, request, obj, form, change): obj.save() diff --git a/store/api.py b/store/api.py index 9bb8dc2..fc3a18e 100644 --- a/store/api.py +++ b/store/api.py @@ -63,6 +63,10 @@ def hello(request): status = 'malformed-tag' break request.session[j] = taglist + if 'architecture' in request.REQUEST: + request.session['architecture'] = request.REQUEST['architecture'] + else: + request.session['architecture'] = '' return JsonResponse({'status': status}) @@ -121,8 +125,13 @@ def appList(request): for i in request.session['conflicts_tag']: regex = '(^|,)%s(,|$)' % (i,) apps = apps.filter(~Q(tags__regex = regex)) + if 'architecture' in request.session: + apps = apps.filter(Q(architecture__exact = request.session['architecture'])|Q(architecture__exact = 'All')) + else: + apps = apps.filter(Q(architecture__exact = 'All')) + + appList = list(apps.values('id', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture').order_by('id')) - appList = list(apps.values('id', 'name', 'vendor__name', 'briefDescription', 'category', 'tags').order_by('id')) for app in appList: app['category_id'] = app['category'] app['category'] = Category.objects.all().filter(id__exact = app['category_id'])[0].name diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index 474d3af..fc6dabe 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -55,6 +55,7 @@ class Migration(migrations.Migration): ('dateAdded', models.DateField(auto_now_add=True)), ('dateModified', models.DateField(auto_now=True)), ('tags', models.TextField(blank=True)), + ('architecture', models.CharField(default=b'All', max_length=20)), ], options={ }, diff --git a/store/models.py b/store/models.py index 7335ab2..02de85d 100644 --- a/store/models.py +++ b/store/models.py @@ -123,6 +123,7 @@ class App(models.Model): dateAdded = models.DateField(auto_now_add = True) dateModified = models.DateField(auto_now = True) tags = models.TextField(blank=True) + architecture = models.CharField(max_length=20, default='All') def __unicode__(self): return self.name + " [" + self.id + "]" diff --git a/store/osandarch.py b/store/osandarch.py new file mode 100644 index 0000000..5e7c3a1 --- /dev/null +++ b/store/osandarch.py @@ -0,0 +1,76 @@ +############################################################################# +## +## Copyright (C) 2018 Pelagicore AG +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the Neptune Deployment Server +## +## $QT_BEGIN_LICENSE:GPL-QTAS$ +## Commercial License Usage +## Licensees holding valid commercial Qt Automotive Suite licenses may use +## this file in accordance with the commercial license agreement provided +## with the Software or, alternatively, in accordance with the terms +## contained in a written agreement between you and The Qt Company. For +## licensing terms and conditions see https://www.qt.io/terms-conditions. +## For further information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 or (at your option) any later version +## approved by the KDE Free Qt Foundation. The licenses are as published by +## the Free Software Foundation and appearing in the file LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +## SPDX-License-Identifier: GPL-3.0 +## +############################################################################# + +# check for file type here. +# those are expected types +# ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.18, not stripped +# ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, not stripped +# ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped +# Mach-O 64-bit x86_64 dynamically linked shared library +# Mach-O 64-bit x86_64 executable +# Mach-O universal binary with 2 architectures: [x86_64: Mach-O 64-bit x86_64 bundle] [i386: Mach-O i386 bundle] [] +# PE32+ executable (console) x86-64, for MS Windows +# PE32+ executable (DLL) (console) x86-64, for MS Windows +# PE32+ executable (DLL) (GUI) x86-64, for MS Windows +# PE32+ executable (GUI) x86-64, for MS Windows + +def getOsArch(str): + os = None + arch = None + fmt = None + if str.startswith("ELF "): + os = "Linux" + arch = str.split(',') + arch = arch[1] + fmt = "elf" + elif str.startswith("Mach-O "): + os = "macOS" + if " universal " in str: + # Universal binary - not supported + raise Exception("Universal binaries are not supported in packages") + else: + arch = str.split(' ') + arch = arch[2] + fmt = "mach_o" + elif str.startswith("PE32+ ") or str.startswith("PE32 "): + os = "Windows" + arch = str.split(',') + arch = arch[0] # Take first part + arch = arch.split(' ') + arch = arch[-1] # Take last element + fmt = "pe32" + if arch: + arch = arch.replace('_', '-') + result = {'os': os, 'arch': arch, 'format': fmt } + if os: + return result + else: + return None diff --git a/store/utilities.py b/store/utilities.py index a464507..b9fa3a4 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -38,12 +38,13 @@ import tarfile import tempfile import base64 import os +import magic from M2Crypto import SMIME, BIO, X509 from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate from django.conf import settings - +import osandarch def validateTag(tag): for i in tag: @@ -161,7 +162,7 @@ def createSignature(hash, signingCertificatePkcs12, signingCertificatePassword): def parsePackageMetadata(packageFile): pkgdata = { } - pkg = tarfile.open(fileobj=packageFile, mode='r:*', encoding='utf-8'); + pkg = tarfile.open(fileobj=packageFile, mode='r:*', encoding='utf-8') fileCount = 0 foundFooter = False @@ -169,6 +170,11 @@ def parsePackageMetadata(packageFile): foundInfo = False foundIcon = False digest = hashlib.new('sha256') + #Init magic sequnce checker + ms = magic.Magic() + osset = set() + archset = set() + pkgfmt = set() for entry in pkg: fileCount = fileCount + 1 @@ -223,9 +229,6 @@ def parsePackageMetadata(packageFile): entryName = entryName[:-1] addToDigest2 = unicode(entryName, 'utf-8').encode('utf-8') - ## print >>sys.stderr, addToDigest1 - ## print >>sys.stderr, addToDigest2 - if entry.isfile(): digest.update(contents) digest.update(addToDigest1) @@ -254,9 +257,20 @@ def parsePackageMetadata(packageFile): pkgdata['icon'] = contents foundIcon = True - elif not foundInfo and not foundInfo and fileCount >= 2: + elif not foundInfo and not foundIcon and fileCount >= 2: raise Exception('package does not start with info.yaml and icon.png - found %s' % entry.name) + if fileCount > 2: + if contents and entry.isfile(): + # check for file type here. + filemagic = ms.from_buffer(contents) + osarch = osandarch.getOsArch(filemagic) + if osarch: + osset.add(osarch['os']) + archset.add(osarch['arch']) + pkgfmt.add(osarch['format']) + print(entry.name, osarch) + # finished enumerating all files try: docs = list(yaml.safe_load_all(footerContents)) @@ -275,6 +289,18 @@ def parsePackageMetadata(packageFile): pkgdata['digest'] = digest.hexdigest() pkgdata['rawDigest'] = digest.digest() + if len(osset) > 1: + raise Exception('Package can not contain binaries for more than one OS') + if len(archset) > 1: + raise Exception('Multiple binary architectures detected in package') + if len(pkgfmt) > 1: + raise Exception('Multiple binary formats detected in package') + if (len(osset) == 0) and (len(archset) == 0) and (len(pkgfmt) == 0): + pkgdata['architecture'] = 'All' + else: + pkgdata['architecture'] = list(archset)[0] + pkgdata['binfmt'] = 'binfmt_' + list(pkgfmt)[0] + pkg.close() return pkgdata @@ -283,7 +309,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): pkgdata = parsePackageMetadata(packageFile) partFields = { 'header': [ 'applicationId', 'diskSpaceUsed' ], - 'info': [ 'id', 'name', 'icon' ], + 'info': [ 'id', 'name', 'icon', 'runtime', 'code' ], 'footer': [ 'digest' ], 'icon': [], 'digest': [] } |