From 6d296dfdff62d24f92239f76b82b4e1d5ea52004 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 26 Jun 2018 20:38:39 +0300 Subject: Applications are now defined uniquely by appid and architecture Change-Id: I99252e75687ae8f0383ac9ecfe68108ddcf35e2a Reviewed-by: Robert Griebl --- store/admin.py | 33 ++++----- store/api.py | 58 ++++++++++++---- store/management/commands/store-upload-package.py | 20 ++---- store/migrations/0001_initial.py | 8 ++- store/models.py | 13 ++-- store/osandarch.py | 83 +++++++++++++++++------ store/utilities.py | 33 +++++---- 7 files changed, 162 insertions(+), 86 deletions(-) diff --git a/store/admin.py b/store/admin.py index 1dbac7f..1cc06da 100644 --- a/store/admin.py +++ b/store/admin.py @@ -120,17 +120,17 @@ class CategoryAdmin(admin.ModelAdmin): class AppAdminForm(forms.ModelForm): class Meta: - exclude = ["id", "name", "tags", "architecture"] + exclude = ["appid", "name", "tags", "architecture", 'version'] appId = "" name = "" def clean(self): cleaned_data = super(AppAdminForm, self).clean() - file = cleaned_data.get('file'); + file = cleaned_data.get('file') # validate package - pkgdata = None; + pkgdata = None try: pkgdata = parseAndValidatePackageMetadata(file) except Exception as error: @@ -140,34 +140,29 @@ class AppAdminForm(forms.ModelForm): self.name = pkgdata['storeName'] self.architecture = pkgdata['architecture'] - try: - a = App.objects.get(name__exact = self.name) - if a.id != pkgdata['info']['id']: - raise forms.ValidationError(_('Validation error: the same package name (%s) is already used for application %s' % (self.name, a.id))) - except App.DoesNotExist: - pass - # check if this really is an update - if hasattr(self, 'instance') and self.instance.id: - if self.appId != self.instance.id: - raise forms.ValidationError(_('Validation error: an update cannot change the application id, tried to change from %s to %s' % (self.instance.id, self.appId))) + if hasattr(self, 'instance') and self.instance.appid: + if self.appId != self.instance.appid: + raise forms.ValidationError(_('Validation error: an update cannot change the application id, tried to change from %s to %s' % (self.instance.appid, self.appId))) + elif self.architecture != self.instance.architecture: + raise forms.ValidationError(_('Validation error: an update cannot change the application architecture from %s to %s' % (self.instance.architecture, self.architecture))) else: try: - if App.objects.get(id__exact = self.appId): - raise forms.ValidationError(_('Validation error: another application with id %s already exists' % str(self.appId))) + if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture): + raise forms.ValidationError(_('Validation error: another application with id %s and architecture %s already exists' % (str(self.appId), str(self.architecture)))) except App.DoesNotExist: pass # write icon into file to serve statically - success, error = writeTempIcon(self.appId, pkgdata['icon']) + success, error = writeTempIcon(self.appId, self.architecture, pkgdata['icon']) if not success: raise forms.ValidationError(_(error)) return cleaned_data def save(self, commit=False): - m = super(AppAdminForm, self).save(commit); - m.id = self.appId + m = super(AppAdminForm, self).save(commit) + m.appid = self.appId m.name = self.name m.architecture = self.architecture @@ -179,7 +174,7 @@ class AppAdminForm(forms.ModelForm): class AppAdmin(admin.ModelAdmin): form = AppAdminForm - list_display = ('name', 'id', 'architecture') + list_display = ('name', 'appid', 'architecture', 'version') def save_model(self, request, obj, form, change): obj.save() diff --git a/store/api.py b/store/api.py index fc3a18e..5656857 100644 --- a/store/api.py +++ b/store/api.py @@ -36,7 +36,7 @@ import shutil import json from django.conf import settings -from django.db.models import Q +from django.db.models import Q, Count from django.http import HttpResponse, HttpResponseForbidden, Http404, JsonResponse from django.contrib import auth from django.template import Context, loader @@ -125,14 +125,32 @@ def appList(request): for i in request.session['conflicts_tag']: regex = '(^|,)%s(,|$)' % (i,) apps = apps.filter(~Q(tags__regex = regex)) - if 'architecture' in request.session: - apps = apps.filter(Q(architecture__exact = request.session['architecture'])|Q(architecture__exact = 'All')) - else: - apps = apps.filter(Q(architecture__exact = 'All')) - appList = list(apps.values('id', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture').order_by('id')) + # Here goes the logic of listing packages when multiple architectures are available + # in /hello request, the target architecture is stored in the session. By definition target machine can support + # both "All" package architecture and it's native one. + # So - here goes filtering by list of architectures + archlist = ['All', ] + if 'architecture' in request.session: + archlist.append(request.session['architecture']) + apps = apps.filter(architecture__in = archlist) + + # After filtering, there are potential duplicates in list. And we should prefer native applications to pure qml ones + # due to speedups offered. + # So - first applications are grouped by appid and numbers of same appids counted. In case where is more than one appid - + # there are two variants of application: for 'All' architecture, and for the architecture supported by the target machine. + # So, as native apps should be preferred + duplicates = ( + apps.values('appid').order_by().annotate(count_id=Count('id')).filter(count_id__gt=1) + ) + # Here we go over duplicates list and filter out 'All' architecture apps. + for duplicate in duplicates: + apps = apps.filter(~Q(appid__exact = duplicate['appid'], architecture__exact = 'All')) # if there is native - 'All' architecture apps are excluded + + appList = list(apps.values('appid', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture', 'version').order_by('appid','architecture')) for app in appList: + app['id'] = app['appid'] app['category_id'] = app['category'] app['category'] = Category.objects.all().filter(id__exact = app['category_id'])[0].name app['vendor'] = app['vendor__name'] @@ -141,23 +159,32 @@ def appList(request): else: app['tags'] = [] del app['vendor__name'] + del app['appid'] # this is not valid JSON, since we are returning a list! return JsonResponse(appList, safe = False) def appDescription(request): + archlist = ['All', ] + if 'architecture' in request.session: + archlist.append(request.session['architecture']) try: - app = App.objects.get(id__exact = request.REQUEST['id']) + app = App.objects.get(appid__exact = request.REQUEST['id'], architecture__in = archlist).order_by('architecture') + app = app.last() return HttpResponse(app.description) except: raise Http404('no such application: %s' % request.REQUEST['id']) def appIcon(request): + archlist = ['All', ] + if 'architecture' in request.session: + archlist.append(request.session['architecture']) try: - app = App.objects.get(id__exact = request.REQUEST['id']) - with open(iconPath(app.id), 'rb') as iconPng: + app = App.objects.filter(appid__exact = request.REQUEST['id'], architecture__in = archlist).order_by('architecture') + app = app.last() + with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: response = HttpResponse(content_type = 'image/png') response.write(iconPng.read()) return response @@ -168,7 +195,9 @@ def appIcon(request): def appPurchase(request): if not request.user.is_authenticated(): return HttpResponseForbidden('no login') - + archlist = ['All', ] + if 'architecture' in request.session: + archlist.append(request.session['architecture']) try: deviceId = str(request.REQUEST.get("device_id", "")) if settings.APPSTORE_BIND_TO_DEVICE_ID: @@ -177,12 +206,13 @@ def appPurchase(request): else: deviceId = '' - app = App.objects.get(id__exact = request.REQUEST['id']) - fromFilePath = packagePath(app.id) + app = App.objects.filter(appid__exact = request.REQUEST['id'], architecture__in=archlist).order_by('architecture') + app = app.last() + fromFilePath = packagePath(app.appid, app.architecture) # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) - toFile = str(app.id) + '_' + str(request.user.id) + '_' + str(deviceId) + '.appkg' + toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(deviceId) + '.appkg' toPath = downloadPath() if not os.path.exists(toPath): os.makedirs(toPath) @@ -229,7 +259,7 @@ def categoryIcon(request): # there are no category icons (yet), so we just return the icon of the first app in this category try: app = App.objects.filter(category__exact = request.REQUEST['id']).order_by('-dateModified')[0] #FIXME - the category icon is unimplemented - with open(iconPath(app.id), 'rb') as iconPng: + with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: response.write(iconPng.read()) except: emptyPng = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x01\x03\x00\x00\x00%\xdbV\xca\x00\x00\x00\x03PLTE\x00\x00\x00\xa7z=\xda\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\nIDAT\x08\xd7c`\x00\x00\x00\x02\x00\x01\xe2!\xbc3\x00\x00\x00\x00IEND\xaeB`\x82' diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 8a174ce..facf15e 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -88,40 +88,32 @@ class Command(BaseCommand): description = options['description'] tags = makeTagList(pkgdata) - try: - a = App.objects.get(name__exact=name) - if a.id != pkgdata['info']['id']: - raise CommandError( - 'Validation error: the same package name (%s) is already used for application %s' % ( - name, a.id)) - except App.DoesNotExist: - pass - - success, error = writeTempIcon(appId,pkgdata['icon']) + success, error = writeTempIcon(appId, architecture, pkgdata['icon']) if not success: raise CommandError(error) exists = False app = None try: - app = App.objects.get(id__exact=appId) + app = App.objects.get(appid__exact=appId, architecture__exact= architecture) exists = True except App.DoesNotExist: pass if exists: + app.appid = appId app.category = category[0] app.vendor = vendor[0] app.name = name app.tags = tags app.description = app.briefDescription = description app.architecture = architecture - app.file.save(packagePath(appId), ContentFile(packagefile.read())) + app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read())) app.save() else: app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor[0], - category=category[0], id=appId, + category=category[0], appid=appId, briefDescription=description, description=description, architecture=architecture) - app.file.save(packagePath(appId), ContentFile(packagefile.read())) + app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read())) app.save() diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index fc6dabe..772b1d2 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -47,7 +47,8 @@ class Migration(migrations.Migration): migrations.CreateModel( name='App', fields=[ - ('id', models.CharField(max_length=200, serialize=False, primary_key=True)), + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('appid', models.CharField(max_length=200)), ('name', models.CharField(max_length=200)), ('file', models.FileField(storage=store.models.OverwriteStorage(), upload_to=store.models.content_file_name)), ('briefDescription', models.TextField()), @@ -56,6 +57,7 @@ class Migration(migrations.Migration): ('dateModified', models.DateField(auto_now=True)), ('tags', models.TextField(blank=True)), ('architecture', models.CharField(default=b'All', max_length=20)), + ('version', models.CharField(default=b'0.0.0', max_length=20)), ], options={ }, @@ -95,4 +97,8 @@ class Migration(migrations.Migration): field=models.ForeignKey(to='store.Vendor'), preserve_default=True, ), + migrations.AlterUniqueTogether( + name='app', + unique_together=set([('appid', 'architecture')]), + ), ] diff --git a/store/models.py b/store/models.py index 02de85d..802d141 100644 --- a/store/models.py +++ b/store/models.py @@ -110,10 +110,10 @@ class OverwriteStorage(FileSystemStorage): return name def content_file_name(instance, filename): - return packagePath(instance.id) + return packagePath(instance.appid, instance.architecture) class App(models.Model): - id = models.CharField(max_length = 200, primary_key=True) + appid = models.CharField(max_length = 200) name = models.CharField(max_length = 200) file = models.FileField(upload_to = content_file_name, storage = OverwriteStorage()) vendor = models.ForeignKey(Vendor) @@ -124,13 +124,18 @@ class App(models.Model): dateModified = models.DateField(auto_now = True) tags = models.TextField(blank=True) architecture = models.CharField(max_length=20, default='All') + version = models.CharField(max_length=20, default='0.0.0') + + class Meta: + """Makes the group of id and arch - a unique identifier""" + unique_together = (('appid', 'architecture', ),) def __unicode__(self): - return self.name + " [" + self.id + "]" + return self.name + " [" + " ".join([self.appid,self.version,self.architecture]) + "]" def save(self, *args, **kwargs): try: - this = App.objects.get(id=self.id) + this = App.objects.get(appid=self.appid,architecture=self.architecture) if this.file != self.file: this.file.delete(save=False) except: diff --git a/store/osandarch.py b/store/osandarch.py index 5e7c3a1..f3aa388 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -42,35 +42,78 @@ # PE32+ executable (DLL) (GUI) x86-64, for MS Windows # PE32+ executable (GUI) x86-64, for MS Windows +def parseMachO(str): # os, arch, bits, endianness + if " universal " in str: + # Universal binary - not supported + raise Exception("Universal binaries are not supported in packages") + os = "macOS" + arch = str.split(' ') + arch = arch[2] + bits = str.split(' ')[1].replace('-bit', '') + endianness = "lsb" + return [os, arch, bits, endianness] + + +def parsePE32(str): + os = "Windows" + arch = str.split(',') + arch = arch[0] # Take first part + arch = arch.split(' ') + arch = arch[-1] # Take last element + bits = '32' + if arch == 'x86-64': + bits = '64' + if arch == '80386': + arch = 'i386' + endianness = "lsb" + return [os, arch, bits, endianness] + + +def parseElfArch(str, architecture): + architecture = architecture.strip() + if architecture.startswith("ARM"): + if 'aarch64' in architecture: + return 'aarch64' + if 'armhf' in str: # this does not work for some reason - from_file() returns longer data than from_buffer() - needs fix + return 'armhf' + elif architecture.startswith("Intel"): + if '80386' in architecture: + return 'i386' + elif architecture.startswith("IBM S/390"): + return 's/390' + elif "PowerPC" in architecture: + return 'powerpc' + return architecture.lower() + + +def parseElf(str): + os = "Linux" + arch = str.split(',') + arch = arch[1] + arch = parseElfArch(str, arch) + bits = str.split(' ')[1].replace('-bit', '') + endianness = str.split(' ')[2].lower() + return [os, arch, bits, endianness] + + def getOsArch(str): os = None arch = None + bits = None + endianness = None fmt = None if str.startswith("ELF "): - os = "Linux" - arch = str.split(',') - arch = arch[1] fmt = "elf" + os, arch, bits, endianness = parseElf(str) elif str.startswith("Mach-O "): - os = "macOS" - if " universal " in str: - # Universal binary - not supported - raise Exception("Universal binaries are not supported in packages") - else: - arch = str.split(' ') - arch = arch[2] - fmt = "mach_o" + fmt = "mach_o" + os, arch, bits, endianness = parseMachO(str) elif str.startswith("PE32+ ") or str.startswith("PE32 "): - os = "Windows" - arch = str.split(',') - arch = arch[0] # Take first part - arch = arch.split(' ') - arch = arch[-1] # Take last element fmt = "pe32" + os, arch, bits, endianness = parsePE32(str) if arch: - arch = arch.replace('_', '-') - result = {'os': os, 'arch': arch, 'format': fmt } + arch = arch.replace('-', '_') + result = [os, arch, endianness, bits, fmt] if os: return result - else: - return None + return None diff --git a/store/utilities.py b/store/utilities.py index b571d79..bce9a7b 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -63,23 +63,23 @@ def makeTagList(pkgdata): tags = ','.join(taglist) return tags -def packagePath(appId = None): +def packagePath(appId = None, architecture = None): path = settings.MEDIA_ROOT + 'packages/' - if appId is not None: - return path + appId + if (appId is not None) and (architecture is not None): + return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') return path -def iconPath(appId = None): +def iconPath(appId = None, architecture = None): path = settings.MEDIA_ROOT + 'icons/' - if appId is not None: - return path + appId + '.png' + if (appId is not None) and (architecture is not None): + return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') + '.png' return path -def writeTempIcon(appId, icon): +def writeTempIcon(appId, architecture, icon): try: if not os.path.exists(iconPath()): os.makedirs(iconPath()) - tempicon = open(iconPath(appId), 'w') + tempicon = open(iconPath(appId, architecture), 'w') tempicon.write(icon) tempicon.flush() tempicon.close() @@ -285,13 +285,18 @@ def parsePackageMetadata(packageFile): if fileCount > 2: if contents and entry.isfile(): # check for file type here. - filemagic = ms.from_buffer(contents) + fil = tempfile.NamedTemporaryFile() #This sequence is done to facilitate showing full type info + fil.write(contents) #libmagic refuses to give full information when called with + fil.seek(0) #from_buffer instead of from_file + filemagic = ms.from_file(fil.name) + fil.close() osarch = osandarch.getOsArch(filemagic) - if osarch: - osset.add(osarch['os']) - archset.add(osarch['arch']) - pkgfmt.add(osarch['format']) - print(entry.name, osarch) + if osarch: #[os, arch, endianness, bits, fmt] + architecture = '-'.join(osarch[1:]) + osset.add(osarch[0]) + archset.add(architecture) + pkgfmt.add(osarch[4]) + print(entry.name, osarch) # finished enumerating all files try: -- cgit v1.2.3