Applications are now defined uniquely by appid and architecture

Change-Id: I99252e75687ae8f0383ac9ecfe68108ddcf35e2a
Reviewed-by: Robert Griebl <robert.griebl@pelagicore.com>

7 files changed, 162 insertions, 86 deletions

diff --git a/store/admin.py b/store/admin.py
index 1dbac7f..1cc06da 100644
--- a/store/admin.py
+++ b/store/admin.py
@@ -120,17 +120,17 @@ class CategoryAdmin(admin.ModelAdmin):
 
 class AppAdminForm(forms.ModelForm):
     class Meta:
-        exclude = ["id", "name", "tags", "architecture"]
+        exclude = ["appid", "name", "tags", "architecture", 'version']
 
     appId = ""
     name = ""
 
     def clean(self):
         cleaned_data = super(AppAdminForm, self).clean()
-        file = cleaned_data.get('file');
+        file = cleaned_data.get('file')
 
         # validate package
-        pkgdata = None;
+        pkgdata = None
         try:
             pkgdata = parseAndValidatePackageMetadata(file)
         except Exception as error:
@@ -140,34 +140,29 @@ class AppAdminForm(forms.ModelForm):
         self.name = pkgdata['storeName']
         self.architecture = pkgdata['architecture']
 
-        try:
-            a = App.objects.get(name__exact = self.name)
-            if a.id != pkgdata['info']['id']:
-                raise forms.ValidationError(_('Validation error: the same package name (%s) is already used for application %s' % (self.name, a.id)))
-        except App.DoesNotExist:
-            pass
-
         # check if this really is an update
-        if hasattr(self, 'instance') and self.instance.id:
-            if self.appId != self.instance.id:
-                raise forms.ValidationError(_('Validation error: an update cannot change the application id, tried to change from %s to %s' % (self.instance.id, self.appId)))
+        if hasattr(self, 'instance') and self.instance.appid:
+            if self.appId != self.instance.appid:
+                raise forms.ValidationError(_('Validation error: an update cannot change the application id, tried to change from %s to %s' % (self.instance.appid, self.appId)))
+            elif self.architecture != self.instance.architecture:
+                raise forms.ValidationError(_('Validation error: an update cannot change the application architecture from %s to %s' % (self.instance.architecture, self.architecture)))
         else:
             try:
-                if App.objects.get(id__exact = self.appId):
-                    raise forms.ValidationError(_('Validation error: another application with id %s already exists' % str(self.appId)))
+                if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture):
+                    raise forms.ValidationError(_('Validation error: another application with id %s and architecture %s already exists' % (str(self.appId), str(self.architecture))))
             except App.DoesNotExist:
                 pass
 
         # write icon into file to serve statically
-        success, error = writeTempIcon(self.appId, pkgdata['icon'])
+        success, error = writeTempIcon(self.appId, self.architecture, pkgdata['icon'])
         if not success:
             raise forms.ValidationError(_(error))
 
         return cleaned_data
 
     def save(self, commit=False):
-        m = super(AppAdminForm, self).save(commit);
-        m.id = self.appId
+        m = super(AppAdminForm, self).save(commit)
+        m.appid = self.appId
         m.name = self.name
         m.architecture = self.architecture
 
@@ -179,7 +174,7 @@ class AppAdminForm(forms.ModelForm):
 
 class AppAdmin(admin.ModelAdmin):
     form = AppAdminForm
-    list_display = ('name', 'id', 'architecture')
+    list_display = ('name', 'appid', 'architecture', 'version')
 
     def save_model(self, request, obj, form, change):
         obj.save()
diff --git a/store/api.py b/store/api.py
index fc3a18e..5656857 100644
--- a/store/api.py
+++ b/store/api.py
@@ -36,7 +36,7 @@ import shutil
 import json
 
 from django.conf import settings
-from django.db.models import Q
+from django.db.models import Q, Count
 from django.http import HttpResponse, HttpResponseForbidden, Http404, JsonResponse
 from django.contrib import auth
 from django.template import Context, loader
@@ -125,14 +125,32 @@ def appList(request):
         for i in request.session['conflicts_tag']:
             regex = '(^|,)%s(,|$)' % (i,)
             apps = apps.filter(~Q(tags__regex = regex))
-    if 'architecture' in request.session:
-        apps = apps.filter(Q(architecture__exact = request.session['architecture'])|Q(architecture__exact = 'All'))
-    else:
-        apps = apps.filter(Q(architecture__exact = 'All'))
 
-    appList = list(apps.values('id', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture').order_by('id'))
+    # Here goes the logic of listing packages when multiple architectures are available
+    # in /hello request, the target architecture is stored in the session. By definition target machine can support
+    # both "All" package architecture and it's native one.
+    # So - here goes filtering by list of architectures
+    archlist = ['All', ]
+    if 'architecture' in request.session:
+        archlist.append(request.session['architecture'])
+    apps = apps.filter(architecture__in = archlist)
+
+    # After filtering, there are potential duplicates in list. And we should prefer native applications to pure qml ones
+    # due to speedups offered.
+    # So - first applications are grouped by appid and numbers of same appids counted. In case where is more than one appid -
+    # there are two variants of application: for 'All' architecture, and for the architecture supported by the target machine.
+    # So, as native apps should be preferred
+    duplicates = (
+        apps.values('appid').order_by().annotate(count_id=Count('id')).filter(count_id__gt=1)
+    )
+    # Here we go over duplicates list and filter out 'All' architecture apps.
+    for duplicate in duplicates:
+        apps = apps.filter(~Q(appid__exact = duplicate['appid'], architecture__exact = 'All')) # if there is native - 'All' architecture apps are excluded
+
+    appList = list(apps.values('appid', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture', 'version').order_by('appid','architecture'))
 
     for app in appList:
+        app['id'] = app['appid']
         app['category_id'] = app['category']
         app['category'] = Category.objects.all().filter(id__exact = app['category_id'])[0].name
         app['vendor'] = app['vendor__name']
@@ -141,23 +159,32 @@ def appList(request):
         else:
             app['tags'] = []
         del app['vendor__name']
+        del app['appid']
 
     # this is not valid JSON, since we are returning a list!
     return JsonResponse(appList, safe = False)
 
 
 def appDescription(request):
+    archlist = ['All', ]
+    if 'architecture' in request.session:
+        archlist.append(request.session['architecture'])
     try:
-        app = App.objects.get(id__exact = request.REQUEST['id'])
+        app = App.objects.get(appid__exact = request.REQUEST['id'], architecture__in = archlist).order_by('architecture')
+        app = app.last()
         return HttpResponse(app.description)
     except:
         raise Http404('no such application: %s' % request.REQUEST['id'])
 
 
 def appIcon(request):
+    archlist = ['All', ]
+    if 'architecture' in request.session:
+        archlist.append(request.session['architecture'])
     try:
-        app = App.objects.get(id__exact = request.REQUEST['id'])
-        with open(iconPath(app.id), 'rb') as iconPng:
+        app = App.objects.filter(appid__exact = request.REQUEST['id'], architecture__in = archlist).order_by('architecture')
+        app = app.last()
+        with open(iconPath(app.appid,app.architecture), 'rb') as iconPng:
             response = HttpResponse(content_type = 'image/png')
             response.write(iconPng.read())
             return response
@@ -168,7 +195,9 @@ def appIcon(request):
 def appPurchase(request):
     if not request.user.is_authenticated():
         return HttpResponseForbidden('no login')
-
+    archlist = ['All', ]
+    if 'architecture' in request.session:
+        archlist.append(request.session['architecture'])
     try:
         deviceId = str(request.REQUEST.get("device_id", ""))
         if settings.APPSTORE_BIND_TO_DEVICE_ID:
@@ -177,12 +206,13 @@ def appPurchase(request):
         else:
             deviceId = ''
 
-        app = App.objects.get(id__exact = request.REQUEST['id'])
-        fromFilePath = packagePath(app.id)
+        app = App.objects.filter(appid__exact = request.REQUEST['id'], architecture__in=archlist).order_by('architecture')
+        app = app.last()
+        fromFilePath = packagePath(app.appid, app.architecture)
 
         # we should not use obvious names here, but just hash the string.
         # this would be a nightmare to debug though and this is a development server :)
-        toFile = str(app.id) + '_' + str(request.user.id) + '_' + str(deviceId) + '.appkg'
+        toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(deviceId) + '.appkg'
         toPath = downloadPath()
         if not os.path.exists(toPath):
             os.makedirs(toPath)
@@ -229,7 +259,7 @@ def categoryIcon(request):
     # there are no category icons (yet), so we just return the icon of the first app in this category
     try:
         app = App.objects.filter(category__exact = request.REQUEST['id']).order_by('-dateModified')[0] #FIXME - the category icon is unimplemented
-        with open(iconPath(app.id), 'rb') as iconPng:
+        with open(iconPath(app.appid,app.architecture), 'rb') as iconPng:
             response.write(iconPng.read())
     except:
         emptyPng = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x01\x03\x00\x00\x00%\xdbV\xca\x00\x00\x00\x03PLTE\x00\x00\x00\xa7z=\xda\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\nIDAT\x08\xd7c`\x00\x00\x00\x02\x00\x01\xe2!\xbc3\x00\x00\x00\x00IEND\xaeB`\x82'
diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py
index 8a174ce..facf15e 100644
--- a/store/management/commands/store-upload-package.py
+++ b/store/management/commands/store-upload-package.py
@@ -88,40 +88,32 @@ class Command(BaseCommand):
         description = options['description']
         tags = makeTagList(pkgdata)
 
-        try:
-            a = App.objects.get(name__exact=name)
-            if a.id != pkgdata['info']['id']:
-                raise CommandError(
-                    'Validation error: the same package name (%s) is already used for application %s' % (
-                        name, a.id))
-        except App.DoesNotExist:
-            pass
-
-        success, error = writeTempIcon(appId,pkgdata['icon'])
+        success, error = writeTempIcon(appId, architecture, pkgdata['icon'])
         if not success:
             raise CommandError(error)
 
         exists = False
         app = None
         try:
-            app = App.objects.get(id__exact=appId)
+            app = App.objects.get(appid__exact=appId, architecture__exact= architecture)
             exists = True
         except App.DoesNotExist:
             pass
 
         if exists:
+            app.appid = appId
             app.category = category[0]
             app.vendor = vendor[0]
             app.name = name
             app.tags = tags
             app.description = app.briefDescription = description
             app.architecture = architecture
-            app.file.save(packagePath(appId), ContentFile(packagefile.read()))
+            app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read()))
             app.save()
         else:
             app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor[0],
-                                                     category=category[0], id=appId,
+                                                     category=category[0], appid=appId,
                                                      briefDescription=description, description=description,
                                                      architecture=architecture)
-            app.file.save(packagePath(appId), ContentFile(packagefile.read()))
+            app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read()))
             app.save()
diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py
index fc6dabe..772b1d2 100644
--- a/store/migrations/0001_initial.py
+++ b/store/migrations/0001_initial.py
@@ -47,7 +47,8 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='App',
             fields=[
-                ('id', models.CharField(max_length=200, serialize=False, primary_key=True)),
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('appid', models.CharField(max_length=200)),
                 ('name', models.CharField(max_length=200)),
                 ('file', models.FileField(storage=store.models.OverwriteStorage(), upload_to=store.models.content_file_name)),
                 ('briefDescription', models.TextField()),
@@ -56,6 +57,7 @@ class Migration(migrations.Migration):
                 ('dateModified', models.DateField(auto_now=True)),
                 ('tags', models.TextField(blank=True)),
                 ('architecture', models.CharField(default=b'All', max_length=20)),
+                ('version', models.CharField(default=b'0.0.0', max_length=20)),
             ],
             options={
             },
@@ -95,4 +97,8 @@ class Migration(migrations.Migration):
             field=models.ForeignKey(to='store.Vendor'),
             preserve_default=True,
         ),
+        migrations.AlterUniqueTogether(
+            name='app',
+            unique_together=set([('appid', 'architecture')]),
+        ),
     ]
diff --git a/store/models.py b/store/models.py
index 02de85d..802d141 100644
--- a/store/models.py
+++ b/store/models.py
@@ -110,10 +110,10 @@ class OverwriteStorage(FileSystemStorage):
         return name
 
 def content_file_name(instance, filename):
-    return packagePath(instance.id)
+    return packagePath(instance.appid, instance.architecture)
 
 class App(models.Model):
-    id = models.CharField(max_length = 200, primary_key=True)
+    appid = models.CharField(max_length = 200)
     name = models.CharField(max_length = 200)
     file = models.FileField(upload_to = content_file_name, storage = OverwriteStorage())
     vendor = models.ForeignKey(Vendor)
@@ -124,13 +124,18 @@ class App(models.Model):
     dateModified = models.DateField(auto_now = True)
     tags = models.TextField(blank=True)
     architecture = models.CharField(max_length=20, default='All')
+    version = models.CharField(max_length=20, default='0.0.0')
+
+    class Meta:
+        """Makes the group of id and arch - a unique identifier"""
+        unique_together = (('appid', 'architecture', ),)
 
     def __unicode__(self):
-        return self.name + " [" + self.id + "]"
+        return self.name + " [" + " ".join([self.appid,self.version,self.architecture]) + "]"
 
     def save(self, *args, **kwargs):
         try:
-            this = App.objects.get(id=self.id)
+            this = App.objects.get(appid=self.appid,architecture=self.architecture)
             if this.file != self.file:
                 this.file.delete(save=False)
         except:
diff --git a/store/osandarch.py b/store/osandarch.py
index 5e7c3a1..f3aa388 100644
--- a/store/osandarch.py
+++ b/store/osandarch.py
@@ -42,35 +42,78 @@
 # PE32+ executable (DLL) (GUI) x86-64, for MS Windows
 # PE32+ executable (GUI) x86-64, for MS Windows
 
+def parseMachO(str):  # os, arch, bits, endianness
+    if " universal " in str:
+        # Universal binary - not supported
+        raise Exception("Universal binaries are not supported in packages")
+    os = "macOS"
+    arch = str.split(' ')
+    arch = arch[2]
+    bits = str.split(' ')[1].replace('-bit', '')
+    endianness = "lsb"
+    return [os, arch, bits, endianness]
+
+
+def parsePE32(str):
+    os = "Windows"
+    arch = str.split(',')
+    arch = arch[0]  # Take first part
+    arch = arch.split(' ')
+    arch = arch[-1]  # Take last element
+    bits = '32'
+    if arch == 'x86-64':
+        bits = '64'
+    if arch == '80386':
+        arch = 'i386'
+    endianness = "lsb"
+    return [os, arch, bits, endianness]
+
+
+def parseElfArch(str, architecture):
+    architecture = architecture.strip()
+    if architecture.startswith("ARM"):
+        if 'aarch64' in architecture:
+            return 'aarch64'
+        if 'armhf' in str:  # this does not work for some reason - from_file() returns longer data than from_buffer() - needs fix
+            return 'armhf'
+    elif architecture.startswith("Intel"):
+        if '80386' in architecture:
+            return 'i386'
+    elif architecture.startswith("IBM S/390"):
+        return 's/390'
+    elif "PowerPC" in architecture:
+        return 'powerpc'
+    return architecture.lower()
+
+
+def parseElf(str):
+    os = "Linux"
+    arch = str.split(',')
+    arch = arch[1]
+    arch = parseElfArch(str, arch)
+    bits = str.split(' ')[1].replace('-bit', '')
+    endianness = str.split(' ')[2].lower()
+    return [os, arch, bits, endianness]
+
+
 def getOsArch(str):
     os = None
     arch = None
+    bits = None
+    endianness = None
     fmt = None
     if str.startswith("ELF "):
-        os = "Linux"
-        arch = str.split(',')
-        arch = arch[1]
         fmt = "elf"
+        os, arch, bits, endianness = parseElf(str)
     elif str.startswith("Mach-O "):
-        os = "macOS"
-        if " universal " in str:
-            # Universal binary - not supported
-            raise Exception("Universal binaries are not supported in packages")
-        else:
-            arch = str.split(' ')
-            arch = arch[2]
-            fmt = "mach_o"
+        fmt = "mach_o"
+        os, arch, bits, endianness = parseMachO(str)
     elif str.startswith("PE32+ ") or str.startswith("PE32 "):
-        os = "Windows"
-        arch = str.split(',')
-        arch = arch[0]  # Take first part
-        arch = arch.split(' ')
-        arch = arch[-1]  # Take last element
         fmt = "pe32"
+        os, arch, bits, endianness = parsePE32(str)
     if arch:
-        arch = arch.replace('_', '-')
-    result = {'os': os, 'arch': arch, 'format': fmt }
+        arch = arch.replace('-', '_')
+    result = [os, arch, endianness, bits, fmt]
     if os:
         return result
-    else:
-        return None
+    return None
diff --git a/store/utilities.py b/store/utilities.py
index b571d79..bce9a7b 100644
--- a/store/utilities.py
+++ b/store/utilities.py
@@ -63,23 +63,23 @@ def makeTagList(pkgdata):
     tags = ','.join(taglist)
     return tags
 
-def packagePath(appId = None):
+def packagePath(appId = None, architecture = None):
     path = settings.MEDIA_ROOT + 'packages/'
-    if appId is not None:
-        return path + appId
+    if (appId is not None) and (architecture is not None):
+        return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_')
     return path
 
-def iconPath(appId = None):
+def iconPath(appId = None, architecture = None):
     path = settings.MEDIA_ROOT + 'icons/'
-    if appId is not None:
-        return path + appId + '.png'
+    if (appId is not None) and (architecture is not None):
+        return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') + '.png'
     return path
 
-def writeTempIcon(appId, icon):
+def writeTempIcon(appId, architecture, icon):
     try:
         if not os.path.exists(iconPath()):
             os.makedirs(iconPath())
-        tempicon = open(iconPath(appId), 'w')
+        tempicon = open(iconPath(appId, architecture), 'w')
         tempicon.write(icon)
         tempicon.flush()
         tempicon.close()
@@ -285,13 +285,18 @@ def parsePackageMetadata(packageFile):
         if fileCount > 2:
             if contents and entry.isfile():
                 # check for file type here.
-                filemagic = ms.from_buffer(contents)
+                fil = tempfile.NamedTemporaryFile() #This sequence is done to facilitate showing full type info
+                fil.write(contents)                 #libmagic refuses to give full information when called with
+                fil.seek(0)                         #from_buffer instead of from_file
+                filemagic = ms.from_file(fil.name)
+                fil.close()
                 osarch = osandarch.getOsArch(filemagic)
-                if osarch:
-                    osset.add(osarch['os'])
-                    archset.add(osarch['arch'])
-                    pkgfmt.add(osarch['format'])
-                print(entry.name, osarch)
+                if osarch: #[os, arch, endianness, bits, fmt]
+                    architecture = '-'.join(osarch[1:])
+                    osset.add(osarch[0])
+                    archset.add(architecture)
+                    pkgfmt.add(osarch[4])
+                    print(entry.name, osarch)
 
     # finished enumerating all files
     try: