AVFCameraUtility: fix UB (std::sort with unfit predicate)

ResolutionPredicate defines an order on the resolutions (expressed as
QSize) of AVCaptureDeviceFormats, from smallest (width as primary,
height as secondary sort key) to largest, as the lexicographical
less-than of width and height.  A a lexicographical order over Strict
Weak Orders is a Strict Weak Order of the product type. So far, so
good.

The logical negation of a Stict Weak Order is, however, not a Strict
Weak Order (not-less-than is greater-or-equal, not greater-than), so
it cannot be used as the predicate in std::sort.

Rewrite the ResolutionPredicate as an adapter that can be used with
std::less _or_ std::greater (or even std::equal_to), piggy-backing on
std::tuple to implement the lexicographical sort for us, then replace
not2(ResolutionPredicate) with ResolutionPredicate<std::greater>.

Rename the predicate to something more apt.

This also solves the use of deprecated (and in C++20, removed)
std::not2.

Change-Id: I6f81b149e53a5b4299b188bf3ce996f638bf3334
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

1 files changed, 9 insertions, 4 deletions

diff --git a/src/plugins/avfoundation/camera/avfcamerautility.mm b/src/plugins/avfoundation/camera/avfcamerautility.mm
index 8a2254c2e..b2460e748 100644
--- a/src/plugins/avfoundation/camera/avfcamerautility.mm
+++ b/src/plugins/avfoundation/camera/avfcamerautility.mm
@@ -47,6 +47,7 @@
 #include <functional>
 #include <algorithm>
 #include <limits>
+#include <tuple>
 
 QT_BEGIN_NAMESPACE
 
@@ -85,14 +86,18 @@ inline bool qt_area_sane(const QSize &size)
            && std::numeric_limits<int>::max() / size.width() >= size.height();
 }
 
-struct ResolutionPredicate : std::binary_function<AVCaptureDeviceFormat *, AVCaptureDeviceFormat *, bool>
+template <template <typename...> class Comp> // std::less or std::greater (or std::equal_to)
+struct ByResolution
 {
     bool operator() (AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2)const
     {
         Q_ASSERT(f1 && f2);
         const QSize r1(qt_device_format_resolution(f1));
         const QSize r2(qt_device_format_resolution(f2));
-        return r1.width() < r2.width() || (r2.width() == r1.width() && r1.height() < r2.height());
+        // use std::tuple for lexicograpical sorting:
+        const Comp<std::tuple<int, int>> op = {};
+        return op(std::make_tuple(r1.width(), r1.height()),
+                  std::make_tuple(r2.width(), r2.height()));
     }
 };
 
@@ -143,7 +148,7 @@ QVector<AVCaptureDeviceFormat *> qt_unique_device_formats(AVCaptureDevice *captu
     if (!formats.size())
         return formats;
 
-    std::sort(formats.begin(), formats.end(), ResolutionPredicate());
+    std::sort(formats.begin(), formats.end(), ByResolution<std::less>());
 
     QSize size(qt_device_format_resolution(formats[0]));
     FourCharCode codec = CMVideoFormatDescriptionGetCodecType(formats[0].formatDescription);
@@ -309,7 +314,7 @@ AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevi
 
     QVector<AVCaptureDeviceFormat *>sorted(qt_unique_device_formats(captureDevice, filter));
     // Sort formats by their resolution in decreasing order:
-    std::sort(sorted.begin(), sorted.end(), std::not2(ResolutionPredicate()));
+    std::sort(sorted.begin(), sorted.end(), ByResolution<std::greater>());
     // We can use only formats with framerate ranges:
     sorted.erase(std::remove_if(sorted.begin(), sorted.end(), FormatHasNoFPSRange()), sorted.end());