1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
/*===-- __clang_cuda_complex_builtins - CUDA impls of runtime complex fns ---===
*
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
* See https://llvm.org/LICENSE.txt for license information.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*
*===-----------------------------------------------------------------------===
*/
#ifndef __CLANG_CUDA_COMPLEX_BUILTINS
#define __CLANG_CUDA_COMPLEX_BUILTINS
// This header defines __muldc3, __mulsc3, __divdc3, and __divsc3. These are
// libgcc functions that clang assumes are available when compiling c99 complex
// operations. (These implementations come from libc++, and have been modified
// to work with CUDA.)
extern "C" inline __device__ double _Complex __muldc3(double __a, double __b,
double __c, double __d) {
double __ac = __a * __c;
double __bd = __b * __d;
double __ad = __a * __d;
double __bc = __b * __c;
double _Complex z;
__real__(z) = __ac - __bd;
__imag__(z) = __ad + __bc;
if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
int __recalc = 0;
if (std::isinf(__a) || std::isinf(__b)) {
__a = std::copysign(std::isinf(__a) ? 1 : 0, __a);
__b = std::copysign(std::isinf(__b) ? 1 : 0, __b);
if (std::isnan(__c))
__c = std::copysign(0, __c);
if (std::isnan(__d))
__d = std::copysign(0, __d);
__recalc = 1;
}
if (std::isinf(__c) || std::isinf(__d)) {
__c = std::copysign(std::isinf(__c) ? 1 : 0, __c);
__d = std::copysign(std::isinf(__d) ? 1 : 0, __d);
if (std::isnan(__a))
__a = std::copysign(0, __a);
if (std::isnan(__b))
__b = std::copysign(0, __b);
__recalc = 1;
}
if (!__recalc && (std::isinf(__ac) || std::isinf(__bd) ||
std::isinf(__ad) || std::isinf(__bc))) {
if (std::isnan(__a))
__a = std::copysign(0, __a);
if (std::isnan(__b))
__b = std::copysign(0, __b);
if (std::isnan(__c))
__c = std::copysign(0, __c);
if (std::isnan(__d))
__d = std::copysign(0, __d);
__recalc = 1;
}
if (__recalc) {
// Can't use std::numeric_limits<double>::infinity() -- that doesn't have
// a device overload (and isn't constexpr before C++11, naturally).
__real__(z) = __builtin_huge_valf() * (__a * __c - __b * __d);
__imag__(z) = __builtin_huge_valf() * (__a * __d + __b * __c);
}
}
return z;
}
extern "C" inline __device__ float _Complex __mulsc3(float __a, float __b,
float __c, float __d) {
float __ac = __a * __c;
float __bd = __b * __d;
float __ad = __a * __d;
float __bc = __b * __c;
float _Complex z;
__real__(z) = __ac - __bd;
__imag__(z) = __ad + __bc;
if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
int __recalc = 0;
if (std::isinf(__a) || std::isinf(__b)) {
__a = std::copysign(std::isinf(__a) ? 1 : 0, __a);
__b = std::copysign(std::isinf(__b) ? 1 : 0, __b);
if (std::isnan(__c))
__c = std::copysign(0, __c);
if (std::isnan(__d))
__d = std::copysign(0, __d);
__recalc = 1;
}
if (std::isinf(__c) || std::isinf(__d)) {
__c = std::copysign(std::isinf(__c) ? 1 : 0, __c);
__d = std::copysign(std::isinf(__d) ? 1 : 0, __d);
if (std::isnan(__a))
__a = std::copysign(0, __a);
if (std::isnan(__b))
__b = std::copysign(0, __b);
__recalc = 1;
}
if (!__recalc && (std::isinf(__ac) || std::isinf(__bd) ||
std::isinf(__ad) || std::isinf(__bc))) {
if (std::isnan(__a))
__a = std::copysign(0, __a);
if (std::isnan(__b))
__b = std::copysign(0, __b);
if (std::isnan(__c))
__c = std::copysign(0, __c);
if (std::isnan(__d))
__d = std::copysign(0, __d);
__recalc = 1;
}
if (__recalc) {
__real__(z) = __builtin_huge_valf() * (__a * __c - __b * __d);
__imag__(z) = __builtin_huge_valf() * (__a * __d + __b * __c);
}
}
return z;
}
extern "C" inline __device__ double _Complex __divdc3(double __a, double __b,
double __c, double __d) {
int __ilogbw = 0;
// Can't use std::max, because that's defined in <algorithm>, and we don't
// want to pull that in for every compile. The CUDA headers define
// ::max(float, float) and ::max(double, double), which is sufficient for us.
double __logbw = std::logb(max(std::abs(__c), std::abs(__d)));
if (std::isfinite(__logbw)) {
__ilogbw = (int)__logbw;
__c = std::scalbn(__c, -__ilogbw);
__d = std::scalbn(__d, -__ilogbw);
}
double __denom = __c * __c + __d * __d;
double _Complex z;
__real__(z) = std::scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
__imag__(z) = std::scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
if ((__denom == 0.0) && (!std::isnan(__a) || !std::isnan(__b))) {
__real__(z) = std::copysign(__builtin_huge_valf(), __c) * __a;
__imag__(z) = std::copysign(__builtin_huge_valf(), __c) * __b;
} else if ((std::isinf(__a) || std::isinf(__b)) && std::isfinite(__c) &&
std::isfinite(__d)) {
__a = std::copysign(std::isinf(__a) ? 1.0 : 0.0, __a);
__b = std::copysign(std::isinf(__b) ? 1.0 : 0.0, __b);
__real__(z) = __builtin_huge_valf() * (__a * __c + __b * __d);
__imag__(z) = __builtin_huge_valf() * (__b * __c - __a * __d);
} else if (std::isinf(__logbw) && __logbw > 0.0 && std::isfinite(__a) &&
std::isfinite(__b)) {
__c = std::copysign(std::isinf(__c) ? 1.0 : 0.0, __c);
__d = std::copysign(std::isinf(__d) ? 1.0 : 0.0, __d);
__real__(z) = 0.0 * (__a * __c + __b * __d);
__imag__(z) = 0.0 * (__b * __c - __a * __d);
}
}
return z;
}
extern "C" inline __device__ float _Complex __divsc3(float __a, float __b,
float __c, float __d) {
int __ilogbw = 0;
float __logbw = std::logb(max(std::abs(__c), std::abs(__d)));
if (std::isfinite(__logbw)) {
__ilogbw = (int)__logbw;
__c = std::scalbn(__c, -__ilogbw);
__d = std::scalbn(__d, -__ilogbw);
}
float __denom = __c * __c + __d * __d;
float _Complex z;
__real__(z) = std::scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
__imag__(z) = std::scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
if (std::isnan(__real__(z)) && std::isnan(__imag__(z))) {
if ((__denom == 0) && (!std::isnan(__a) || !std::isnan(__b))) {
__real__(z) = std::copysign(__builtin_huge_valf(), __c) * __a;
__imag__(z) = std::copysign(__builtin_huge_valf(), __c) * __b;
} else if ((std::isinf(__a) || std::isinf(__b)) && std::isfinite(__c) &&
std::isfinite(__d)) {
__a = std::copysign(std::isinf(__a) ? 1 : 0, __a);
__b = std::copysign(std::isinf(__b) ? 1 : 0, __b);
__real__(z) = __builtin_huge_valf() * (__a * __c + __b * __d);
__imag__(z) = __builtin_huge_valf() * (__b * __c - __a * __d);
} else if (std::isinf(__logbw) && __logbw > 0 && std::isfinite(__a) &&
std::isfinite(__b)) {
__c = std::copysign(std::isinf(__c) ? 1 : 0, __c);
__d = std::copysign(std::isinf(__d) ? 1 : 0, __d);
__real__(z) = 0 * (__a * __c + __b * __d);
__imag__(z) = 0 * (__b * __c - __a * __d);
}
}
return z;
}
#endif // __CLANG_CUDA_COMPLEX_BUILTINS
|
