You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

common.h 19 kB

13 years ago
10 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
13 years ago
13 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821
  1. /*********************************************************************/
  2. /* Copyright 2009, 2010 The University of Texas at Austin. */
  3. /* All rights reserved. */
  4. /* */
  5. /* Redistribution and use in source and binary forms, with or */
  6. /* without modification, are permitted provided that the following */
  7. /* conditions are met: */
  8. /* */
  9. /* 1. Redistributions of source code must retain the above */
  10. /* copyright notice, this list of conditions and the following */
  11. /* disclaimer. */
  12. /* */
  13. /* 2. Redistributions in binary form must reproduce the above */
  14. /* copyright notice, this list of conditions and the following */
  15. /* disclaimer in the documentation and/or other materials */
  16. /* provided with the distribution. */
  17. /* */
  18. /* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
  19. /* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
  20. /* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
  21. /* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
  22. /* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
  23. /* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
  24. /* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
  25. /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
  26. /* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
  27. /* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
  28. /* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
  29. /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
  30. /* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
  31. /* POSSIBILITY OF SUCH DAMAGE. */
  32. /* */
  33. /* The views and conclusions contained in the software and */
  34. /* documentation are those of the authors and should not be */
  35. /* interpreted as representing official policies, either expressed */
  36. /* or implied, of The University of Texas at Austin. */
  37. /*********************************************************************/
  38. #ifndef COMMON_H
  39. #define COMMON_H
  40. #ifdef __cplusplus
  41. extern "C" {
  42. /* Assume C declarations for C++ */
  43. #endif /* __cplusplus */
  44. #ifndef _GNU_SOURCE
  45. #define _GNU_SOURCE
  46. #endif
  47. #ifndef __USE_XOPEN
  48. #define __USE_XOPEN
  49. #endif
  50. #ifndef __USE_SVID
  51. #define __USE_SVID
  52. #endif
  53. #ifdef BUILD_KERNEL
  54. #include "config_kernel.h"
  55. #else
  56. #include "config.h"
  57. #endif
  58. #undef ENABLE_SSE_EXCEPTION
  59. #if defined(SMP_SERVER) || defined(SMP_ONDEMAND)
  60. #define SMP
  61. #endif
  62. #if defined(OS_WINNT) || defined(OS_CYGWIN_NT) || defined(OS_INTERIX)
  63. #define WINDOWS_ABI
  64. #define OS_WINDOWS
  65. #ifdef DOUBLE
  66. #define DOUBLE_DEFINED DOUBLE
  67. #undef DOUBLE
  68. #endif
  69. #endif
  70. #if !defined(NOINCLUDE) && !defined(ASSEMBLER)
  71. #include <stdio.h>
  72. #include <stdlib.h>
  73. #include <string.h>
  74. #if !defined(_MSC_VER)
  75. #include <unistd.h>
  76. #endif
  77. #include <time.h>
  78. #ifdef OS_LINUX
  79. #include <malloc.h>
  80. #include <sched.h>
  81. #endif
  82. #if defined(OS_DARWIN) || defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_ANDROID)
  83. #include <sched.h>
  84. #endif
  85. #ifdef OS_ANDROID
  86. #define NO_SYSV_IPC
  87. //Android NDK only supports complex.h since Android 5.0
  88. #if __ANDROID_API__ < 21
  89. #define FORCE_OPENBLAS_COMPLEX_STRUCT
  90. #endif
  91. #endif
  92. #ifdef OS_WINDOWS
  93. #ifdef ATOM
  94. #define GOTO_ATOM ATOM
  95. #undef ATOM
  96. #endif
  97. #include <windows.h>
  98. #include <math.h>
  99. #ifdef GOTO_ATOM
  100. #define ATOM GOTO_ATOM
  101. #undef GOTO_ATOM
  102. #endif
  103. #else
  104. #include <sys/mman.h>
  105. #ifndef NO_SYSV_IPC
  106. #include <sys/shm.h>
  107. #endif
  108. #include <sys/time.h>
  109. #include <time.h>
  110. #include <unistd.h>
  111. #include <math.h>
  112. #ifdef SMP
  113. #include <pthread.h>
  114. #endif
  115. #endif
  116. #if defined(OS_SUNOS)
  117. #include <thread.h>
  118. #endif
  119. #ifdef __DECC
  120. #include <c_asm.h>
  121. #include <machine/builtins.h>
  122. #endif
  123. #if defined(ARCH_IA64) && defined(ENABLE_SSE_EXCEPTION)
  124. #include <fenv.h>
  125. #endif
  126. #endif
  127. #if defined(OS_WINDOWS) && defined(DOUBLE_DEFINED)
  128. #define DOUBLE DOUBLE_DEFINED
  129. #undef DOUBLE_DEFINED
  130. #endif
  131. #undef DEBUG_INFO
  132. #define SMP_DEBUG
  133. #undef MALLOC_DEBUG
  134. #undef SMP_ALLOC_DEBUG
  135. #ifndef ZERO
  136. #ifdef XDOUBLE
  137. #define ZERO 0.e0L
  138. #elif defined DOUBLE
  139. #define ZERO 0.e0
  140. #else
  141. #define ZERO 0.e0f
  142. #endif
  143. #endif
  144. #ifndef ONE
  145. #ifdef XDOUBLE
  146. #define ONE 1.e0L
  147. #elif defined DOUBLE
  148. #define ONE 1.e0
  149. #else
  150. #define ONE 1.e0f
  151. #endif
  152. #endif
  153. #define BITMASK(a, b, c) ((((a) >> (b)) & (c)))
  154. #define ALLOCA_ALIGN 63UL
  155. #define NUM_BUFFERS (MAX_CPU_NUMBER * 2)
  156. #ifdef NEEDBUNDERSCORE
  157. #define BLASFUNC(FUNC) FUNC##_
  158. #else
  159. #define BLASFUNC(FUNC) FUNC
  160. #endif
  161. #undef USE_PTHREAD_LOCK
  162. #undef USE_PTHREAD_SPINLOCK
  163. #if defined(USE_PTHREAD_LOCK) && defined(USE_PTHREAD_SPINLOCK)
  164. #error "You can't specify both LOCK operation!"
  165. #endif
  166. #ifdef SMP
  167. #define USE_PTHREAD_LOCK
  168. #undef USE_PTHREAD_SPINLOCK
  169. #endif
  170. #ifdef OS_WINDOWS
  171. #undef USE_PTHREAD_LOCK
  172. #undef USE_PTHREAD_SPINLOCK
  173. #endif
  174. #if defined(USE_PTHREAD_LOCK)
  175. #define LOCK_COMMAND(x) pthread_mutex_lock(x)
  176. #define UNLOCK_COMMAND(x) pthread_mutex_unlock(x)
  177. #elif defined(USE_PTHREAD_SPINLOCK)
  178. #ifndef ASSEMBLER
  179. typedef volatile int pthread_spinlock_t;
  180. int pthread_spin_lock (pthread_spinlock_t *__lock);
  181. int pthread_spin_unlock (pthread_spinlock_t *__lock);
  182. #endif
  183. #define LOCK_COMMAND(x) pthread_spin_lock(x)
  184. #define UNLOCK_COMMAND(x) pthread_spin_unlock(x)
  185. #else
  186. #define LOCK_COMMAND(x) blas_lock(x)
  187. #define UNLOCK_COMMAND(x) blas_unlock(x)
  188. #endif
  189. #define GOTO_SHMID 0x510510
  190. #if 0
  191. #ifndef __CUDACC__
  192. #define __global__
  193. #define __device__
  194. #define __host__
  195. #define __shared__
  196. #endif
  197. #endif
  198. #ifndef ASSEMBLER
  199. #ifdef QUAD_PRECISION
  200. typedef struct {
  201. unsigned long x[2];
  202. } xdouble;
  203. #elif defined EXPRECISION
  204. #define xdouble long double
  205. #else
  206. #define xdouble double
  207. #endif
  208. #if defined(OS_WINDOWS) && defined(__64BIT__)
  209. typedef long long BLASLONG;
  210. typedef unsigned long long BLASULONG;
  211. #else
  212. typedef long BLASLONG;
  213. typedef unsigned long BLASULONG;
  214. #endif
  215. #ifdef USE64BITINT
  216. typedef BLASLONG blasint;
  217. #else
  218. typedef int blasint;
  219. #endif
  220. #else
  221. #ifdef USE64BITINT
  222. #define INTSHIFT 3
  223. #define INTSIZE 8
  224. #else
  225. #define INTSHIFT 2
  226. #define INTSIZE 4
  227. #endif
  228. #endif
  229. #ifdef XDOUBLE
  230. #define FLOAT xdouble
  231. #ifdef QUAD_PRECISION
  232. #define XFLOAT xidouble
  233. #endif
  234. #ifdef QUAD_PRECISION
  235. #define SIZE 32
  236. #define BASE_SHIFT 5
  237. #define ZBASE_SHIFT 6
  238. #else
  239. #define SIZE 16
  240. #define BASE_SHIFT 4
  241. #define ZBASE_SHIFT 5
  242. #endif
  243. #elif defined(DOUBLE)
  244. #define FLOAT double
  245. #define SIZE 8
  246. #define BASE_SHIFT 3
  247. #define ZBASE_SHIFT 4
  248. #else
  249. #define FLOAT float
  250. #define SIZE 4
  251. #define BASE_SHIFT 2
  252. #define ZBASE_SHIFT 3
  253. #endif
  254. #ifndef XFLOAT
  255. #define XFLOAT FLOAT
  256. #endif
  257. #ifndef COMPLEX
  258. #define COMPSIZE 1
  259. #else
  260. #define COMPSIZE 2
  261. #endif
  262. #define Address_H(x) (((x)+(1<<15))>>16)
  263. #define Address_L(x) ((x)-((Address_H(x))<<16))
  264. #ifndef MAX_CPU_NUMBER
  265. #define MAX_CPU_NUMBER 2
  266. #endif
  267. #if defined(OS_SUNOS)
  268. #define YIELDING thr_yield()
  269. #endif
  270. #if defined(OS_WINDOWS)
  271. #if defined(_MSC_VER) && !defined(__clang__)
  272. #define YIELDING YieldProcessor()
  273. #else
  274. #define YIELDING SwitchToThread()
  275. #endif
  276. #endif
  277. #if defined(ARMV7) || defined(ARMV6) || defined(ARMV8) || defined(ARMV5)
  278. #define YIELDING asm volatile ("nop;nop;nop;nop;nop;nop;nop;nop; \n");
  279. #endif
  280. #ifdef BULLDOZER
  281. #ifndef YIELDING
  282. #define YIELDING __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
  283. #endif
  284. #endif
  285. /*
  286. #ifdef PILEDRIVER
  287. #ifndef YIELDING
  288. #define YIELDING __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
  289. #endif
  290. #endif
  291. */
  292. /*
  293. #ifdef STEAMROLLER
  294. #ifndef YIELDING
  295. #define YIELDING __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
  296. #endif
  297. #endif
  298. */
  299. #ifndef YIELDING
  300. #define YIELDING sched_yield()
  301. #endif
  302. /***
  303. To alloc job_t on heap or statck.
  304. please https://github.com/xianyi/OpenBLAS/issues/246
  305. ***/
  306. #if defined(OS_WINDOWS)
  307. #define GETRF_MEM_ALLOC_THRESHOLD 32
  308. #define BLAS3_MEM_ALLOC_THRESHOLD 32
  309. #endif
  310. #ifndef GETRF_MEM_ALLOC_THRESHOLD
  311. #define GETRF_MEM_ALLOC_THRESHOLD 80
  312. #endif
  313. #ifndef BLAS3_MEM_ALLOC_THRESHOLD
  314. #define BLAS3_MEM_ALLOC_THRESHOLD 160
  315. #endif
  316. #ifdef QUAD_PRECISION
  317. #include "common_quad.h"
  318. #endif
  319. #ifdef ARCH_ALPHA
  320. #include "common_alpha.h"
  321. #endif
  322. #ifdef ARCH_X86
  323. #include "common_x86.h"
  324. #endif
  325. #ifdef ARCH_X86_64
  326. #include "common_x86_64.h"
  327. #endif
  328. #ifdef ARCH_IA64
  329. #include "common_ia64.h"
  330. #endif
  331. #ifdef ARCH_POWER
  332. #include "common_power.h"
  333. #endif
  334. #ifdef sparc
  335. #include "common_sparc.h"
  336. #endif
  337. #ifdef ARCH_MIPS64
  338. #include "common_mips64.h"
  339. #endif
  340. #ifdef ARCH_ARM
  341. #include "common_arm.h"
  342. #endif
  343. #ifdef ARCH_ARM64
  344. #include "common_arm64.h"
  345. #endif
  346. #ifndef ASSEMBLER
  347. #ifdef OS_WINDOWS
  348. typedef char env_var_t[MAX_PATH];
  349. #define readenv(p, n) GetEnvironmentVariable((LPCTSTR)(n), (LPTSTR)(p), sizeof(p))
  350. #else
  351. typedef char* env_var_t;
  352. #define readenv(p, n) ((p)=getenv(n))
  353. #endif
  354. #if !defined(RPCC_DEFINED) && !defined(OS_WINDOWS)
  355. #ifdef _POSIX_MONOTONIC_CLOCK
  356. #if defined(__GLIBC_PREREQ) // cut the if condition if two lines, otherwise will fail at __GLIBC_PREREQ(2, 17)
  357. #if __GLIBC_PREREQ(2, 17) // don't require -lrt
  358. #define USE_MONOTONIC
  359. #endif
  360. #elif defined(OS_ANDROID)
  361. #define USE_MONOTONIC
  362. #endif
  363. #endif
  364. /* use similar scale as x86 rdtsc for timeouts to work correctly */
  365. static inline unsigned long long rpcc(void){
  366. #ifdef USE_MONOTONIC
  367. struct timespec ts;
  368. clock_gettime(CLOCK_MONOTONIC, &ts);
  369. return (unsigned long long)ts.tv_sec * 1000000000ull + ts.tv_nsec;
  370. #else
  371. struct timeval tv;
  372. gettimeofday(&tv,NULL);
  373. return (unsigned long long)tv.tv_sec * 1000000000ull + tv.tv_usec * 1000;
  374. #endif
  375. }
  376. #define RPCC_DEFINED
  377. #define RPCC64BIT
  378. #endif // !RPCC_DEFINED
  379. #if !defined(BLAS_LOCK_DEFINED) && defined(__GNUC__)
  380. static void __inline blas_lock(volatile BLASULONG *address){
  381. do {
  382. while (*address) {YIELDING;};
  383. } while (!__sync_bool_compare_and_swap(address, 0, 1));
  384. }
  385. #define BLAS_LOCK_DEFINED
  386. #endif
  387. #ifndef RPCC_DEFINED
  388. #error "rpcc() implementation is missing for your platform"
  389. #endif
  390. #ifndef BLAS_LOCK_DEFINED
  391. #error "blas_lock() implementation is missing for your platform"
  392. #endif
  393. #endif // !ASSEMBLER
  394. #ifdef OS_LINUX
  395. #include "common_linux.h"
  396. #endif
  397. #define MMAP_ACCESS (PROT_READ | PROT_WRITE)
  398. #ifdef __NetBSD__
  399. #define MMAP_POLICY (MAP_PRIVATE | MAP_ANON)
  400. #else
  401. #define MMAP_POLICY (MAP_PRIVATE | MAP_ANONYMOUS)
  402. #endif
  403. #include "param.h"
  404. #include "common_param.h"
  405. #ifndef STDERR
  406. #define STDERR stderr
  407. #endif
  408. #ifndef MASK
  409. #define MASK(a, b) (((a) + ((b) - 1)) & ~((b) - 1))
  410. #endif
  411. #if defined(XDOUBLE) || defined(DOUBLE)
  412. #define FLOATRET FLOAT
  413. #else
  414. #ifdef NEED_F2CCONV
  415. #define FLOATRET double
  416. #else
  417. #define FLOATRET float
  418. #endif
  419. #endif
  420. #ifndef ASSEMBLER
  421. #ifndef NOINCLUDE
  422. /* Inclusion of a standard header file is needed for definition of __STDC_*
  423. predefined macros with some compilers (e.g. GCC 4.7 on Linux). This occurs
  424. as a side effect of including either <features.h> or <stdc-predef.h>. */
  425. #include <stdio.h>
  426. #endif // NOINCLUDE
  427. /* C99 supports complex floating numbers natively, which GCC also offers as an
  428. extension since version 3.0. If neither are available, use a compatible
  429. structure as fallback (see Clause 6.2.5.13 of the C99 standard). */
  430. #if ((defined(__STDC_IEC_559_COMPLEX__) || __STDC_VERSION__ >= 199901L || \
  431. (__GNUC__ >= 3 && !defined(__cplusplus))) && !(defined(FORCE_OPENBLAS_COMPLEX_STRUCT)))
  432. #define OPENBLAS_COMPLEX_C99
  433. #ifndef __cplusplus
  434. #include <complex.h>
  435. #endif
  436. typedef float _Complex openblas_complex_float;
  437. typedef double _Complex openblas_complex_double;
  438. typedef xdouble _Complex openblas_complex_xdouble;
  439. #define openblas_make_complex_float(real, imag) ((real) + ((imag) * _Complex_I))
  440. #define openblas_make_complex_double(real, imag) ((real) + ((imag) * _Complex_I))
  441. #define openblas_make_complex_xdouble(real, imag) ((real) + ((imag) * _Complex_I))
  442. #else
  443. #define OPENBLAS_COMPLEX_STRUCT
  444. typedef struct { float real, imag; } openblas_complex_float;
  445. typedef struct { double real, imag; } openblas_complex_double;
  446. typedef struct { xdouble real, imag; } openblas_complex_xdouble;
  447. #define openblas_make_complex_float(real, imag) {(real), (imag)}
  448. #define openblas_make_complex_double(real, imag) {(real), (imag)}
  449. #define openblas_make_complex_xdouble(real, imag) {(real), (imag)}
  450. #endif
  451. #ifdef XDOUBLE
  452. #define OPENBLAS_COMPLEX_FLOAT openblas_complex_xdouble
  453. #define OPENBLAS_MAKE_COMPLEX_FLOAT(r,i) openblas_make_complex_xdouble(r,i)
  454. #elif defined(DOUBLE)
  455. #define OPENBLAS_COMPLEX_FLOAT openblas_complex_double
  456. #define OPENBLAS_MAKE_COMPLEX_FLOAT(r,i) openblas_make_complex_double(r,i)
  457. #else
  458. #define OPENBLAS_COMPLEX_FLOAT openblas_complex_float
  459. #define OPENBLAS_MAKE_COMPLEX_FLOAT(r,i) openblas_make_complex_float(r,i)
  460. #endif
  461. #if defined(C_PGI) || defined(C_SUN)
  462. #define CREAL(X) (*((FLOAT *)&X + 0))
  463. #define CIMAG(X) (*((FLOAT *)&X + 1))
  464. #else
  465. #ifdef OPENBLAS_COMPLEX_STRUCT
  466. #define CREAL(Z) ((Z).real)
  467. #define CIMAG(Z) ((Z).imag)
  468. #else
  469. #define CREAL __real__
  470. #define CIMAG __imag__
  471. #endif
  472. #endif
  473. #endif // ASSEMBLER
  474. #ifndef IFLUSH
  475. #define IFLUSH
  476. #endif
  477. #ifndef IFLUSH_HALF
  478. #define IFLUSH_HALF
  479. #endif
  480. #if defined(C_GCC) && (( __GNUC__ <= 3) || ((__GNUC__ == 4) && (__GNUC_MINOR__ < 2)))
  481. #ifdef USE_OPENMP
  482. #undef USE_OPENMP
  483. #endif
  484. #endif
  485. #if defined(C_MSVC)
  486. #define inline __inline
  487. #endif
  488. #ifndef ASSEMBLER
  489. #ifndef MIN
  490. #define MIN(a,b) (a>b? b:a)
  491. #endif
  492. #ifndef MAX
  493. #define MAX(a,b) (a<b? b:a)
  494. #endif
  495. #define TOUPPER(a) {if ((a) > 0x60) (a) -= 0x20;}
  496. #if defined(__FreeBSD__) || defined(__APPLE__)
  497. #define MAP_ANONYMOUS MAP_ANON
  498. #endif
  499. /* Common Memory Management Routine */
  500. void blas_set_parameter(void);
  501. int blas_get_cpu_number(void);
  502. void *blas_memory_alloc (int);
  503. void blas_memory_free (void *);
  504. void *blas_memory_alloc_nolock (int); //use malloc without blas_lock
  505. void blas_memory_free_nolock (void *);
  506. int get_num_procs (void);
  507. #if defined(OS_LINUX) && defined(SMP) && !defined(NO_AFFINITY)
  508. int get_num_nodes (void);
  509. int get_num_proc (int);
  510. int get_node_equal (void);
  511. #endif
  512. void goto_set_num_threads(int);
  513. void gotoblas_affinity_init(void);
  514. void gotoblas_affinity_quit(void);
  515. void gotoblas_dynamic_init(void);
  516. void gotoblas_dynamic_quit(void);
  517. void gotoblas_profile_init(void);
  518. void gotoblas_profile_quit(void);
  519. #ifdef USE_OPENMP
  520. int omp_in_parallel(void);
  521. int omp_get_num_procs(void);
  522. #else
  523. #ifdef __ELF__
  524. int omp_in_parallel (void) __attribute__ ((weak));
  525. int omp_get_num_procs(void) __attribute__ ((weak));
  526. #endif
  527. #endif
  528. static __inline void blas_unlock(volatile BLASULONG *address){
  529. MB;
  530. *address = 0;
  531. }
  532. #ifdef OS_WINDOWS
  533. static __inline int readenv_atoi(char *env) {
  534. env_var_t p;
  535. return readenv(p,env) ? 0 : atoi(p);
  536. }
  537. #else
  538. static __inline int readenv_atoi(char *env) {
  539. char *p;
  540. if (( p = getenv(env) ))
  541. return (atoi(p));
  542. else
  543. return(0);
  544. }
  545. #endif
  546. #if !defined(XDOUBLE) || !defined(QUAD_PRECISION)
  547. static __inline void compinv(FLOAT *b, FLOAT ar, FLOAT ai){
  548. #ifndef UNIT
  549. FLOAT ratio, den;
  550. if (
  551. #ifdef XDOUBLE
  552. (fabsl(ar)) >= (fabsl(ai))
  553. #elif defined DOUBLE
  554. (fabs (ar)) >= (fabs (ai))
  555. #else
  556. (fabsf(ar)) >= (fabsf(ai))
  557. #endif
  558. ) {
  559. ratio = ai / ar;
  560. den = (FLOAT)(ONE / (ar * (ONE + ratio * ratio)));
  561. ar = den;
  562. ai = -ratio * den;
  563. } else {
  564. ratio = ar / ai;
  565. den = (FLOAT)(ONE /(ai * (ONE + ratio * ratio)));
  566. ar = ratio * den;
  567. ai = -den;
  568. }
  569. b[0] = ar;
  570. b[1] = ai;
  571. #else
  572. b[0] = ONE;
  573. b[1] = ZERO;
  574. #endif
  575. }
  576. #endif
  577. #ifdef MALLOC_DEBUG
  578. void *blas_debug_alloc(int);
  579. void *blas_debug_free(void *);
  580. #undef malloc
  581. #undef free
  582. #define malloc(a) blas_debug_alloc(a)
  583. #define free(a) blas_debug_free (a)
  584. #endif
  585. #ifndef COPYOVERHEAD
  586. #define GEMMRETTYPE int
  587. #else
  588. typedef struct {
  589. double outercopy;
  590. double innercopy;
  591. double kernel;
  592. double mflops;
  593. } copyoverhead_t;
  594. #define GEMMRETTYPE copyoverhead_t
  595. #endif
  596. #endif
  597. #ifndef BUILD_KERNEL
  598. #define KNAME(A, B) A
  599. #else
  600. #define KNAME(A, B) A##B
  601. #endif
  602. #include "common_interface.h"
  603. #ifdef SANITY_CHECK
  604. #include "common_reference.h"
  605. #endif
  606. #include "common_macro.h"
  607. #include "common_level1.h"
  608. #include "common_level2.h"
  609. #include "common_level3.h"
  610. #include "common_lapack.h"
  611. #ifdef CBLAS
  612. # define OPENBLAS_CONST /* see comment in cblas.h */
  613. # include "cblas.h"
  614. #endif
  615. #ifndef ASSEMBLER
  616. #include "common_stackalloc.h"
  617. #if 0
  618. #include "symcopy.h"
  619. #endif
  620. #if defined(SMP_SERVER) && defined(SMP_ONDEMAND)
  621. #error Both SMP_SERVER and SMP_ONDEMAND are specified.
  622. #endif
  623. #if defined(SMP_SERVER) || defined(SMP_ONDEMAND)
  624. #include "common_thread.h"
  625. #endif
  626. #endif
  627. #define INFO_NUM 99
  628. #ifndef DEFAULT_CPU_NUMBER
  629. #define DEFAULT_CPU_NUMBER 4
  630. #endif
  631. #ifndef IDEBUG_START
  632. #define IDEBUG_START
  633. #endif
  634. #ifndef IDEBUG_END
  635. #define IDEBUG_END
  636. #endif
  637. #if !defined(ASSEMBLER) && defined(FUNCTION_PROFILE)
  638. typedef struct {
  639. int func;
  640. unsigned long long calls, fops, area, cycles, tcycles;
  641. } func_profile_t;
  642. extern func_profile_t function_profile_table[];
  643. extern int gotoblas_profile;
  644. #ifdef XDOUBLE
  645. #define NUMOPT QNUMOPT
  646. #elif defined DOUBLE
  647. #define NUMOPT DNUMOPT
  648. #else
  649. #define NUMOPT SNUMOPT
  650. #endif
  651. #define FUNCTION_PROFILE_START() { unsigned long long profile_start = rpcc(), profile_end;
  652. #ifdef SMP
  653. #define FUNCTION_PROFILE_END(COMP, AREA, OPS) \
  654. if (gotoblas_profile) { \
  655. profile_end = rpcc(); \
  656. function_profile_table[PROFILE_FUNC_NAME].calls ++; \
  657. function_profile_table[PROFILE_FUNC_NAME].area += SIZE * COMPSIZE * (AREA); \
  658. function_profile_table[PROFILE_FUNC_NAME].fops += (COMP) * (OPS) / NUMOPT; \
  659. function_profile_table[PROFILE_FUNC_NAME].cycles += (profile_end - profile_start); \
  660. function_profile_table[PROFILE_FUNC_NAME].tcycles += blas_cpu_number * (profile_end - profile_start); \
  661. } \
  662. }
  663. #else
  664. #define FUNCTION_PROFILE_END(COMP, AREA, OPS) \
  665. if (gotoblas_profile) { \
  666. profile_end = rpcc(); \
  667. function_profile_table[PROFILE_FUNC_NAME].calls ++; \
  668. function_profile_table[PROFILE_FUNC_NAME].area += SIZE * COMPSIZE * (AREA); \
  669. function_profile_table[PROFILE_FUNC_NAME].fops += (COMP) * (OPS) / NUMOPT; \
  670. function_profile_table[PROFILE_FUNC_NAME].cycles += (profile_end - profile_start); \
  671. function_profile_table[PROFILE_FUNC_NAME].tcycles += (profile_end - profile_start); \
  672. } \
  673. }
  674. #endif
  675. #else
  676. #define FUNCTION_PROFILE_START()
  677. #define FUNCTION_PROFILE_END(COMP, AREA, OPS)
  678. #endif
  679. #if 1
  680. #define PRINT_DEBUG_CNAME
  681. #define PRINT_DEBUG_NAME
  682. #else
  683. #define PRINT_DEBUG_CNAME if (readenv_atoi("GOTO_DEBUG")) fprintf(stderr, "GotoBLAS : %s\n", CHAR_CNAME)
  684. #define PRINT_DEBUG_NAME if (readenv_atoi("GOTO_DEBUG")) fprintf(stderr, "GotoBLAS : %s\n", CHAR_NAME)
  685. #endif
  686. #ifdef __cplusplus
  687. }
  688. #endif /* __cplusplus */
  689. #endif