LLVM OpenMP* Runtime Library
z_Linux_util.cpp
1/*
2 * z_Linux_util.cpp -- platform specific routines.
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#include "kmp.h"
14#include "kmp_affinity.h"
15#include "kmp_i18n.h"
16#include "kmp_io.h"
17#include "kmp_itt.h"
18#include "kmp_lock.h"
19#include "kmp_stats.h"
20#include "kmp_str.h"
21#include "kmp_wait_release.h"
22#include "kmp_wrapper_getpid.h"
23
24#if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25#include <alloca.h>
26#endif
27#include <math.h> // HUGE_VAL.
28#if KMP_OS_LINUX
29#include <semaphore.h>
30#endif // KMP_OS_LINUX
31#include <sys/resource.h>
32#include <sys/syscall.h>
33#include <sys/time.h>
34#include <sys/times.h>
35#include <unistd.h>
36
37#if KMP_OS_LINUX
38#include <sys/sysinfo.h>
39#if KMP_USE_FUTEX
40// We should really include <futex.h>, but that causes compatibility problems on
41// different Linux* OS distributions that either require that you include (or
42// break when you try to include) <pci/types.h>. Since all we need is the two
43// macros below (which are part of the kernel ABI, so can't change) we just
44// define the constants here and don't include <futex.h>
45#ifndef FUTEX_WAIT
46#define FUTEX_WAIT 0
47#endif
48#ifndef FUTEX_WAKE
49#define FUTEX_WAKE 1
50#endif
51#endif
52#elif KMP_OS_DARWIN
53#include <mach/mach.h>
54#include <sys/sysctl.h>
55#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
56#include <sys/types.h>
57#include <sys/sysctl.h>
58#include <sys/user.h>
59#include <pthread_np.h>
60#elif KMP_OS_NETBSD || KMP_OS_OPENBSD
61#include <sys/types.h>
62#include <sys/sysctl.h>
63#endif
64
65#include <ctype.h>
66#include <dirent.h>
67#include <fcntl.h>
68
69struct kmp_sys_timer {
70 struct timespec start;
71};
72
73// Convert timespec to nanoseconds.
74#define TS2NS(timespec) \
75 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
76
77static struct kmp_sys_timer __kmp_sys_timer_data;
78
79#if KMP_HANDLE_SIGNALS
80typedef void (*sig_func_t)(int);
81STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
82static sigset_t __kmp_sigset;
83#endif
84
85static int __kmp_init_runtime = FALSE;
86
87static int __kmp_fork_count = 0;
88
89static pthread_condattr_t __kmp_suspend_cond_attr;
90static pthread_mutexattr_t __kmp_suspend_mutex_attr;
91
92static kmp_cond_align_t __kmp_wait_cv;
93static kmp_mutex_align_t __kmp_wait_mx;
94
95kmp_uint64 __kmp_ticks_per_msec = 1000000;
96
97#ifdef DEBUG_SUSPEND
98static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
99 KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
100 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
101 cond->c_cond.__c_waiting);
102}
103#endif
104
105#if ((KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED)
106
107/* Affinity support */
108
109void __kmp_affinity_bind_thread(int which) {
110 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
111 "Illegal set affinity operation when not capable");
112
113 kmp_affin_mask_t *mask;
114 KMP_CPU_ALLOC_ON_STACK(mask);
115 KMP_CPU_ZERO(mask);
116 KMP_CPU_SET(which, mask);
117 __kmp_set_system_affinity(mask, TRUE);
118 KMP_CPU_FREE_FROM_STACK(mask);
119}
120
121/* Determine if we can access affinity functionality on this version of
122 * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
123 * __kmp_affin_mask_size to the appropriate value (0 means not capable). */
124void __kmp_affinity_determine_capable(const char *env_var) {
125 // Check and see if the OS supports thread affinity.
126
127#if KMP_OS_LINUX
128#define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
129#define KMP_CPU_SET_TRY_SIZE CACHE_LINE
130#elif KMP_OS_FREEBSD
131#define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
132#endif
133
134#if KMP_OS_LINUX
135 long gCode;
136 unsigned char *buf;
137 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
138 int verbose = __kmp_affinity.flags.verbose;
139 int warnings = __kmp_affinity.flags.warnings;
140 enum affinity_type type = __kmp_affinity.type;
141
142 // If the syscall returns a suggestion for the size,
143 // then we don't have to search for an appropriate size.
144 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
145 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
146 "initial getaffinity call returned %ld errno = %d\n",
147 gCode, errno));
148
149 if (gCode < 0 && errno != EINVAL) {
150 // System call not supported
151 if (verbose ||
152 (warnings && (type != affinity_none) && (type != affinity_default) &&
153 (type != affinity_disabled))) {
154 int error = errno;
155 kmp_msg_t err_code = KMP_ERR(error);
156 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
157 err_code, __kmp_msg_null);
158 if (__kmp_generate_warnings == kmp_warnings_off) {
159 __kmp_str_free(&err_code.str);
160 }
161 }
162 KMP_AFFINITY_DISABLE();
163 KMP_INTERNAL_FREE(buf);
164 return;
165 } else if (gCode > 0) {
166 // The optimal situation: the OS returns the size of the buffer it expects.
167 KMP_AFFINITY_ENABLE(gCode);
168 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
169 "affinity supported (mask size %d)\n",
170 (int)__kmp_affin_mask_size));
171 KMP_INTERNAL_FREE(buf);
172 return;
173 }
174
175 // Call the getaffinity system call repeatedly with increasing set sizes
176 // until we succeed, or reach an upper bound on the search.
177 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
178 "searching for proper set size\n"));
179 int size;
180 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
181 gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
182 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
183 "getaffinity for mask size %ld returned %ld errno = %d\n",
184 size, gCode, errno));
185
186 if (gCode < 0) {
187 if (errno == ENOSYS) {
188 // We shouldn't get here
189 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
190 "inconsistent OS call behavior: errno == ENOSYS for mask "
191 "size %d\n",
192 size));
193 if (verbose ||
194 (warnings && (type != affinity_none) &&
195 (type != affinity_default) && (type != affinity_disabled))) {
196 int error = errno;
197 kmp_msg_t err_code = KMP_ERR(error);
198 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
199 err_code, __kmp_msg_null);
200 if (__kmp_generate_warnings == kmp_warnings_off) {
201 __kmp_str_free(&err_code.str);
202 }
203 }
204 KMP_AFFINITY_DISABLE();
205 KMP_INTERNAL_FREE(buf);
206 return;
207 }
208 continue;
209 }
210
211 KMP_AFFINITY_ENABLE(gCode);
212 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
213 "affinity supported (mask size %d)\n",
214 (int)__kmp_affin_mask_size));
215 KMP_INTERNAL_FREE(buf);
216 return;
217 }
218#elif KMP_OS_FREEBSD
219 long gCode;
220 unsigned char *buf;
221 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
222 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
223 reinterpret_cast<cpuset_t *>(buf));
224 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
225 "initial getaffinity call returned %d errno = %d\n",
226 gCode, errno));
227 if (gCode == 0) {
228 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
229 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
230 "affinity supported (mask size %d)\n",
231 (int)__kmp_affin_mask_size));
232 KMP_INTERNAL_FREE(buf);
233 return;
234 }
235#endif
236 KMP_INTERNAL_FREE(buf);
237
238 // Affinity is not supported
239 KMP_AFFINITY_DISABLE();
240 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
241 "cannot determine mask size - affinity not supported\n"));
242 if (verbose || (warnings && (type != affinity_none) &&
243 (type != affinity_default) && (type != affinity_disabled))) {
244 KMP_WARNING(AffCantGetMaskSize, env_var);
245 }
246}
247
248#endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
249
250#if KMP_USE_FUTEX
251
252int __kmp_futex_determine_capable() {
253 int loc = 0;
254 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
255 int retval = (rc == 0) || (errno != ENOSYS);
256
257 KA_TRACE(10,
258 ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
259 KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
260 retval ? "" : " not"));
261
262 return retval;
263}
264
265#endif // KMP_USE_FUTEX
266
267#if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (!KMP_ASM_INTRINS)
268/* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
269 use compare_and_store for these routines */
270
271kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
272 kmp_int8 old_value, new_value;
273
274 old_value = TCR_1(*p);
275 new_value = old_value | d;
276
277 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
278 KMP_CPU_PAUSE();
279 old_value = TCR_1(*p);
280 new_value = old_value | d;
281 }
282 return old_value;
283}
284
285kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
286 kmp_int8 old_value, new_value;
287
288 old_value = TCR_1(*p);
289 new_value = old_value & d;
290
291 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
292 KMP_CPU_PAUSE();
293 old_value = TCR_1(*p);
294 new_value = old_value & d;
295 }
296 return old_value;
297}
298
299kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
300 kmp_uint32 old_value, new_value;
301
302 old_value = TCR_4(*p);
303 new_value = old_value | d;
304
305 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
306 KMP_CPU_PAUSE();
307 old_value = TCR_4(*p);
308 new_value = old_value | d;
309 }
310 return old_value;
311}
312
313kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
314 kmp_uint32 old_value, new_value;
315
316 old_value = TCR_4(*p);
317 new_value = old_value & d;
318
319 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
320 KMP_CPU_PAUSE();
321 old_value = TCR_4(*p);
322 new_value = old_value & d;
323 }
324 return old_value;
325}
326
327#if KMP_ARCH_X86
328kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
329 kmp_int8 old_value, new_value;
330
331 old_value = TCR_1(*p);
332 new_value = old_value + d;
333
334 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
335 KMP_CPU_PAUSE();
336 old_value = TCR_1(*p);
337 new_value = old_value + d;
338 }
339 return old_value;
340}
341
342kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
343 kmp_int64 old_value, new_value;
344
345 old_value = TCR_8(*p);
346 new_value = old_value + d;
347
348 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
349 KMP_CPU_PAUSE();
350 old_value = TCR_8(*p);
351 new_value = old_value + d;
352 }
353 return old_value;
354}
355#endif /* KMP_ARCH_X86 */
356
357kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
358 kmp_uint64 old_value, new_value;
359
360 old_value = TCR_8(*p);
361 new_value = old_value | d;
362 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
363 KMP_CPU_PAUSE();
364 old_value = TCR_8(*p);
365 new_value = old_value | d;
366 }
367 return old_value;
368}
369
370kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
371 kmp_uint64 old_value, new_value;
372
373 old_value = TCR_8(*p);
374 new_value = old_value & d;
375 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
376 KMP_CPU_PAUSE();
377 old_value = TCR_8(*p);
378 new_value = old_value & d;
379 }
380 return old_value;
381}
382
383#endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
384
385void __kmp_terminate_thread(int gtid) {
386 int status;
387 kmp_info_t *th = __kmp_threads[gtid];
388
389 if (!th)
390 return;
391
392#ifdef KMP_CANCEL_THREADS
393 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
394 status = pthread_cancel(th->th.th_info.ds.ds_thread);
395 if (status != 0 && status != ESRCH) {
396 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
397 __kmp_msg_null);
398 }
399#endif
400 KMP_YIELD(TRUE);
401} //
402
403/* Set thread stack info according to values returned by pthread_getattr_np().
404 If values are unreasonable, assume call failed and use incremental stack
405 refinement method instead. Returns TRUE if the stack parameters could be
406 determined exactly, FALSE if incremental refinement is necessary. */
407static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
408 int stack_data;
409#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
410 KMP_OS_HURD
411 pthread_attr_t attr;
412 int status;
413 size_t size = 0;
414 void *addr = 0;
415
416 /* Always do incremental stack refinement for ubermaster threads since the
417 initial thread stack range can be reduced by sibling thread creation so
418 pthread_attr_getstack may cause thread gtid aliasing */
419 if (!KMP_UBER_GTID(gtid)) {
420
421 /* Fetch the real thread attributes */
422 status = pthread_attr_init(&attr);
423 KMP_CHECK_SYSFAIL("pthread_attr_init", status);
424#if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
425 status = pthread_attr_get_np(pthread_self(), &attr);
426 KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
427#else
428 status = pthread_getattr_np(pthread_self(), &attr);
429 KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
430#endif
431 status = pthread_attr_getstack(&attr, &addr, &size);
432 KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
433 KA_TRACE(60,
434 ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
435 " %lu, low addr: %p\n",
436 gtid, size, addr));
437 status = pthread_attr_destroy(&attr);
438 KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
439 }
440
441 if (size != 0 && addr != 0) { // was stack parameter determination successful?
442 /* Store the correct base and size */
443 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
444 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
445 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
446 return TRUE;
447 }
448#endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
449 || KMP_OS_HURD */
450 /* Use incremental refinement starting from initial conservative estimate */
451 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
452 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
453 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
454 return FALSE;
455}
456
457static void *__kmp_launch_worker(void *thr) {
458 int status, old_type, old_state;
459#ifdef KMP_BLOCK_SIGNALS
460 sigset_t new_set, old_set;
461#endif /* KMP_BLOCK_SIGNALS */
462 void *exit_val;
463#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
464 KMP_OS_OPENBSD || KMP_OS_HURD
465 void *volatile padding = 0;
466#endif
467 int gtid;
468
469 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
470 __kmp_gtid_set_specific(gtid);
471#ifdef KMP_TDATA_GTID
472 __kmp_gtid = gtid;
473#endif
474#if KMP_STATS_ENABLED
475 // set thread local index to point to thread-specific stats
476 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
477 __kmp_stats_thread_ptr->startLife();
478 KMP_SET_THREAD_STATE(IDLE);
480#endif
481
482#if USE_ITT_BUILD
483 __kmp_itt_thread_name(gtid);
484#endif /* USE_ITT_BUILD */
485
486#if KMP_AFFINITY_SUPPORTED
487 __kmp_affinity_set_init_mask(gtid, FALSE);
488#endif
489
490#ifdef KMP_CANCEL_THREADS
491 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
492 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
493 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
494 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
495 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
496#endif
497
498#if KMP_ARCH_X86 || KMP_ARCH_X86_64
499 // Set FP control regs to be a copy of the parallel initialization thread's.
500 __kmp_clear_x87_fpu_status_word();
501 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
502 __kmp_load_mxcsr(&__kmp_init_mxcsr);
503#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
504
505#ifdef KMP_BLOCK_SIGNALS
506 status = sigfillset(&new_set);
507 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
508 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
509 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
510#endif /* KMP_BLOCK_SIGNALS */
511
512#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
513 KMP_OS_OPENBSD
514 if (__kmp_stkoffset > 0 && gtid > 0) {
515 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
516 (void)padding;
517 }
518#endif
519
520 KMP_MB();
521 __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
522
523 __kmp_check_stack_overlap((kmp_info_t *)thr);
524
525 exit_val = __kmp_launch_thread((kmp_info_t *)thr);
526
527#ifdef KMP_BLOCK_SIGNALS
528 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
529 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
530#endif /* KMP_BLOCK_SIGNALS */
531
532 return exit_val;
533}
534
535#if KMP_USE_MONITOR
536/* The monitor thread controls all of the threads in the complex */
537
538static void *__kmp_launch_monitor(void *thr) {
539 int status, old_type, old_state;
540#ifdef KMP_BLOCK_SIGNALS
541 sigset_t new_set;
542#endif /* KMP_BLOCK_SIGNALS */
543 struct timespec interval;
544
545 KMP_MB(); /* Flush all pending memory write invalidates. */
546
547 KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
548
549 /* register us as the monitor thread */
550 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
551#ifdef KMP_TDATA_GTID
552 __kmp_gtid = KMP_GTID_MONITOR;
553#endif
554
555 KMP_MB();
556
557#if USE_ITT_BUILD
558 // Instruct Intel(R) Threading Tools to ignore monitor thread.
559 __kmp_itt_thread_ignore();
560#endif /* USE_ITT_BUILD */
561
562 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
563 (kmp_info_t *)thr);
564
565 __kmp_check_stack_overlap((kmp_info_t *)thr);
566
567#ifdef KMP_CANCEL_THREADS
568 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
569 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
570 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
571 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
572 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
573#endif
574
575#if KMP_REAL_TIME_FIX
576 // This is a potential fix which allows application with real-time scheduling
577 // policy work. However, decision about the fix is not made yet, so it is
578 // disabled by default.
579 { // Are program started with real-time scheduling policy?
580 int sched = sched_getscheduler(0);
581 if (sched == SCHED_FIFO || sched == SCHED_RR) {
582 // Yes, we are a part of real-time application. Try to increase the
583 // priority of the monitor.
584 struct sched_param param;
585 int max_priority = sched_get_priority_max(sched);
586 int rc;
587 KMP_WARNING(RealTimeSchedNotSupported);
588 sched_getparam(0, &param);
589 if (param.sched_priority < max_priority) {
590 param.sched_priority += 1;
591 rc = sched_setscheduler(0, sched, &param);
592 if (rc != 0) {
593 int error = errno;
594 kmp_msg_t err_code = KMP_ERR(error);
595 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
596 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
597 if (__kmp_generate_warnings == kmp_warnings_off) {
598 __kmp_str_free(&err_code.str);
599 }
600 }
601 } else {
602 // We cannot abort here, because number of CPUs may be enough for all
603 // the threads, including the monitor thread, so application could
604 // potentially work...
605 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
606 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
607 __kmp_msg_null);
608 }
609 }
610 // AC: free thread that waits for monitor started
611 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
612 }
613#endif // KMP_REAL_TIME_FIX
614
615 KMP_MB(); /* Flush all pending memory write invalidates. */
616
617 if (__kmp_monitor_wakeups == 1) {
618 interval.tv_sec = 1;
619 interval.tv_nsec = 0;
620 } else {
621 interval.tv_sec = 0;
622 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
623 }
624
625 KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
626
627 while (!TCR_4(__kmp_global.g.g_done)) {
628 struct timespec now;
629 struct timeval tval;
630
631 /* This thread monitors the state of the system */
632
633 KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
634
635 status = gettimeofday(&tval, NULL);
636 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
637 TIMEVAL_TO_TIMESPEC(&tval, &now);
638
639 now.tv_sec += interval.tv_sec;
640 now.tv_nsec += interval.tv_nsec;
641
642 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
643 now.tv_sec += 1;
644 now.tv_nsec -= KMP_NSEC_PER_SEC;
645 }
646
647 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
648 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
649 // AC: the monitor should not fall asleep if g_done has been set
650 if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
651 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
652 &__kmp_wait_mx.m_mutex, &now);
653 if (status != 0) {
654 if (status != ETIMEDOUT && status != EINTR) {
655 KMP_SYSFAIL("pthread_cond_timedwait", status);
656 }
657 }
658 }
659 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
660 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
661
662 TCW_4(__kmp_global.g.g_time.dt.t_value,
663 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
664
665 KMP_MB(); /* Flush all pending memory write invalidates. */
666 }
667
668 KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
669
670#ifdef KMP_BLOCK_SIGNALS
671 status = sigfillset(&new_set);
672 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
673 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
674 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
675#endif /* KMP_BLOCK_SIGNALS */
676
677 KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
678
679 if (__kmp_global.g.g_abort != 0) {
680 /* now we need to terminate the worker threads */
681 /* the value of t_abort is the signal we caught */
682
683 int gtid;
684
685 KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
686 __kmp_global.g.g_abort));
687
688 /* terminate the OpenMP worker threads */
689 /* TODO this is not valid for sibling threads!!
690 * the uber master might not be 0 anymore.. */
691 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
692 __kmp_terminate_thread(gtid);
693
694 __kmp_cleanup();
695
696 KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
697 __kmp_global.g.g_abort));
698
699 if (__kmp_global.g.g_abort > 0)
700 raise(__kmp_global.g.g_abort);
701 }
702
703 KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
704
705 return thr;
706}
707#endif // KMP_USE_MONITOR
708
709void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
710 pthread_t handle;
711 pthread_attr_t thread_attr;
712 int status;
713
714 th->th.th_info.ds.ds_gtid = gtid;
715
716#if KMP_STATS_ENABLED
717 // sets up worker thread stats
718 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
719
720 // th->th.th_stats is used to transfer thread-specific stats-pointer to
721 // __kmp_launch_worker. So when thread is created (goes into
722 // __kmp_launch_worker) it will set its thread local pointer to
723 // th->th.th_stats
724 if (!KMP_UBER_GTID(gtid)) {
725 th->th.th_stats = __kmp_stats_list->push_back(gtid);
726 } else {
727 // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
728 // so set the th->th.th_stats field to it.
729 th->th.th_stats = __kmp_stats_thread_ptr;
730 }
731 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
732
733#endif // KMP_STATS_ENABLED
734
735 if (KMP_UBER_GTID(gtid)) {
736 KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
737 th->th.th_info.ds.ds_thread = pthread_self();
738 __kmp_set_stack_info(gtid, th);
739 __kmp_check_stack_overlap(th);
740 return;
741 }
742
743 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
744
745 KMP_MB(); /* Flush all pending memory write invalidates. */
746
747#ifdef KMP_THREAD_ATTR
748 status = pthread_attr_init(&thread_attr);
749 if (status != 0) {
750 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
751 }
752 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
753 if (status != 0) {
754 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
755 }
756
757 /* Set stack size for this thread now.
758 The multiple of 2 is there because on some machines, requesting an unusual
759 stacksize causes the thread to have an offset before the dummy alloca()
760 takes place to create the offset. Since we want the user to have a
761 sufficient stacksize AND support a stack offset, we alloca() twice the
762 offset so that the upcoming alloca() does not eliminate any premade offset,
763 and also gives the user the stack space they requested for all threads */
764 stack_size += gtid * __kmp_stkoffset * 2;
765
766 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
767 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
768 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
769
770#ifdef _POSIX_THREAD_ATTR_STACKSIZE
771 status = pthread_attr_setstacksize(&thread_attr, stack_size);
772#ifdef KMP_BACKUP_STKSIZE
773 if (status != 0) {
774 if (!__kmp_env_stksize) {
775 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
776 __kmp_stksize = KMP_BACKUP_STKSIZE;
777 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
778 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
779 "bytes\n",
780 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
781 status = pthread_attr_setstacksize(&thread_attr, stack_size);
782 }
783 }
784#endif /* KMP_BACKUP_STKSIZE */
785 if (status != 0) {
786 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
787 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
788 }
789#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
790
791#endif /* KMP_THREAD_ATTR */
792
793 status =
794 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th);
795 if (status != 0 || !handle) { // ??? Why do we check handle??
796#ifdef _POSIX_THREAD_ATTR_STACKSIZE
797 if (status == EINVAL) {
798 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
799 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
800 }
801 if (status == ENOMEM) {
802 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
803 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
804 }
805#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
806 if (status == EAGAIN) {
807 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
808 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
809 }
810 KMP_SYSFAIL("pthread_create", status);
811 }
812
813 th->th.th_info.ds.ds_thread = handle;
814
815#ifdef KMP_THREAD_ATTR
816 status = pthread_attr_destroy(&thread_attr);
817 if (status) {
818 kmp_msg_t err_code = KMP_ERR(status);
819 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
820 __kmp_msg_null);
821 if (__kmp_generate_warnings == kmp_warnings_off) {
822 __kmp_str_free(&err_code.str);
823 }
824 }
825#endif /* KMP_THREAD_ATTR */
826
827 KMP_MB(); /* Flush all pending memory write invalidates. */
828
829 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
830
831} // __kmp_create_worker
832
833#if KMP_USE_MONITOR
834void __kmp_create_monitor(kmp_info_t *th) {
835 pthread_t handle;
836 pthread_attr_t thread_attr;
837 size_t size;
838 int status;
839 int auto_adj_size = FALSE;
840
841 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
842 // We don't need monitor thread in case of MAX_BLOCKTIME
843 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
844 "MAX blocktime\n"));
845 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
846 th->th.th_info.ds.ds_gtid = 0;
847 return;
848 }
849 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
850
851 KMP_MB(); /* Flush all pending memory write invalidates. */
852
853 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
854 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
855#if KMP_REAL_TIME_FIX
856 TCW_4(__kmp_global.g.g_time.dt.t_value,
857 -1); // Will use it for synchronization a bit later.
858#else
859 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
860#endif // KMP_REAL_TIME_FIX
861
862#ifdef KMP_THREAD_ATTR
863 if (__kmp_monitor_stksize == 0) {
864 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
865 auto_adj_size = TRUE;
866 }
867 status = pthread_attr_init(&thread_attr);
868 if (status != 0) {
869 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
870 }
871 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
872 if (status != 0) {
873 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
874 }
875
876#ifdef _POSIX_THREAD_ATTR_STACKSIZE
877 status = pthread_attr_getstacksize(&thread_attr, &size);
878 KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
879#else
880 size = __kmp_sys_min_stksize;
881#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
882#endif /* KMP_THREAD_ATTR */
883
884 if (__kmp_monitor_stksize == 0) {
885 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
886 }
887 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
888 __kmp_monitor_stksize = __kmp_sys_min_stksize;
889 }
890
891 KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
892 "requested stacksize = %lu bytes\n",
893 size, __kmp_monitor_stksize));
894
895retry:
896
897/* Set stack size for this thread now. */
898#ifdef _POSIX_THREAD_ATTR_STACKSIZE
899 KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
900 __kmp_monitor_stksize));
901 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
902 if (status != 0) {
903 if (auto_adj_size) {
904 __kmp_monitor_stksize *= 2;
905 goto retry;
906 }
907 kmp_msg_t err_code = KMP_ERR(status);
908 __kmp_msg(kmp_ms_warning, // should this be fatal? BB
909 KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
910 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
911 if (__kmp_generate_warnings == kmp_warnings_off) {
912 __kmp_str_free(&err_code.str);
913 }
914 }
915#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
916
917 status =
918 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
919
920 if (status != 0) {
921#ifdef _POSIX_THREAD_ATTR_STACKSIZE
922 if (status == EINVAL) {
923 if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
924 __kmp_monitor_stksize *= 2;
925 goto retry;
926 }
927 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
928 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
929 __kmp_msg_null);
930 }
931 if (status == ENOMEM) {
932 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
933 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
934 __kmp_msg_null);
935 }
936#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
937 if (status == EAGAIN) {
938 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
939 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
940 }
941 KMP_SYSFAIL("pthread_create", status);
942 }
943
944 th->th.th_info.ds.ds_thread = handle;
945
946#if KMP_REAL_TIME_FIX
947 // Wait for the monitor thread is really started and set its *priority*.
948 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
949 sizeof(__kmp_global.g.g_time.dt.t_value));
950 __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
951 &__kmp_neq_4, NULL);
952#endif // KMP_REAL_TIME_FIX
953
954#ifdef KMP_THREAD_ATTR
955 status = pthread_attr_destroy(&thread_attr);
956 if (status != 0) {
957 kmp_msg_t err_code = KMP_ERR(status);
958 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
959 __kmp_msg_null);
960 if (__kmp_generate_warnings == kmp_warnings_off) {
961 __kmp_str_free(&err_code.str);
962 }
963 }
964#endif
965
966 KMP_MB(); /* Flush all pending memory write invalidates. */
967
968 KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
969 th->th.th_info.ds.ds_thread));
970
971} // __kmp_create_monitor
972#endif // KMP_USE_MONITOR
973
974void __kmp_exit_thread(int exit_status) {
975 pthread_exit((void *)(intptr_t)exit_status);
976} // __kmp_exit_thread
977
978#if KMP_USE_MONITOR
979void __kmp_resume_monitor();
980
981void __kmp_reap_monitor(kmp_info_t *th) {
982 int status;
983 void *exit_val;
984
985 KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
986 " %#.8lx\n",
987 th->th.th_info.ds.ds_thread));
988
989 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
990 // If both tid and gtid are 0, it means the monitor did not ever start.
991 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
992 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
993 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
994 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
995 return;
996 }
997
998 KMP_MB(); /* Flush all pending memory write invalidates. */
999
1000 /* First, check to see whether the monitor thread exists to wake it up. This
1001 is to avoid performance problem when the monitor sleeps during
1002 blocktime-size interval */
1003
1004 status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1005 if (status != ESRCH) {
1006 __kmp_resume_monitor(); // Wake up the monitor thread
1007 }
1008 KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1009 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1010 if (exit_val != th) {
1011 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1012 }
1013
1014 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1015 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1016
1017 KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1018 " %#.8lx\n",
1019 th->th.th_info.ds.ds_thread));
1020
1021 KMP_MB(); /* Flush all pending memory write invalidates. */
1022}
1023#endif // KMP_USE_MONITOR
1024
1025void __kmp_reap_worker(kmp_info_t *th) {
1026 int status;
1027 void *exit_val;
1028
1029 KMP_MB(); /* Flush all pending memory write invalidates. */
1030
1031 KA_TRACE(
1032 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1033
1034 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1035#ifdef KMP_DEBUG
1036 /* Don't expose these to the user until we understand when they trigger */
1037 if (status != 0) {
1038 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1039 }
1040 if (exit_val != th) {
1041 KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1042 "exit_val = %p\n",
1043 th->th.th_info.ds.ds_gtid, exit_val));
1044 }
1045#else
1046 (void)status; // unused variable
1047#endif /* KMP_DEBUG */
1048
1049 KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1050 th->th.th_info.ds.ds_gtid));
1051
1052 KMP_MB(); /* Flush all pending memory write invalidates. */
1053}
1054
1055#if KMP_HANDLE_SIGNALS
1056
1057static void __kmp_null_handler(int signo) {
1058 // Do nothing, for doing SIG_IGN-type actions.
1059} // __kmp_null_handler
1060
1061static void __kmp_team_handler(int signo) {
1062 if (__kmp_global.g.g_abort == 0) {
1063/* Stage 1 signal handler, let's shut down all of the threads */
1064#ifdef KMP_DEBUG
1065 __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo);
1066#endif
1067 switch (signo) {
1068 case SIGHUP:
1069 case SIGINT:
1070 case SIGQUIT:
1071 case SIGILL:
1072 case SIGABRT:
1073 case SIGFPE:
1074 case SIGBUS:
1075 case SIGSEGV:
1076#ifdef SIGSYS
1077 case SIGSYS:
1078#endif
1079 case SIGTERM:
1080 if (__kmp_debug_buf) {
1081 __kmp_dump_debug_buffer();
1082 }
1083 __kmp_unregister_library(); // cleanup shared memory
1084 KMP_MB(); // Flush all pending memory write invalidates.
1085 TCW_4(__kmp_global.g.g_abort, signo);
1086 KMP_MB(); // Flush all pending memory write invalidates.
1087 TCW_4(__kmp_global.g.g_done, TRUE);
1088 KMP_MB(); // Flush all pending memory write invalidates.
1089 break;
1090 default:
1091#ifdef KMP_DEBUG
1092 __kmp_debug_printf("__kmp_team_handler: unknown signal type");
1093#endif
1094 break;
1095 }
1096 }
1097} // __kmp_team_handler
1098
1099static void __kmp_sigaction(int signum, const struct sigaction *act,
1100 struct sigaction *oldact) {
1101 int rc = sigaction(signum, act, oldact);
1102 KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1103}
1104
1105static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1106 int parallel_init) {
1107 KMP_MB(); // Flush all pending memory write invalidates.
1108 KB_TRACE(60,
1109 ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1110 if (parallel_init) {
1111 struct sigaction new_action;
1112 struct sigaction old_action;
1113 new_action.sa_handler = handler_func;
1114 new_action.sa_flags = 0;
1115 sigfillset(&new_action.sa_mask);
1116 __kmp_sigaction(sig, &new_action, &old_action);
1117 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1118 sigaddset(&__kmp_sigset, sig);
1119 } else {
1120 // Restore/keep user's handler if one previously installed.
1121 __kmp_sigaction(sig, &old_action, NULL);
1122 }
1123 } else {
1124 // Save initial/system signal handlers to see if user handlers installed.
1125 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1126 }
1127 KMP_MB(); // Flush all pending memory write invalidates.
1128} // __kmp_install_one_handler
1129
1130static void __kmp_remove_one_handler(int sig) {
1131 KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1132 if (sigismember(&__kmp_sigset, sig)) {
1133 struct sigaction old;
1134 KMP_MB(); // Flush all pending memory write invalidates.
1135 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1136 if ((old.sa_handler != __kmp_team_handler) &&
1137 (old.sa_handler != __kmp_null_handler)) {
1138 // Restore the users signal handler.
1139 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1140 "restoring: sig=%d\n",
1141 sig));
1142 __kmp_sigaction(sig, &old, NULL);
1143 }
1144 sigdelset(&__kmp_sigset, sig);
1145 KMP_MB(); // Flush all pending memory write invalidates.
1146 }
1147} // __kmp_remove_one_handler
1148
1149void __kmp_install_signals(int parallel_init) {
1150 KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1151 if (__kmp_handle_signals || !parallel_init) {
1152 // If ! parallel_init, we do not install handlers, just save original
1153 // handlers. Let us do it even __handle_signals is 0.
1154 sigemptyset(&__kmp_sigset);
1155 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1156 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1157 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1158 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1159 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1160 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1161 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1162 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1163#ifdef SIGSYS
1164 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1165#endif // SIGSYS
1166 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1167#ifdef SIGPIPE
1168 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1169#endif // SIGPIPE
1170 }
1171} // __kmp_install_signals
1172
1173void __kmp_remove_signals(void) {
1174 int sig;
1175 KB_TRACE(10, ("__kmp_remove_signals()\n"));
1176 for (sig = 1; sig < NSIG; ++sig) {
1177 __kmp_remove_one_handler(sig);
1178 }
1179} // __kmp_remove_signals
1180
1181#endif // KMP_HANDLE_SIGNALS
1182
1183void __kmp_enable(int new_state) {
1184#ifdef KMP_CANCEL_THREADS
1185 int status, old_state;
1186 status = pthread_setcancelstate(new_state, &old_state);
1187 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1188 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1189#endif
1190}
1191
1192void __kmp_disable(int *old_state) {
1193#ifdef KMP_CANCEL_THREADS
1194 int status;
1195 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1196 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1197#endif
1198}
1199
1200static void __kmp_atfork_prepare(void) {
1201 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1202 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1203}
1204
1205static void __kmp_atfork_parent(void) {
1206 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1207 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1208}
1209
1210/* Reset the library so execution in the child starts "all over again" with
1211 clean data structures in initial states. Don't worry about freeing memory
1212 allocated by parent, just abandon it to be safe. */
1213static void __kmp_atfork_child(void) {
1214 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1215 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1216 /* TODO make sure this is done right for nested/sibling */
1217 // ATT: Memory leaks are here? TODO: Check it and fix.
1218 /* KMP_ASSERT( 0 ); */
1219
1220 ++__kmp_fork_count;
1221
1222#if KMP_AFFINITY_SUPPORTED
1223#if KMP_OS_LINUX || KMP_OS_FREEBSD
1224 // reset the affinity in the child to the initial thread
1225 // affinity in the parent
1226 kmp_set_thread_affinity_mask_initial();
1227#endif
1228 // Set default not to bind threads tightly in the child (we're expecting
1229 // over-subscription after the fork and this can improve things for
1230 // scripting languages that use OpenMP inside process-parallel code).
1231 if (__kmp_nested_proc_bind.bind_types != NULL) {
1232 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1233 }
1234 for (kmp_affinity_t *affinity : __kmp_affinities)
1235 *affinity = KMP_AFFINITY_INIT(affinity->env_var);
1236 __kmp_affin_fullMask = nullptr;
1237 __kmp_affin_origMask = nullptr;
1238#endif // KMP_AFFINITY_SUPPORTED
1239
1240#if KMP_USE_MONITOR
1241 __kmp_init_monitor = 0;
1242#endif
1243 __kmp_init_parallel = FALSE;
1244 __kmp_init_middle = FALSE;
1245 __kmp_init_serial = FALSE;
1246 TCW_4(__kmp_init_gtid, FALSE);
1247 __kmp_init_common = FALSE;
1248
1249 TCW_4(__kmp_init_user_locks, FALSE);
1250#if !KMP_USE_DYNAMIC_LOCK
1251 __kmp_user_lock_table.used = 1;
1252 __kmp_user_lock_table.allocated = 0;
1253 __kmp_user_lock_table.table = NULL;
1254 __kmp_lock_blocks = NULL;
1255#endif
1256
1257 __kmp_all_nth = 0;
1258 TCW_4(__kmp_nth, 0);
1259
1260 __kmp_thread_pool = NULL;
1261 __kmp_thread_pool_insert_pt = NULL;
1262 __kmp_team_pool = NULL;
1263
1264 /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1265 here so threadprivate doesn't use stale data */
1266 KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1267 __kmp_threadpriv_cache_list));
1268
1269 while (__kmp_threadpriv_cache_list != NULL) {
1270
1271 if (*__kmp_threadpriv_cache_list->addr != NULL) {
1272 KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1273 &(*__kmp_threadpriv_cache_list->addr)));
1274
1275 *__kmp_threadpriv_cache_list->addr = NULL;
1276 }
1277 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1278 }
1279
1280 __kmp_init_runtime = FALSE;
1281
1282 /* reset statically initialized locks */
1283 __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1284 __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1285 __kmp_init_bootstrap_lock(&__kmp_console_lock);
1286 __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1287
1288#if USE_ITT_BUILD
1289 __kmp_itt_reset(); // reset ITT's global state
1290#endif /* USE_ITT_BUILD */
1291
1292 {
1293 // Child process often get terminated without any use of OpenMP. That might
1294 // cause mapped shared memory file to be left unattended. Thus we postpone
1295 // library registration till middle initialization in the child process.
1296 __kmp_need_register_serial = FALSE;
1297 __kmp_serial_initialize();
1298 }
1299
1300 /* This is necessary to make sure no stale data is left around */
1301 /* AC: customers complain that we use unsafe routines in the atfork
1302 handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1303 in dynamic_link when check the presence of shared tbbmalloc library.
1304 Suggestion is to make the library initialization lazier, similar
1305 to what done for __kmpc_begin(). */
1306 // TODO: synchronize all static initializations with regular library
1307 // startup; look at kmp_global.cpp and etc.
1308 //__kmp_internal_begin ();
1309}
1310
1311void __kmp_register_atfork(void) {
1312 if (__kmp_need_register_atfork) {
1313 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1314 __kmp_atfork_child);
1315 KMP_CHECK_SYSFAIL("pthread_atfork", status);
1316 __kmp_need_register_atfork = FALSE;
1317 }
1318}
1319
1320void __kmp_suspend_initialize(void) {
1321 int status;
1322 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1323 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1324 status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1325 KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1326}
1327
1328void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1329 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1330 int new_value = __kmp_fork_count + 1;
1331 // Return if already initialized
1332 if (old_value == new_value)
1333 return;
1334 // Wait, then return if being initialized
1335 if (old_value == -1 || !__kmp_atomic_compare_store(
1336 &th->th.th_suspend_init_count, old_value, -1)) {
1337 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1338 KMP_CPU_PAUSE();
1339 }
1340 } else {
1341 // Claim to be the initializer and do initializations
1342 int status;
1343 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1344 &__kmp_suspend_cond_attr);
1345 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1346 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1347 &__kmp_suspend_mutex_attr);
1348 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1349 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1350 }
1351}
1352
1353void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1354 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1355 /* this means we have initialize the suspension pthread objects for this
1356 thread in this instance of the process */
1357 int status;
1358
1359 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1360 if (status != 0 && status != EBUSY) {
1361 KMP_SYSFAIL("pthread_cond_destroy", status);
1362 }
1363 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1364 if (status != 0 && status != EBUSY) {
1365 KMP_SYSFAIL("pthread_mutex_destroy", status);
1366 }
1367 --th->th.th_suspend_init_count;
1368 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1369 __kmp_fork_count);
1370 }
1371}
1372
1373// return true if lock obtained, false otherwise
1374int __kmp_try_suspend_mx(kmp_info_t *th) {
1375 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
1376}
1377
1378void __kmp_lock_suspend_mx(kmp_info_t *th) {
1379 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1380 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1381}
1382
1383void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1384 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1385 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1386}
1387
1388/* This routine puts the calling thread to sleep after setting the
1389 sleep bit for the indicated flag variable to true. */
1390template <class C>
1391static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1392 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1393 kmp_info_t *th = __kmp_threads[th_gtid];
1394 int status;
1395 typename C::flag_t old_spin;
1396
1397 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1398 flag->get()));
1399
1400 __kmp_suspend_initialize_thread(th);
1401
1402 __kmp_lock_suspend_mx(th);
1403
1404 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1405 th_gtid, flag->get()));
1406
1407 /* TODO: shouldn't this use release semantics to ensure that
1408 __kmp_suspend_initialize_thread gets called first? */
1409 old_spin = flag->set_sleeping();
1410 TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1411 th->th.th_sleep_loc_type = flag->get_type();
1412 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1413 __kmp_pause_status != kmp_soft_paused) {
1414 flag->unset_sleeping();
1415 TCW_PTR(th->th.th_sleep_loc, NULL);
1416 th->th.th_sleep_loc_type = flag_unset;
1417 __kmp_unlock_suspend_mx(th);
1418 return;
1419 }
1420 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1421 " was %x\n",
1422 th_gtid, flag->get(), flag->load(), old_spin));
1423
1424 if (flag->done_check_val(old_spin) || flag->done_check()) {
1425 flag->unset_sleeping();
1426 TCW_PTR(th->th.th_sleep_loc, NULL);
1427 th->th.th_sleep_loc_type = flag_unset;
1428 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1429 "for spin(%p)\n",
1430 th_gtid, flag->get()));
1431 } else {
1432 /* Encapsulate in a loop as the documentation states that this may
1433 "with low probability" return when the condition variable has
1434 not been signaled or broadcast */
1435 int deactivated = FALSE;
1436
1437 while (flag->is_sleeping()) {
1438#ifdef DEBUG_SUSPEND
1439 char buffer[128];
1440 __kmp_suspend_count++;
1441 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1442 __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1443 buffer);
1444#endif
1445 // Mark the thread as no longer active (only in the first iteration of the
1446 // loop).
1447 if (!deactivated) {
1448 th->th.th_active = FALSE;
1449 if (th->th.th_active_in_pool) {
1450 th->th.th_active_in_pool = FALSE;
1451 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1452 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1453 }
1454 deactivated = TRUE;
1455 }
1456
1457 KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1458 KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1459
1460#if USE_SUSPEND_TIMEOUT
1461 struct timespec now;
1462 struct timeval tval;
1463 int msecs;
1464
1465 status = gettimeofday(&tval, NULL);
1466 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1467 TIMEVAL_TO_TIMESPEC(&tval, &now);
1468
1469 msecs = (4 * __kmp_dflt_blocktime) + 200;
1470 now.tv_sec += msecs / 1000;
1471 now.tv_nsec += (msecs % 1000) * 1000;
1472
1473 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1474 "pthread_cond_timedwait\n",
1475 th_gtid));
1476 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1477 &th->th.th_suspend_mx.m_mutex, &now);
1478#else
1479 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1480 " pthread_cond_wait\n",
1481 th_gtid));
1482 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1483 &th->th.th_suspend_mx.m_mutex);
1484#endif // USE_SUSPEND_TIMEOUT
1485
1486 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1487 KMP_SYSFAIL("pthread_cond_wait", status);
1488 }
1489
1490 KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1491
1492 if (!flag->is_sleeping() &&
1493 ((status == EINTR) || (status == ETIMEDOUT))) {
1494 // if interrupt or timeout, and thread is no longer sleeping, we need to
1495 // make sure sleep_loc gets reset; however, this shouldn't be needed if
1496 // we woke up with resume
1497 flag->unset_sleeping();
1498 TCW_PTR(th->th.th_sleep_loc, NULL);
1499 th->th.th_sleep_loc_type = flag_unset;
1500 }
1501#ifdef KMP_DEBUG
1502 if (status == ETIMEDOUT) {
1503 if (flag->is_sleeping()) {
1504 KF_TRACE(100,
1505 ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1506 } else {
1507 KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1508 "not set!\n",
1509 th_gtid));
1510 TCW_PTR(th->th.th_sleep_loc, NULL);
1511 th->th.th_sleep_loc_type = flag_unset;
1512 }
1513 } else if (flag->is_sleeping()) {
1514 KF_TRACE(100,
1515 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1516 }
1517#endif
1518 } // while
1519
1520 // Mark the thread as active again (if it was previous marked as inactive)
1521 if (deactivated) {
1522 th->th.th_active = TRUE;
1523 if (TCR_4(th->th.th_in_pool)) {
1524 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1525 th->th.th_active_in_pool = TRUE;
1526 }
1527 }
1528 }
1529 // We may have had the loop variable set before entering the loop body;
1530 // so we need to reset sleep_loc.
1531 TCW_PTR(th->th.th_sleep_loc, NULL);
1532 th->th.th_sleep_loc_type = flag_unset;
1533
1534 KMP_DEBUG_ASSERT(!flag->is_sleeping());
1535 KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1536#ifdef DEBUG_SUSPEND
1537 {
1538 char buffer[128];
1539 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1540 __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1541 buffer);
1542 }
1543#endif
1544
1545 __kmp_unlock_suspend_mx(th);
1546 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1547}
1548
1549template <bool C, bool S>
1550void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1551 __kmp_suspend_template(th_gtid, flag);
1552}
1553template <bool C, bool S>
1554void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1555 __kmp_suspend_template(th_gtid, flag);
1556}
1557template <bool C, bool S>
1558void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1559 __kmp_suspend_template(th_gtid, flag);
1560}
1561void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1562 __kmp_suspend_template(th_gtid, flag);
1563}
1564
1565template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1566template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1567template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1568template void
1569__kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1570template void
1571__kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1572
1573/* This routine signals the thread specified by target_gtid to wake up
1574 after setting the sleep bit indicated by the flag argument to FALSE.
1575 The target thread must already have called __kmp_suspend_template() */
1576template <class C>
1577static inline void __kmp_resume_template(int target_gtid, C *flag) {
1578 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1579 kmp_info_t *th = __kmp_threads[target_gtid];
1580 int status;
1581
1582#ifdef KMP_DEBUG
1583 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1584#endif
1585
1586 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1587 gtid, target_gtid));
1588 KMP_DEBUG_ASSERT(gtid != target_gtid);
1589
1590 __kmp_suspend_initialize_thread(th);
1591
1592 __kmp_lock_suspend_mx(th);
1593
1594 if (!flag || flag != th->th.th_sleep_loc) {
1595 // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1596 // different location; wake up at new location
1597 flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1598 }
1599
1600 // First, check if the flag is null or its type has changed. If so, someone
1601 // else woke it up.
1602 if (!flag) { // Thread doesn't appear to be sleeping on anything
1603 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1604 "awake: flag(%p)\n",
1605 gtid, target_gtid, (void *)NULL));
1606 __kmp_unlock_suspend_mx(th);
1607 return;
1608 } else if (flag->get_type() != th->th.th_sleep_loc_type) {
1609 // Flag type does not appear to match this function template; possibly the
1610 // thread is sleeping on something else. Try null resume again.
1611 KF_TRACE(
1612 5,
1613 ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1614 "spin(%p) type=%d ptr_type=%d\n",
1615 gtid, target_gtid, flag, flag->get(), flag->get_type(),
1616 th->th.th_sleep_loc_type));
1617 __kmp_unlock_suspend_mx(th);
1618 __kmp_null_resume_wrapper(th);
1619 return;
1620 } else { // if multiple threads are sleeping, flag should be internally
1621 // referring to a specific thread here
1622 if (!flag->is_sleeping()) {
1623 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1624 "awake: flag(%p): %u\n",
1625 gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1626 __kmp_unlock_suspend_mx(th);
1627 return;
1628 }
1629 }
1630 KMP_DEBUG_ASSERT(flag);
1631 flag->unset_sleeping();
1632 TCW_PTR(th->th.th_sleep_loc, NULL);
1633 th->th.th_sleep_loc_type = flag_unset;
1634
1635 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1636 "sleep bit for flag's loc(%p): %u\n",
1637 gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1638
1639#ifdef DEBUG_SUSPEND
1640 {
1641 char buffer[128];
1642 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1643 __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1644 target_gtid, buffer);
1645 }
1646#endif
1647 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1648 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1649 __kmp_unlock_suspend_mx(th);
1650 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1651 " for T#%d\n",
1652 gtid, target_gtid));
1653}
1654
1655template <bool C, bool S>
1656void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1657 __kmp_resume_template(target_gtid, flag);
1658}
1659template <bool C, bool S>
1660void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1661 __kmp_resume_template(target_gtid, flag);
1662}
1663template <bool C, bool S>
1664void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1665 __kmp_resume_template(target_gtid, flag);
1666}
1667void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1668 __kmp_resume_template(target_gtid, flag);
1669}
1670
1671template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1672template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1673template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1674template void
1675__kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1676
1677#if KMP_USE_MONITOR
1678void __kmp_resume_monitor() {
1679 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1680 int status;
1681#ifdef KMP_DEBUG
1682 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1683 KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1684 KMP_GTID_MONITOR));
1685 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1686#endif
1687 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1688 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1689#ifdef DEBUG_SUSPEND
1690 {
1691 char buffer[128];
1692 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1693 __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1694 KMP_GTID_MONITOR, buffer);
1695 }
1696#endif
1697 status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1698 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1699 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1700 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1701 KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1702 " for T#%d\n",
1703 gtid, KMP_GTID_MONITOR));
1704}
1705#endif // KMP_USE_MONITOR
1706
1707void __kmp_yield() { sched_yield(); }
1708
1709void __kmp_gtid_set_specific(int gtid) {
1710 if (__kmp_init_gtid) {
1711 int status;
1712 status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1713 (void *)(intptr_t)(gtid + 1));
1714 KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1715 } else {
1716 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1717 }
1718}
1719
1720int __kmp_gtid_get_specific() {
1721 int gtid;
1722 if (!__kmp_init_gtid) {
1723 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1724 "KMP_GTID_SHUTDOWN\n"));
1725 return KMP_GTID_SHUTDOWN;
1726 }
1727 gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1728 if (gtid == 0) {
1729 gtid = KMP_GTID_DNE;
1730 } else {
1731 gtid--;
1732 }
1733 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1734 __kmp_gtid_threadprivate_key, gtid));
1735 return gtid;
1736}
1737
1738double __kmp_read_cpu_time(void) {
1739 /*clock_t t;*/
1740 struct tms buffer;
1741
1742 /*t =*/times(&buffer);
1743
1744 return (double)(buffer.tms_utime + buffer.tms_cutime) /
1745 (double)CLOCKS_PER_SEC;
1746}
1747
1748int __kmp_read_system_info(struct kmp_sys_info *info) {
1749 int status;
1750 struct rusage r_usage;
1751
1752 memset(info, 0, sizeof(*info));
1753
1754 status = getrusage(RUSAGE_SELF, &r_usage);
1755 KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1756
1757 // The maximum resident set size utilized (in kilobytes)
1758 info->maxrss = r_usage.ru_maxrss;
1759 // The number of page faults serviced without any I/O
1760 info->minflt = r_usage.ru_minflt;
1761 // The number of page faults serviced that required I/O
1762 info->majflt = r_usage.ru_majflt;
1763 // The number of times a process was "swapped" out of memory
1764 info->nswap = r_usage.ru_nswap;
1765 // The number of times the file system had to perform input
1766 info->inblock = r_usage.ru_inblock;
1767 // The number of times the file system had to perform output
1768 info->oublock = r_usage.ru_oublock;
1769 // The number of times a context switch was voluntarily
1770 info->nvcsw = r_usage.ru_nvcsw;
1771 // The number of times a context switch was forced
1772 info->nivcsw = r_usage.ru_nivcsw;
1773
1774 return (status != 0);
1775}
1776
1777void __kmp_read_system_time(double *delta) {
1778 double t_ns;
1779 struct timeval tval;
1780 struct timespec stop;
1781 int status;
1782
1783 status = gettimeofday(&tval, NULL);
1784 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1785 TIMEVAL_TO_TIMESPEC(&tval, &stop);
1786 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1787 *delta = (t_ns * 1e-9);
1788}
1789
1790void __kmp_clear_system_time(void) {
1791 struct timeval tval;
1792 int status;
1793 status = gettimeofday(&tval, NULL);
1794 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1795 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1796}
1797
1798static int __kmp_get_xproc(void) {
1799
1800 int r = 0;
1801
1802#if KMP_OS_LINUX
1803
1804 __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r));
1805
1806#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1807 KMP_OS_HURD
1808
1809 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1810
1811#elif KMP_OS_DARWIN
1812
1813 // Bug C77011 High "OpenMP Threads and number of active cores".
1814
1815 // Find the number of available CPUs.
1816 kern_return_t rc;
1817 host_basic_info_data_t info;
1818 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
1819 rc = host_info(mach_host_self(), HOST_BASIC_INFO, (host_info_t)&info, &num);
1820 if (rc == 0 && num == HOST_BASIC_INFO_COUNT) {
1821 // Cannot use KA_TRACE() here because this code works before trace support
1822 // is initialized.
1823 r = info.avail_cpus;
1824 } else {
1825 KMP_WARNING(CantGetNumAvailCPU);
1826 KMP_INFORM(AssumedNumCPU);
1827 }
1828
1829#else
1830
1831#error "Unknown or unsupported OS."
1832
1833#endif
1834
1835 return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1836
1837} // __kmp_get_xproc
1838
1839int __kmp_read_from_file(char const *path, char const *format, ...) {
1840 int result;
1841 va_list args;
1842
1843 va_start(args, format);
1844 FILE *f = fopen(path, "rb");
1845 if (f == NULL)
1846 return 0;
1847 result = vfscanf(f, format, args);
1848 fclose(f);
1849
1850 return result;
1851}
1852
1853void __kmp_runtime_initialize(void) {
1854 int status;
1855 pthread_mutexattr_t mutex_attr;
1856 pthread_condattr_t cond_attr;
1857
1858 if (__kmp_init_runtime) {
1859 return;
1860 }
1861
1862#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1863 if (!__kmp_cpuinfo.initialized) {
1864 __kmp_query_cpuid(&__kmp_cpuinfo);
1865 }
1866#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1867
1868 __kmp_xproc = __kmp_get_xproc();
1869
1870#if !KMP_32_BIT_ARCH
1871 struct rlimit rlim;
1872 // read stack size of calling thread, save it as default for worker threads;
1873 // this should be done before reading environment variables
1874 status = getrlimit(RLIMIT_STACK, &rlim);
1875 if (status == 0) { // success?
1876 __kmp_stksize = rlim.rlim_cur;
1877 __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed
1878 }
1879#endif /* KMP_32_BIT_ARCH */
1880
1881 if (sysconf(_SC_THREADS)) {
1882
1883 /* Query the maximum number of threads */
1884 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth));
1885 if (__kmp_sys_max_nth == -1) {
1886 /* Unlimited threads for NPTL */
1887 __kmp_sys_max_nth = INT_MAX;
1888 } else if (__kmp_sys_max_nth <= 1) {
1889 /* Can't tell, just use PTHREAD_THREADS_MAX */
1890 __kmp_sys_max_nth = KMP_MAX_NTH;
1891 }
1892
1893 /* Query the minimum stack size */
1894 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1895 if (__kmp_sys_min_stksize <= 1) {
1896 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1897 }
1898 }
1899
1900 /* Set up minimum number of threads to switch to TLS gtid */
1901 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
1902
1903 status = pthread_key_create(&__kmp_gtid_threadprivate_key,
1904 __kmp_internal_end_dest);
1905 KMP_CHECK_SYSFAIL("pthread_key_create", status);
1906 status = pthread_mutexattr_init(&mutex_attr);
1907 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1908 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
1909 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1910 status = pthread_mutexattr_destroy(&mutex_attr);
1911 KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
1912 status = pthread_condattr_init(&cond_attr);
1913 KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1914 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
1915 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1916 status = pthread_condattr_destroy(&cond_attr);
1917 KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
1918#if USE_ITT_BUILD
1919 __kmp_itt_initialize();
1920#endif /* USE_ITT_BUILD */
1921
1922 __kmp_init_runtime = TRUE;
1923}
1924
1925void __kmp_runtime_destroy(void) {
1926 int status;
1927
1928 if (!__kmp_init_runtime) {
1929 return; // Nothing to do.
1930 }
1931
1932#if USE_ITT_BUILD
1933 __kmp_itt_destroy();
1934#endif /* USE_ITT_BUILD */
1935
1936 status = pthread_key_delete(__kmp_gtid_threadprivate_key);
1937 KMP_CHECK_SYSFAIL("pthread_key_delete", status);
1938
1939 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
1940 if (status != 0 && status != EBUSY) {
1941 KMP_SYSFAIL("pthread_mutex_destroy", status);
1942 }
1943 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
1944 if (status != 0 && status != EBUSY) {
1945 KMP_SYSFAIL("pthread_cond_destroy", status);
1946 }
1947#if KMP_AFFINITY_SUPPORTED
1948 __kmp_affinity_uninitialize();
1949#endif
1950
1951 __kmp_init_runtime = FALSE;
1952}
1953
1954/* Put the thread to sleep for a time period */
1955/* NOTE: not currently used anywhere */
1956void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); }
1957
1958/* Calculate the elapsed wall clock time for the user */
1959void __kmp_elapsed(double *t) {
1960 int status;
1961#ifdef FIX_SGI_CLOCK
1962 struct timespec ts;
1963
1964 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
1965 KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
1966 *t =
1967 (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
1968#else
1969 struct timeval tv;
1970
1971 status = gettimeofday(&tv, NULL);
1972 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1973 *t =
1974 (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
1975#endif
1976}
1977
1978/* Calculate the elapsed wall clock tick for the user */
1979void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
1980
1981/* Return the current time stamp in nsec */
1982kmp_uint64 __kmp_now_nsec() {
1983 struct timeval t;
1984 gettimeofday(&t, NULL);
1985 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
1986 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
1987 return nsec;
1988}
1989
1990#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1991/* Measure clock ticks per millisecond */
1992void __kmp_initialize_system_tick() {
1993 kmp_uint64 now, nsec2, diff;
1994 kmp_uint64 delay = 100000; // 50~100 usec on most machines.
1995 kmp_uint64 nsec = __kmp_now_nsec();
1996 kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
1997 while ((now = __kmp_hardware_timestamp()) < goal)
1998 ;
1999 nsec2 = __kmp_now_nsec();
2000 diff = nsec2 - nsec;
2001 if (diff > 0) {
2002 kmp_uint64 tpms = ((kmp_uint64)1e6 * (delay + (now - goal)) / diff);
2003 if (tpms > 0)
2004 __kmp_ticks_per_msec = tpms;
2005 }
2006}
2007#endif
2008
2009/* Determine whether the given address is mapped into the current address
2010 space. */
2011
2012int __kmp_is_address_mapped(void *addr) {
2013
2014 int found = 0;
2015 int rc;
2016
2017#if KMP_OS_LINUX || KMP_OS_HURD
2018
2019 /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2020 address ranges mapped into the address space. */
2021
2022 char *name = __kmp_str_format("/proc/%d/maps", getpid());
2023 FILE *file = NULL;
2024
2025 file = fopen(name, "r");
2026 KMP_ASSERT(file != NULL);
2027
2028 for (;;) {
2029
2030 void *beginning = NULL;
2031 void *ending = NULL;
2032 char perms[5];
2033
2034 rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2035 if (rc == EOF) {
2036 break;
2037 }
2038 KMP_ASSERT(rc == 3 &&
2039 KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2040
2041 // Ending address is not included in the region, but beginning is.
2042 if ((addr >= beginning) && (addr < ending)) {
2043 perms[2] = 0; // 3th and 4th character does not matter.
2044 if (strcmp(perms, "rw") == 0) {
2045 // Memory we are looking for should be readable and writable.
2046 found = 1;
2047 }
2048 break;
2049 }
2050 }
2051
2052 // Free resources.
2053 fclose(file);
2054 KMP_INTERNAL_FREE(name);
2055#elif KMP_OS_FREEBSD
2056 char *buf;
2057 size_t lstsz;
2058 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2059 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2060 if (rc < 0)
2061 return 0;
2062 // We pass from number of vm entry's semantic
2063 // to size of whole entry map list.
2064 lstsz = lstsz * 4 / 3;
2065 buf = reinterpret_cast<char *>(kmpc_malloc(lstsz));
2066 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2067 if (rc < 0) {
2068 kmpc_free(buf);
2069 return 0;
2070 }
2071
2072 char *lw = buf;
2073 char *up = buf + lstsz;
2074
2075 while (lw < up) {
2076 struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2077 size_t cursz = cur->kve_structsize;
2078 if (cursz == 0)
2079 break;
2080 void *start = reinterpret_cast<void *>(cur->kve_start);
2081 void *end = reinterpret_cast<void *>(cur->kve_end);
2082 // Readable/Writable addresses within current map entry
2083 if ((addr >= start) && (addr < end)) {
2084 if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2085 (cur->kve_protection & KVME_PROT_WRITE) != 0) {
2086 found = 1;
2087 break;
2088 }
2089 }
2090 lw += cursz;
2091 }
2092 kmpc_free(buf);
2093
2094#elif KMP_OS_DARWIN
2095
2096 /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2097 using vm interface. */
2098
2099 int buffer;
2100 vm_size_t count;
2101 rc = vm_read_overwrite(
2102 mach_task_self(), // Task to read memory of.
2103 (vm_address_t)(addr), // Address to read from.
2104 1, // Number of bytes to be read.
2105 (vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2106 &count // Address of var to save number of read bytes in.
2107 );
2108 if (rc == 0) {
2109 // Memory successfully read.
2110 found = 1;
2111 }
2112
2113#elif KMP_OS_NETBSD
2114
2115 int mib[5];
2116 mib[0] = CTL_VM;
2117 mib[1] = VM_PROC;
2118 mib[2] = VM_PROC_MAP;
2119 mib[3] = getpid();
2120 mib[4] = sizeof(struct kinfo_vmentry);
2121
2122 size_t size;
2123 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2124 KMP_ASSERT(!rc);
2125 KMP_ASSERT(size);
2126
2127 size = size * 4 / 3;
2128 struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2129 KMP_ASSERT(kiv);
2130
2131 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2132 KMP_ASSERT(!rc);
2133 KMP_ASSERT(size);
2134
2135 for (size_t i = 0; i < size; i++) {
2136 if (kiv[i].kve_start >= (uint64_t)addr &&
2137 kiv[i].kve_end <= (uint64_t)addr) {
2138 found = 1;
2139 break;
2140 }
2141 }
2142 KMP_INTERNAL_FREE(kiv);
2143#elif KMP_OS_OPENBSD
2144
2145 int mib[3];
2146 mib[0] = CTL_KERN;
2147 mib[1] = KERN_PROC_VMMAP;
2148 mib[2] = getpid();
2149
2150 size_t size;
2151 uint64_t end;
2152 rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2153 KMP_ASSERT(!rc);
2154 KMP_ASSERT(size);
2155 end = size;
2156
2157 struct kinfo_vmentry kiv = {.kve_start = 0};
2158
2159 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2160 KMP_ASSERT(size);
2161 if (kiv.kve_end == end)
2162 break;
2163
2164 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2165 found = 1;
2166 break;
2167 }
2168 kiv.kve_start += 1;
2169 }
2170#elif KMP_OS_DRAGONFLY
2171
2172 // FIXME(DragonFly): Implement this
2173 found = 1;
2174
2175#else
2176
2177#error "Unknown or unsupported OS"
2178
2179#endif
2180
2181 return found;
2182
2183} // __kmp_is_address_mapped
2184
2185#ifdef USE_LOAD_BALANCE
2186
2187#if KMP_OS_DARWIN || KMP_OS_NETBSD
2188
2189// The function returns the rounded value of the system load average
2190// during given time interval which depends on the value of
2191// __kmp_load_balance_interval variable (default is 60 sec, other values
2192// may be 300 sec or 900 sec).
2193// It returns -1 in case of error.
2194int __kmp_get_load_balance(int max) {
2195 double averages[3];
2196 int ret_avg = 0;
2197
2198 int res = getloadavg(averages, 3);
2199
2200 // Check __kmp_load_balance_interval to determine which of averages to use.
2201 // getloadavg() may return the number of samples less than requested that is
2202 // less than 3.
2203 if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2204 ret_avg = (int)averages[0]; // 1 min
2205 } else if ((__kmp_load_balance_interval >= 180 &&
2206 __kmp_load_balance_interval < 600) &&
2207 (res >= 2)) {
2208 ret_avg = (int)averages[1]; // 5 min
2209 } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2210 ret_avg = (int)averages[2]; // 15 min
2211 } else { // Error occurred
2212 return -1;
2213 }
2214
2215 return ret_avg;
2216}
2217
2218#else // Linux* OS
2219
2220// The function returns number of running (not sleeping) threads, or -1 in case
2221// of error. Error could be reported if Linux* OS kernel too old (without
2222// "/proc" support). Counting running threads stops if max running threads
2223// encountered.
2224int __kmp_get_load_balance(int max) {
2225 static int permanent_error = 0;
2226 static int glb_running_threads = 0; // Saved count of the running threads for
2227 // the thread balance algorithm
2228 static double glb_call_time = 0; /* Thread balance algorithm call time */
2229
2230 int running_threads = 0; // Number of running threads in the system.
2231
2232 DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2233 struct dirent *proc_entry = NULL;
2234
2235 kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2236 DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2237 struct dirent *task_entry = NULL;
2238 int task_path_fixed_len;
2239
2240 kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2241 int stat_file = -1;
2242 int stat_path_fixed_len;
2243
2244#ifdef KMP_DEBUG
2245 int total_processes = 0; // Total number of processes in system.
2246#endif
2247
2248 double call_time = 0.0;
2249
2250 __kmp_str_buf_init(&task_path);
2251 __kmp_str_buf_init(&stat_path);
2252
2253 __kmp_elapsed(&call_time);
2254
2255 if (glb_call_time &&
2256 (call_time - glb_call_time < __kmp_load_balance_interval)) {
2257 running_threads = glb_running_threads;
2258 goto finish;
2259 }
2260
2261 glb_call_time = call_time;
2262
2263 // Do not spend time on scanning "/proc/" if we have a permanent error.
2264 if (permanent_error) {
2265 running_threads = -1;
2266 goto finish;
2267 }
2268
2269 if (max <= 0) {
2270 max = INT_MAX;
2271 }
2272
2273 // Open "/proc/" directory.
2274 proc_dir = opendir("/proc");
2275 if (proc_dir == NULL) {
2276 // Cannot open "/prroc/". Probably the kernel does not support it. Return an
2277 // error now and in subsequent calls.
2278 running_threads = -1;
2279 permanent_error = 1;
2280 goto finish;
2281 }
2282
2283 // Initialize fixed part of task_path. This part will not change.
2284 __kmp_str_buf_cat(&task_path, "/proc/", 6);
2285 task_path_fixed_len = task_path.used; // Remember number of used characters.
2286
2287 proc_entry = readdir(proc_dir);
2288 while (proc_entry != NULL) {
2289 // Proc entry is a directory and name starts with a digit. Assume it is a
2290 // process' directory.
2291 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2292
2293#ifdef KMP_DEBUG
2294 ++total_processes;
2295#endif
2296 // Make sure init process is the very first in "/proc", so we can replace
2297 // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2298 // 1. We are going to check that total_processes == 1 => d_name == "1" is
2299 // true (where "=>" is implication). Since C++ does not have => operator,
2300 // let us replace it with its equivalent: a => b == ! a || b.
2301 KMP_DEBUG_ASSERT(total_processes != 1 ||
2302 strcmp(proc_entry->d_name, "1") == 0);
2303
2304 // Construct task_path.
2305 task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2306 __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2307 KMP_STRLEN(proc_entry->d_name));
2308 __kmp_str_buf_cat(&task_path, "/task", 5);
2309
2310 task_dir = opendir(task_path.str);
2311 if (task_dir == NULL) {
2312 // Process can finish between reading "/proc/" directory entry and
2313 // opening process' "task/" directory. So, in general case we should not
2314 // complain, but have to skip this process and read the next one. But on
2315 // systems with no "task/" support we will spend lot of time to scan
2316 // "/proc/" tree again and again without any benefit. "init" process
2317 // (its pid is 1) should exist always, so, if we cannot open
2318 // "/proc/1/task/" directory, it means "task/" is not supported by
2319 // kernel. Report an error now and in the future.
2320 if (strcmp(proc_entry->d_name, "1") == 0) {
2321 running_threads = -1;
2322 permanent_error = 1;
2323 goto finish;
2324 }
2325 } else {
2326 // Construct fixed part of stat file path.
2327 __kmp_str_buf_clear(&stat_path);
2328 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2329 __kmp_str_buf_cat(&stat_path, "/", 1);
2330 stat_path_fixed_len = stat_path.used;
2331
2332 task_entry = readdir(task_dir);
2333 while (task_entry != NULL) {
2334 // It is a directory and name starts with a digit.
2335 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2336
2337 // Construct complete stat file path. Easiest way would be:
2338 // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2339 // task_entry->d_name );
2340 // but seriae of __kmp_str_buf_cat works a bit faster.
2341 stat_path.used =
2342 stat_path_fixed_len; // Reset stat path to its fixed part.
2343 __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2344 KMP_STRLEN(task_entry->d_name));
2345 __kmp_str_buf_cat(&stat_path, "/stat", 5);
2346
2347 // Note: Low-level API (open/read/close) is used. High-level API
2348 // (fopen/fclose) works ~ 30 % slower.
2349 stat_file = open(stat_path.str, O_RDONLY);
2350 if (stat_file == -1) {
2351 // We cannot report an error because task (thread) can terminate
2352 // just before reading this file.
2353 } else {
2354 /* Content of "stat" file looks like:
2355 24285 (program) S ...
2356
2357 It is a single line (if program name does not include funny
2358 symbols). First number is a thread id, then name of executable
2359 file name in paretheses, then state of the thread. We need just
2360 thread state.
2361
2362 Good news: Length of program name is 15 characters max. Longer
2363 names are truncated.
2364
2365 Thus, we need rather short buffer: 15 chars for program name +
2366 2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2367
2368 Bad news: Program name may contain special symbols like space,
2369 closing parenthesis, or even new line. This makes parsing
2370 "stat" file not 100 % reliable. In case of fanny program names
2371 parsing may fail (report incorrect thread state).
2372
2373 Parsing "status" file looks more promissing (due to different
2374 file structure and escaping special symbols) but reading and
2375 parsing of "status" file works slower.
2376 -- ln
2377 */
2378 char buffer[65];
2379 ssize_t len;
2380 len = read(stat_file, buffer, sizeof(buffer) - 1);
2381 if (len >= 0) {
2382 buffer[len] = 0;
2383 // Using scanf:
2384 // sscanf( buffer, "%*d (%*s) %c ", & state );
2385 // looks very nice, but searching for a closing parenthesis
2386 // works a bit faster.
2387 char *close_parent = strstr(buffer, ") ");
2388 if (close_parent != NULL) {
2389 char state = *(close_parent + 2);
2390 if (state == 'R') {
2391 ++running_threads;
2392 if (running_threads >= max) {
2393 goto finish;
2394 }
2395 }
2396 }
2397 }
2398 close(stat_file);
2399 stat_file = -1;
2400 }
2401 }
2402 task_entry = readdir(task_dir);
2403 }
2404 closedir(task_dir);
2405 task_dir = NULL;
2406 }
2407 }
2408 proc_entry = readdir(proc_dir);
2409 }
2410
2411 // There _might_ be a timing hole where the thread executing this
2412 // code get skipped in the load balance, and running_threads is 0.
2413 // Assert in the debug builds only!!!
2414 KMP_DEBUG_ASSERT(running_threads > 0);
2415 if (running_threads <= 0) {
2416 running_threads = 1;
2417 }
2418
2419finish: // Clean up and exit.
2420 if (proc_dir != NULL) {
2421 closedir(proc_dir);
2422 }
2423 __kmp_str_buf_free(&task_path);
2424 if (task_dir != NULL) {
2425 closedir(task_dir);
2426 }
2427 __kmp_str_buf_free(&stat_path);
2428 if (stat_file != -1) {
2429 close(stat_file);
2430 }
2431
2432 glb_running_threads = running_threads;
2433
2434 return running_threads;
2435
2436} // __kmp_get_load_balance
2437
2438#endif // KMP_OS_DARWIN
2439
2440#endif // USE_LOAD_BALANCE
2441
2442#if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2443 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2444 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64)
2445
2446// we really only need the case with 1 argument, because CLANG always build
2447// a struct of pointers to shared variables referenced in the outlined function
2448int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2449 void *p_argv[]
2450#if OMPT_SUPPORT
2451 ,
2452 void **exit_frame_ptr
2453#endif
2454) {
2455#if OMPT_SUPPORT
2456 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2457#endif
2458
2459 switch (argc) {
2460 default:
2461 fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2462 fflush(stderr);
2463 exit(-1);
2464 case 0:
2465 (*pkfn)(&gtid, &tid);
2466 break;
2467 case 1:
2468 (*pkfn)(&gtid, &tid, p_argv[0]);
2469 break;
2470 case 2:
2471 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2472 break;
2473 case 3:
2474 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2475 break;
2476 case 4:
2477 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2478 break;
2479 case 5:
2480 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2481 break;
2482 case 6:
2483 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2484 p_argv[5]);
2485 break;
2486 case 7:
2487 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2488 p_argv[5], p_argv[6]);
2489 break;
2490 case 8:
2491 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2492 p_argv[5], p_argv[6], p_argv[7]);
2493 break;
2494 case 9:
2495 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2496 p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2497 break;
2498 case 10:
2499 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2500 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2501 break;
2502 case 11:
2503 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2504 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2505 break;
2506 case 12:
2507 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2508 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2509 p_argv[11]);
2510 break;
2511 case 13:
2512 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2513 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2514 p_argv[11], p_argv[12]);
2515 break;
2516 case 14:
2517 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2518 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2519 p_argv[11], p_argv[12], p_argv[13]);
2520 break;
2521 case 15:
2522 (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2523 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2524 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2525 break;
2526 }
2527
2528 return 1;
2529}
2530
2531#endif
2532
2533#if KMP_OS_LINUX
2534// Functions for hidden helper task
2535namespace {
2536// Condition variable for initializing hidden helper team
2537pthread_cond_t hidden_helper_threads_initz_cond_var;
2538pthread_mutex_t hidden_helper_threads_initz_lock;
2539volatile int hidden_helper_initz_signaled = FALSE;
2540
2541// Condition variable for deinitializing hidden helper team
2542pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2543pthread_mutex_t hidden_helper_threads_deinitz_lock;
2544volatile int hidden_helper_deinitz_signaled = FALSE;
2545
2546// Condition variable for the wrapper function of main thread
2547pthread_cond_t hidden_helper_main_thread_cond_var;
2548pthread_mutex_t hidden_helper_main_thread_lock;
2549volatile int hidden_helper_main_thread_signaled = FALSE;
2550
2551// Semaphore for worker threads. We don't use condition variable here in case
2552// that when multiple signals are sent at the same time, only one thread might
2553// be waken.
2554sem_t hidden_helper_task_sem;
2555} // namespace
2556
2557void __kmp_hidden_helper_worker_thread_wait() {
2558 int status = sem_wait(&hidden_helper_task_sem);
2559 KMP_CHECK_SYSFAIL("sem_wait", status);
2560}
2561
2562void __kmp_do_initialize_hidden_helper_threads() {
2563 // Initialize condition variable
2564 int status =
2565 pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr);
2566 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2567
2568 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr);
2569 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2570
2571 status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr);
2572 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2573
2574 status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr);
2575 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2576
2577 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr);
2578 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2579
2580 status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr);
2581 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2582
2583 // Initialize the semaphore
2584 status = sem_init(&hidden_helper_task_sem, 0, 0);
2585 KMP_CHECK_SYSFAIL("sem_init", status);
2586
2587 // Create a new thread to finish initialization
2588 pthread_t handle;
2589 status = pthread_create(
2590 &handle, nullptr,
2591 [](void *) -> void * {
2592 __kmp_hidden_helper_threads_initz_routine();
2593 return nullptr;
2594 },
2595 nullptr);
2596 KMP_CHECK_SYSFAIL("pthread_create", status);
2597}
2598
2599void __kmp_hidden_helper_threads_initz_wait() {
2600 // Initial thread waits here for the completion of the initialization. The
2601 // condition variable will be notified by main thread of hidden helper teams.
2602 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2603 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2604
2605 if (!TCR_4(hidden_helper_initz_signaled)) {
2606 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var,
2607 &hidden_helper_threads_initz_lock);
2608 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2609 }
2610
2611 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2612 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2613}
2614
2615void __kmp_hidden_helper_initz_release() {
2616 // After all initialization, reset __kmp_init_hidden_helper_threads to false.
2617 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2618 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2619
2620 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var);
2621 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2622
2623 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2624
2625 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2626 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2627}
2628
2629void __kmp_hidden_helper_main_thread_wait() {
2630 // The main thread of hidden helper team will be blocked here. The
2631 // condition variable can only be signal in the destructor of RTL.
2632 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2633 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2634
2635 if (!TCR_4(hidden_helper_main_thread_signaled)) {
2636 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var,
2637 &hidden_helper_main_thread_lock);
2638 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2639 }
2640
2641 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2642 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2643}
2644
2645void __kmp_hidden_helper_main_thread_release() {
2646 // The initial thread of OpenMP RTL should call this function to wake up the
2647 // main thread of hidden helper team.
2648 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2649 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2650
2651 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var);
2652 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
2653
2654 // The hidden helper team is done here
2655 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
2656
2657 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2658 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2659}
2660
2661void __kmp_hidden_helper_worker_thread_signal() {
2662 int status = sem_post(&hidden_helper_task_sem);
2663 KMP_CHECK_SYSFAIL("sem_post", status);
2664}
2665
2666void __kmp_hidden_helper_threads_deinitz_wait() {
2667 // Initial thread waits here for the completion of the deinitialization. The
2668 // condition variable will be notified by main thread of hidden helper teams.
2669 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
2670 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2671
2672 if (!TCR_4(hidden_helper_deinitz_signaled)) {
2673 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var,
2674 &hidden_helper_threads_deinitz_lock);
2675 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2676 }
2677
2678 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
2679 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2680}
2681
2682void __kmp_hidden_helper_threads_deinitz_release() {
2683 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
2684 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2685
2686 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var);
2687 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2688
2689 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
2690
2691 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
2692 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2693}
2694#else // KMP_OS_LINUX
2695void __kmp_hidden_helper_worker_thread_wait() {
2696 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2697}
2698
2699void __kmp_do_initialize_hidden_helper_threads() {
2700 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2701}
2702
2703void __kmp_hidden_helper_threads_initz_wait() {
2704 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2705}
2706
2707void __kmp_hidden_helper_initz_release() {
2708 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2709}
2710
2711void __kmp_hidden_helper_main_thread_wait() {
2712 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2713}
2714
2715void __kmp_hidden_helper_main_thread_release() {
2716 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2717}
2718
2719void __kmp_hidden_helper_worker_thread_signal() {
2720 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2721}
2722
2723void __kmp_hidden_helper_threads_deinitz_wait() {
2724 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2725}
2726
2727void __kmp_hidden_helper_threads_deinitz_release() {
2728 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
2729}
2730#endif // KMP_OS_LINUX
2731
2732// end of file //
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.
Definition: kmp_stats.h:937