28 #include <common/config.h>
41 #ifdef STARPU_HAVE_HWLOC
53 #ifdef STARPU_USE_MPI_MASTER_SLAVE
63 #define STARPU_MAX_PIPELINE 4
65 enum initialization { UNINITIALIZED = 0, CHANGING, INITIALIZED };
67 struct _starpu_ctx_change_list;
72 starpu_pthread_mutex_t mutex;
73 enum starpu_worker_archtype arch;
75 struct starpu_perfmodel_arch perf_arch;
95 #ifdef STARPU_SPINLOCK_CHECK
96 const char *relax_on_file;
98 const char *relax_on_func;
99 const char *relax_off_file;
101 const char *relax_off_func;
120 starpu_pthread_t thread_changing_ctx;
128 struct _starpu_ctx_change_list ctx_change_list;
129 struct starpu_task_list local_tasks;
135 struct starpu_task *current_tasks[STARPU_MAX_PIPELINE];
136 #ifdef STARPU_SIMGRID
137 starpu_pthread_wait_t wait;
140 struct timespec cl_start;
141 struct timespec cl_expend;
142 struct timespec cl_end;
148 unsigned worker_is_running;
149 unsigned worker_is_initialized;
164 unsigned removed_from_ctx[STARPU_NMAX_SCHED_CTXS+1];
176 unsigned shares_tasks_lists[STARPU_NMAX_SCHED_CTXS+1];
178 unsigned poped_in_ctx[STARPU_NMAX_SCHED_CTXS+1];
184 unsigned reverse_phase[2];
194 #ifdef STARPU_HAVE_HWLOC
195 hwloc_bitmap_t hwloc_cpu_set;
196 hwloc_obj_t hwloc_obj;
200 char padding[STARPU_CACHELINE_SIZE];
209 int combined_workerid[STARPU_NMAXWORKERS];
212 starpu_pthread_mutex_t count_mutex;
218 #ifdef STARPU_HAVE_HWLOC
219 hwloc_bitmap_t hwloc_cpu_set;
233 starpu_pthread_mutex_t mutex;
240 unsigned set_is_initialized;
243 #ifdef STARPU_USE_MPI_MASTER_SLAVE
255 unsigned nsched_ctxs;
257 #ifdef STARPU_HAVE_HWLOC
294 unsigned nworkerpercuda;
295 int cuda_th_per_stream;
303 unsigned nhwmpidevices;
306 unsigned nmpicores[STARPU_MAXMPIDEVS];
311 unsigned nmicdevices;
314 unsigned nmiccores[STARPU_MAXMICDEVS];
362 #ifdef STARPU_HAVE_HWLOC
369 char currently_bound[STARPU_NMAXWORKERS];
370 char currently_shared[STARPU_NMAXWORKERS];
406 starpu_pthread_mutex_t submitted_mutex;
425 struct starpu_conf
conf;
445 extern int _starpu_worker_parallel_blocks;
448 extern int _starpu_keys_initialized STARPU_ATTRIBUTE_INTERNAL;
449 extern starpu_pthread_key_t _starpu_worker_key STARPU_ATTRIBUTE_INTERNAL;
450 extern starpu_pthread_key_t _starpu_worker_set_key STARPU_ATTRIBUTE_INTERNAL;
453 void _starpu_set_argc_argv(
int *argc,
char ***argv);
454 int *_starpu_get_argc();
455 char ***_starpu_get_argv();
458 void _starpu_conf_check_environment(
struct starpu_conf *
conf);
461 void _starpu_may_pause(
void);
464 static inline unsigned _starpu_machine_is_running(
void)
470 ANNOTATE_HAPPENS_AFTER(&_starpu_config.running);
471 ret = _starpu_config.running;
472 ANNOTATE_HAPPENS_BEFORE(&_starpu_config.running);
481 uint32_t _starpu_worker_exists(
struct starpu_task *);
484 uint32_t _starpu_can_submit_cuda_task(
void);
487 uint32_t _starpu_can_submit_cpu_task(
void);
490 uint32_t _starpu_can_submit_opencl_task(
void);
494 unsigned _starpu_worker_can_block(
unsigned memnode,
struct _starpu_worker *worker);
499 void _starpu_block_worker(
int workerid, starpu_pthread_cond_t *cond, starpu_pthread_mutex_t *mutex);
502 void _starpu_driver_start(
struct _starpu_worker *worker,
unsigned fut_key,
unsigned sync);
504 void _starpu_worker_start(
struct _starpu_worker *worker,
unsigned fut_key,
unsigned sync);
506 static inline unsigned _starpu_worker_get_count(
void)
508 return _starpu_config.topology.nworkers;
510 #define starpu_worker_get_count _starpu_worker_get_count
515 static inline void _starpu_set_local_worker_key(
struct _starpu_worker *worker)
517 STARPU_ASSERT(_starpu_keys_initialized);
518 STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_key, worker);
523 static inline struct _starpu_worker *_starpu_get_local_worker_key(
void)
525 if (!_starpu_keys_initialized)
527 return (
struct _starpu_worker *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_key);
533 static inline void _starpu_set_local_worker_set_key(
struct _starpu_worker_set *worker)
535 STARPU_ASSERT(_starpu_keys_initialized);
536 STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_set_key, worker);
543 if (!_starpu_keys_initialized)
545 return (
struct _starpu_worker_set *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_set_key);
550 static inline struct _starpu_worker *_starpu_get_worker_struct(
unsigned id)
552 STARPU_ASSERT(
id < starpu_worker_get_count());
553 return &_starpu_config.workers[id];
558 static inline struct _starpu_sched_ctx *_starpu_get_sched_ctx_struct(
unsigned id)
560 return (
id > STARPU_NMAX_SCHED_CTXS) ? NULL : &_starpu_config.sched_ctxs[
id];
569 return &_starpu_config;
573 static inline int _starpu_get_disable_kernels(
void)
575 return _starpu_config.disable_kernels;
581 return _starpu_config.workers[workerid].status;
588 _starpu_config.workers[workerid].status = status;
594 return &_starpu_config.sched_ctxs[STARPU_GLOBAL_SCHED_CTX];
597 int starpu_worker_get_nids_by_type(
enum starpu_worker_archtype type,
int *workerids,
int maxsize);
601 int starpu_worker_get_nids_ctx_free_by_type(
enum starpu_worker_archtype type,
int *workerids,
int maxsize);
603 static inline unsigned _starpu_worker_mutex_is_sched_mutex(
int workerid, starpu_pthread_mutex_t *mutex)
609 static inline int _starpu_worker_get_nsched_ctxs(
int workerid)
611 return _starpu_config.workers[
workerid].nsched_ctxs;
615 static inline unsigned _starpu_get_nsched_ctxs(
void)
619 return _starpu_config.topology.nsched_ctxs;
623 static inline int _starpu_worker_get_id(
void)
627 worker = _starpu_get_local_worker_key();
639 #define starpu_worker_get_id _starpu_worker_get_id
643 static inline unsigned __starpu_worker_get_id_check(
const char *f,
int l)
647 int id = starpu_worker_get_id();
648 STARPU_ASSERT_MSG(
id>=0,
"%s:%d Cannot be called from outside a worker\n", f, l);
651 #define _starpu_worker_get_id_check(f,l) __starpu_worker_get_id_check(f,l)
653 enum starpu_node_kind _starpu_worker_get_node_kind(
enum starpu_worker_archtype type);
657 struct _starpu_sched_ctx* _starpu_worker_get_ctx_stream(
unsigned stream_workerid);
664 static inline void _starpu_worker_request_blocking_in_parallel(
struct _starpu_worker *
const worker)
666 _starpu_worker_parallel_blocks = 1;
687 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
688 #ifdef STARPU_SIMGRID
689 starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->
workerid]);
705 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
713 static inline void _starpu_worker_request_unblocking_in_parallel(
struct _starpu_worker *
const worker)
734 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
748 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
762 static inline void _starpu_worker_process_block_in_parallel_requests(
struct _starpu_worker *
const worker)
777 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
794 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
814 #ifdef STARPU_SPINLOCK_CHECK
815 static inline void __starpu_worker_enter_sched_op(
struct _starpu_worker *
const worker,
const char*file,
int line,
const char* func)
817 static inline void _starpu_worker_enter_sched_op(
struct _starpu_worker *
const worker)
824 _starpu_worker_process_block_in_parallel_requests(worker);
831 _starpu_worker_process_block_in_parallel_requests(worker);
853 #ifdef STARPU_SPINLOCK_CHECK
854 worker->relax_on_file = file;
855 worker->relax_on_line = line;
856 worker->relax_on_func = func;
859 #ifdef STARPU_SPINLOCK_CHECK
860 #define _starpu_worker_enter_sched_op(worker) __starpu_worker_enter_sched_op((worker), __FILE__, __LINE__, __starpu_func__)
868 #ifdef STARPU_SPINLOCK_CHECK
869 static inline void __starpu_worker_leave_sched_op(
struct _starpu_worker *
const worker,
const char*file,
int line,
const char* func)
871 static inline void _starpu_worker_leave_sched_op(
struct _starpu_worker *
const worker)
876 #ifdef STARPU_SPINLOCK_CHECK
877 worker->relax_off_file = file;
878 worker->relax_off_line = line;
879 worker->relax_off_func = func;
882 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
885 #ifdef STARPU_SPINLOCK_CHECK
886 #define _starpu_worker_leave_sched_op(worker) __starpu_worker_leave_sched_op((worker), __FILE__, __LINE__, __starpu_func__)
889 static inline int _starpu_worker_sched_op_pending(
void)
891 int workerid = starpu_worker_get_id();
895 STARPU_ASSERT(worker != NULL);
908 static inline void _starpu_worker_enter_changing_ctx_op(
struct _starpu_worker *
const worker)
931 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
932 #ifdef STARPU_SIMGRID
933 starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->
workerid]);
950 static inline void _starpu_worker_leave_changing_ctx_op(
struct _starpu_worker *
const worker)
954 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
959 #ifdef STARPU_SPINLOCK_CHECK
960 static inline void __starpu_worker_relax_on(
const char*file,
int line,
const char* func)
962 static inline void _starpu_worker_relax_on(
void)
970 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
971 #ifdef STARPU_SPINLOCK_CHECK
972 STARPU_ASSERT_MSG(worker->
state_relax_refcnt<UINT_MAX,
"relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
977 #ifdef STARPU_SPINLOCK_CHECK
978 worker->relax_on_file = file;
979 worker->relax_on_line = line;
980 worker->relax_on_func = func;
982 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
983 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
985 #ifdef STARPU_SPINLOCK_CHECK
986 #define _starpu_worker_relax_on() __starpu_worker_relax_on(__FILE__, __LINE__, __starpu_func__)
988 #define starpu_worker_relax_on _starpu_worker_relax_on
991 #ifdef STARPU_SPINLOCK_CHECK
992 static inline void __starpu_worker_relax_on_locked(
struct _starpu_worker *worker,
const char*file,
int line,
const char* func)
994 static inline void _starpu_worker_relax_on_locked(
struct _starpu_worker *worker)
999 #ifdef STARPU_SPINLOCK_CHECK
1000 STARPU_ASSERT_MSG(worker->
state_relax_refcnt<UINT_MAX,
"relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1005 #ifdef STARPU_SPINLOCK_CHECK
1006 worker->relax_on_file = file;
1007 worker->relax_on_line = line;
1008 worker->relax_on_func = func;
1010 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
1012 #ifdef STARPU_SPINLOCK_CHECK
1013 #define _starpu_worker_relax_on_locked(worker) __starpu_worker_relax_on_locked(worker,__FILE__, __LINE__, __starpu_func__)
1016 #ifdef STARPU_SPINLOCK_CHECK
1017 static inline void __starpu_worker_relax_off(
const char*file,
int line,
const char* func)
1019 static inline void _starpu_worker_relax_off(
void)
1022 int workerid = starpu_worker_get_id();
1026 STARPU_ASSERT(worker != NULL);
1029 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1030 #ifdef STARPU_SPINLOCK_CHECK
1031 STARPU_ASSERT_MSG(worker->
state_relax_refcnt>0,
"relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1036 #ifdef STARPU_SPINLOCK_CHECK
1037 worker->relax_off_file = file;
1038 worker->relax_off_line = line;
1039 worker->relax_off_func = func;
1041 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1043 #ifdef STARPU_SPINLOCK_CHECK
1044 #define _starpu_worker_relax_off() __starpu_worker_relax_off(__FILE__, __LINE__, __starpu_func__)
1046 #define starpu_worker_relax_off _starpu_worker_relax_off
1048 #ifdef STARPU_SPINLOCK_CHECK
1049 static inline void __starpu_worker_relax_off_locked(
const char*file,
int line,
const char* func)
1051 static inline void _starpu_worker_relax_off_locked(
void)
1054 int workerid = starpu_worker_get_id();
1058 STARPU_ASSERT(worker != NULL);
1061 #ifdef STARPU_SPINLOCK_CHECK
1062 STARPU_ASSERT_MSG(worker->
state_relax_refcnt>0,
"relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1067 #ifdef STARPU_SPINLOCK_CHECK
1068 worker->relax_off_file = file;
1069 worker->relax_off_line = line;
1070 worker->relax_off_func = func;
1073 #ifdef STARPU_SPINLOCK_CHECK
1074 #define _starpu_worker_relax_off_locked() __starpu_worker_relax_off_locked(__FILE__, __LINE__, __starpu_func__)
1077 static inline int _starpu_worker_get_relax_state(
void)
1079 int workerid = starpu_worker_get_id();
1083 STARPU_ASSERT(worker != NULL);
1086 #define starpu_worker_get_relax_state _starpu_worker_get_relax_state
1092 static inline void _starpu_worker_lock(
int workerid)
1095 STARPU_ASSERT(worker != NULL);
1096 int cur_workerid = starpu_worker_get_id();
1099 starpu_worker_relax_on();
1101 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1109 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1113 static inline int _starpu_worker_trylock(
int workerid)
1115 struct _starpu_worker *cur_worker = _starpu_get_local_worker_key();
1116 int cur_workerid = cur_worker->
workerid;
1118 STARPU_ASSERT(worker != NULL);
1121 int ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&cur_worker->
sched_mutex);
1129 ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&worker->
sched_mutex);
1135 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1138 _starpu_worker_relax_on_locked(cur_worker);
1139 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&cur_worker->
sched_mutex);
1143 static inline void _starpu_worker_unlock(
int workerid)
1146 STARPU_ASSERT(worker != NULL);
1147 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1148 int cur_workerid = starpu_worker_get_id();
1151 starpu_worker_relax_off();
1155 static inline void _starpu_worker_lock_self(
void)
1157 int workerid = starpu_worker_get_id_check();
1159 STARPU_ASSERT(worker != NULL);
1160 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1163 static inline void _starpu_worker_unlock_self(
void)
1165 int workerid = starpu_worker_get_id_check();
1167 STARPU_ASSERT(worker != NULL);
1168 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1171 static inline int _starpu_wake_worker_relax(
int workerid)
1174 int ret = starpu_wake_worker_locked(
workerid);
1179 int starpu_wake_worker_relax_light(
int workerid);
1185 void _starpu_worker_refuse_task(
struct _starpu_worker *worker,
struct starpu_task *task);
Definition: barrier_counter.h:27
_starpu_worker_status
Definition: errorcheck.h:26
void _starpu_worker_apply_deferred_ctx_changes(void)
Definition: sched_ctx_list.h:25
Definition: workers.h:204
char padding[STARPU_CACHELINE_SIZE]
Definition: workers.h:224
unsigned memory_node
Definition: workers.h:208
struct starpu_perfmodel_arch perf_arch
Definition: workers.h:205
uint32_t worker_mask
Definition: workers.h:206
Definition: workers.h:359
int mpi_nodeid
Definition: workers.h:393
struct _starpu_combined_worker combined_workers[STARPU_NMAX_COMBINEDWORKERS]
Definition: workers.h:404
int opencl_nodeid
Definition: workers.h:389
unsigned running
Definition: workers.h:428
int current_cuda_gpuid
Definition: workers.h:373
unsigned nbindid
Definition: workers.h:417
int pause_depth
Definition: workers.h:434
int current_opencl_gpuid
Definition: workers.h:376
int cpus_nodeid
Definition: workers.h:385
int mic_nodeid
Definition: workers.h:391
struct _starpu_machine_config::@4 * bindid_workers
struct _starpu_sched_ctx sched_ctxs[STARPU_NMAX_SCHED_CTXS+1]
Definition: workers.h:437
int current_bindid
Definition: workers.h:368
struct _starpu_worker workers[STARPU_NMAXWORKERS]
Definition: workers.h:400
uint32_t worker_mask
Definition: workers.h:422
int current_mic_deviceid
Definition: workers.h:379
int cuda_nodeid
Definition: workers.h:387
char padding2[STARPU_CACHELINE_SIZE]
Definition: workers.h:409
unsigned submitting
Definition: workers.h:440
char padding1[STARPU_CACHELINE_SIZE]
Definition: workers.h:396
int current_mpi_deviceid
Definition: workers.h:382
struct starpu_conf conf
Definition: workers.h:425
Definition: workers.h:248
struct starpu_tree * tree
Definition: workers.h:262
unsigned nmpidevices
Definition: workers.h:302
unsigned nhwmicdevices
Definition: workers.h:310
hwloc_topology_t hwtopology
Definition: workers.h:259
unsigned workers_mpi_ms_deviceid[STARPU_NMAXWORKERS]
Definition: workers.h:355
unsigned workers_cuda_gpuid[STARPU_NMAXWORKERS]
Definition: workers.h:337
unsigned nhwopenclgpus
Definition: workers.h:282
unsigned workers_opencl_gpuid[STARPU_NMAXWORKERS]
Definition: workers.h:345
unsigned nworkers
Definition: workers.h:250
unsigned ncudagpus
Definition: workers.h:293
unsigned nhwmpi
Definition: workers.h:287
unsigned ncpus
Definition: workers.h:290
unsigned ncombinedworkers
Definition: workers.h:253
unsigned nhwcudagpus
Definition: workers.h:277
unsigned nopenclgpus
Definition: workers.h:299
unsigned workers_nbindid
Definition: workers.h:329
unsigned nhwmpicores[STARPU_MAXMPIDEVS]
Definition: workers.h:305
unsigned nhwpus
Definition: workers.h:272
unsigned workers_bindid[STARPU_NMAXWORKERS]
Definition: workers.h:323
unsigned nhwmiccores[STARPU_MAXMICDEVS]
Definition: workers.h:313
unsigned nhwcpus
Definition: workers.h:267
Definition: sched_ctx.h:46
unsigned id
Definition: sched_ctx.h:48
Definition: workers.h:232
unsigned started
Definition: workers.h:236
starpu_pthread_t worker_thread
Definition: workers.h:234
starpu_pthread_cond_t ready_cond
Definition: workers.h:239
unsigned state_blocked_in_parallel_observed
Definition: workers.h:107
struct starpu_task * current_task
Definition: workers.h:134
unsigned state_block_in_parallel_ack
Definition: workers.h:109
unsigned state_relax_refcnt
Definition: workers.h:94
struct starpu_task * task_transferring
Definition: workers.h:170
unsigned state_changing_ctx_waiting
Definition: workers.h:104
unsigned subworkerid
Definition: workers.h:78
unsigned devid
Definition: workers.h:77
starpu_pthread_t thread_changing_ctx
Definition: workers.h:120
int current_rank
Definition: workers.h:82
unsigned state_unblock_in_parallel_req
Definition: workers.h:110
unsigned nb_buffers_transferred
Definition: workers.h:168
starpu_pthread_t worker_thread
Definition: workers.h:76
starpu_pthread_cond_t ready_cond
Definition: workers.h:85
unsigned memory_node
Definition: workers.h:86
int workerid
Definition: workers.h:80
unsigned nb_buffers_totransfer
Definition: workers.h:169
unsigned state_keep_awake
Definition: workers.h:151
struct _starpu_worker_set * set
Definition: workers.h:147
int worker_size
Definition: workers.h:83
unsigned pop_ctx_priority
Definition: workers.h:186
unsigned state_unblock_in_parallel_ack
Definition: workers.h:111
unsigned numa_memory_node
Definition: workers.h:87
unsigned char pipeline_stuck
Definition: workers.h:146
unsigned char pipeline_length
Definition: workers.h:145
unsigned nsched_ctxs
Definition: workers.h:159
unsigned spinning_backoff
Definition: workers.h:166
unsigned state_blocked_in_parallel
Definition: workers.h:106
unsigned current_ordered_task_order
Definition: workers.h:133
starpu_pthread_cond_t started_cond
Definition: workers.h:84
starpu_pthread_mutex_t sched_mutex
Definition: workers.h:93
unsigned run_by_starpu
Definition: workers.h:154
unsigned local_ordered_tasks_size
Definition: workers.h:131
struct starpu_task ** local_ordered_tasks
Definition: workers.h:130
unsigned is_slave_somewhere
Definition: workers.h:187
starpu_pthread_cond_t sched_cond
Definition: workers.h:92
unsigned has_prev_init
Definition: workers.h:162
unsigned char ntasks
Definition: workers.h:144
int bindid
Definition: workers.h:79
unsigned state_block_in_parallel_req
Definition: workers.h:108
unsigned state_changing_ctx_notice
Definition: workers.h:105
unsigned state_sched_op_pending
Definition: workers.h:103
int combined_workerid
Definition: workers.h:81
unsigned char first_task
Definition: workers.h:143
unsigned current_ordered_task
Definition: workers.h:132
unsigned block_in_parallel_ref_count
Definition: workers.h:119