From 13f0cede049eea5e7069b8d2f5802d8dd986cc5e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 31 Jan 2019 20:55:14 +0900 Subject: [PATCH 1/4] bpo-32243: add pthread_condattr_setclock to AC_CHECK_FUNCS --- configure | 3 ++- configure.ac | 3 ++- pyconfig.h.in | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configure b/configure index ebd9f904b09afb6..a8fa0c1616d3e0d 100755 --- a/configure +++ b/configure @@ -11448,7 +11448,8 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ - pthread_init pthread_kill putenv pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pthread_condattr_setclock pthread_init pthread_kill putenv pwrite pwritev pwritev2 \ + readlink readlinkat readv realpath renameat \ sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ diff --git a/configure.ac b/configure.ac index 721edb015ea36fb..41a229d44944aae 100644 --- a/configure.ac +++ b/configure.ac @@ -3506,7 +3506,8 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ - pthread_init pthread_kill putenv pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pthread_condattr_setclock pthread_init pthread_kill putenv pwrite pwritev pwritev2 \ + readlink readlinkat readv realpath renameat \ sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ diff --git a/pyconfig.h.in b/pyconfig.h.in index a2a56230fc13be3..918fdf4c879092c 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -753,6 +753,9 @@ /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES +/* Define to 1 if you have the `pthread_condattr_setclock' function. */ +#undef HAVE_PTHREAD_CONDATTR_SETCLOCK + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR From 2deb0f6cb4bed9ec342ac1811052cd6eba5e18bd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Feb 2019 22:03:24 +0900 Subject: [PATCH 2/4] Use pthread_condattr_setclock if possible --- Makefile.pre.in | 5 ++- Python/ceval.c | 1 + Python/condvar.h | 38 ++++------------- Python/thread_pthread.h | 92 ++++++++++++++++++++++++++++++++--------- 4 files changed, 85 insertions(+), 51 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index f8216971958a4e6..1c536f813f8df16 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -939,7 +939,8 @@ regen-opcode-targets: $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new -Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h +Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \ + $(srcdir)/Python/condvar.h Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \ $(srcdir)/Python/importlib_zipimport.h @@ -1838,7 +1839,7 @@ patchcheck: @DEF_MAKE_RULE@ # Dependencies -Python/thread.o: @THREADHEADERS@ +Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h # Declare targets that aren't real files .PHONY: all build_all sharedmods check-clean-src oldsharedmods test quicktest diff --git a/Python/ceval.c b/Python/ceval.c index 3db7c7c92a0e04c..03456f6d2ab4f6f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -171,6 +171,7 @@ PyEval_InitThreads(void) { if (gil_created()) return; + PyThread_init_thread(); create_gil(); take_gil(_PyThreadState_GET()); _PyRuntime.ceval.pending.main_thread = PyThread_get_thread_ident(); diff --git a/Python/condvar.h b/Python/condvar.h index 13517409aeab03f..36b9032bbc1a180 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -48,19 +48,10 @@ * POSIX support */ -#define PyCOND_ADD_MICROSECONDS(tv, interval) \ -do { /* TODO: add overflow and truncation checks */ \ - tv.tv_usec += (long) interval; \ - tv.tv_sec += tv.tv_usec / 1000000; \ - tv.tv_usec %= 1000000; \ -} while (0) - -/* We assume all modern POSIX systems have gettimeofday() */ -#ifdef GETTIMEOFDAY_NO_TZ -#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv) -#else -#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL) -#endif +/* These private functions are implemented in Python/thread_pthread.h */ +int _PyThread_cond_init(PyCOND_T *cond); +void _PyThread_cond_after(long long us, struct timespec *abs); +int _PyThread_cond_timedwait(PyCOND_T *cond, PyMUTEX_T *mut, struct timespec *abs); /* The following functions return 0 on success, nonzero on error */ #define PyMUTEX_INIT(mut) pthread_mutex_init((mut), NULL) @@ -68,7 +59,7 @@ do { /* TODO: add overflow and truncation checks */ \ #define PyMUTEX_LOCK(mut) pthread_mutex_lock(mut) #define PyMUTEX_UNLOCK(mut) pthread_mutex_unlock(mut) -#define PyCOND_INIT(cond) pthread_cond_init((cond), NULL) +#define PyCOND_INIT(cond) _PyThread_cond_init(cond) #define PyCOND_FINI(cond) pthread_cond_destroy(cond) #define PyCOND_SIGNAL(cond) pthread_cond_signal(cond) #define PyCOND_BROADCAST(cond) pthread_cond_broadcast(cond) @@ -78,22 +69,9 @@ do { /* TODO: add overflow and truncation checks */ \ Py_LOCAL_INLINE(int) PyCOND_TIMEDWAIT(PyCOND_T *cond, PyMUTEX_T *mut, long long us) { - int r; - struct timespec ts; - struct timeval deadline; - - PyCOND_GETTIMEOFDAY(&deadline); - PyCOND_ADD_MICROSECONDS(deadline, us); - ts.tv_sec = deadline.tv_sec; - ts.tv_nsec = deadline.tv_usec * 1000; - - r = pthread_cond_timedwait((cond), (mut), &ts); - if (r == ETIMEDOUT) - return 1; - else if (r) - return -1; - else - return 0; + struct timespec abs; + _PyThread_cond_after(us, &abs); + return _PyThread_cond_timedwait(cond, mut, &abs); } #elif defined(NT_THREADS) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 09e53a14aa473ba..299ab0e629600ba 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -56,16 +56,6 @@ #endif #endif -#if !defined(pthread_attr_default) -# define pthread_attr_default ((pthread_attr_t *)NULL) -#endif -#if !defined(pthread_mutexattr_default) -# define pthread_mutexattr_default ((pthread_mutexattr_t *)NULL) -#endif -#if !defined(pthread_condattr_default) -# define pthread_condattr_default ((pthread_condattr_t *)NULL) -#endif - /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: @@ -110,6 +100,70 @@ do { \ } while(0) +/* + * pthread_cond support + */ + +#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) +// monotonic is supported statically. It doesn't mean it works on runtime. +#define CONDATTR_MONOTONIC +#endif + +// NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. +static pthread_condattr_t *condattr_monotonic = NULL; + +static void +init_condattr() +{ +#ifdef CONDATTR_MONOTONIC + static pthread_condattr_t ca; + pthread_condattr_init(&ca); + if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) { + condattr_monotonic = &ca; // Use monotonic clock + } +#endif +} + +int +_PyThread_cond_init(PyCOND_T *cond) +{ + return pthread_cond_init(cond, condattr_monotonic); +} + +void +_PyThread_cond_after(long long us, struct timespec *abs) +{ +#ifdef CONDATTR_MONOTONIC + if (condattr_monotonic) { + clock_gettime(CLOCK_MONOTONIC, abs); + abs->tv_sec += us / 1000000; + abs->tv_nsec += (us % 1000000) * 1000; + abs->tv_sec += abs->tv_nsec / 1000000000; + abs->tv_nsec %= 1000000000; + return; + } +#endif + + struct timespec ts; + MICROSECONDS_TO_TIMESPEC(us, ts); + *abs = ts; +} + +int +_PyThread_cond_timedwait(PyCOND_T *cond, PyMUTEX_T *mut, struct timespec *abs) +{ + int r = pthread_cond_timedwait(cond, mut, abs); + if (r == ETIMEDOUT) { + return 1; + } + if (r) { + return -1; + } + return 0; +} + + + /* A pthread mutex isn't sufficient to model the Python lock type * because, according to Draft 5 of the docs (P1003.4a/D5), both of the * following are undefined: @@ -146,6 +200,7 @@ PyThread__init_thread(void) extern void pthread_init(void); pthread_init(); #endif + init_condattr(); } /* @@ -462,8 +517,7 @@ PyThread_allocate_lock(void) memset((void *)lock, '\0', sizeof(pthread_lock)); lock->locked = 0; - status = pthread_mutex_init(&lock->mut, - pthread_mutexattr_default); + status = pthread_mutex_init(&lock->mut, NULL); CHECK_STATUS_PTHREAD("pthread_mutex_init"); /* Mark the pthread mutex underlying a Python mutex as pure happens-before. We can't simply mark the @@ -472,8 +526,7 @@ PyThread_allocate_lock(void) will cause errors. */ _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(&lock->mut); - status = pthread_cond_init(&lock->lock_released, - pthread_condattr_default); + status = _PyThread_cond_init(&lock->lock_released); CHECK_STATUS_PTHREAD("pthread_cond_init"); if (error) { @@ -532,18 +585,19 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, success = PY_LOCK_ACQUIRED; } else if (microseconds != 0) { - struct timespec ts; - if (microseconds > 0) - MICROSECONDS_TO_TIMESPEC(microseconds, ts); + struct timespec abs; + if (microseconds > 0) { + _PyThread_cond_after(microseconds, &abs); + } /* continue trying until we get the lock */ /* mut must be locked by me -- part of the condition * protocol */ while (success == PY_LOCK_FAILURE) { if (microseconds > 0) { - status = pthread_cond_timedwait( + status = _PyThread_cond_timedwait( &thelock->lock_released, - &thelock->mut, &ts); + &thelock->mut, &abs); if (status == ETIMEDOUT) break; CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); From d6a0eea4dae405089cad5541c150fb4cc1cd4649 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 5 Feb 2019 12:48:27 +0900 Subject: [PATCH 3/4] Add news --- .../Core and Builtins/2019-02-05-12-48-23.bpo-12822.0x2NDx.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-02-05-12-48-23.bpo-12822.0x2NDx.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-02-05-12-48-23.bpo-12822.0x2NDx.rst b/Misc/NEWS.d/next/Core and Builtins/2019-02-05-12-48-23.bpo-12822.0x2NDx.rst new file mode 100644 index 000000000000000..a3de435f527e293 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-02-05-12-48-23.bpo-12822.0x2NDx.rst @@ -0,0 +1,2 @@ +Use monotonic clock for ``pthread_cond_timedwait`` when +``pthread_condattr_setclock`` and ``CLOCK_MONOTONIC`` are available. From 926f48d11b81f91ef38bc8b664bdc92504378fb4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 6 Feb 2019 01:11:58 +0900 Subject: [PATCH 4/4] Fix error on macOS --- Python/condvar.h | 10 ++++++++-- Python/thread_pthread.h | 21 +++++---------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Python/condvar.h b/Python/condvar.h index 36b9032bbc1a180..f54adc5ea8e8297 100644 --- a/Python/condvar.h +++ b/Python/condvar.h @@ -51,7 +51,6 @@ /* These private functions are implemented in Python/thread_pthread.h */ int _PyThread_cond_init(PyCOND_T *cond); void _PyThread_cond_after(long long us, struct timespec *abs); -int _PyThread_cond_timedwait(PyCOND_T *cond, PyMUTEX_T *mut, struct timespec *abs); /* The following functions return 0 on success, nonzero on error */ #define PyMUTEX_INIT(mut) pthread_mutex_init((mut), NULL) @@ -71,7 +70,14 @@ PyCOND_TIMEDWAIT(PyCOND_T *cond, PyMUTEX_T *mut, long long us) { struct timespec abs; _PyThread_cond_after(us, &abs); - return _PyThread_cond_timedwait(cond, mut, &abs); + int ret = pthread_cond_timedwait(cond, mut, &abs); + if (ret == ETIMEDOUT) { + return 1; + } + if (ret) { + return -1; + } + return 0; } #elif defined(NT_THREADS) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 299ab0e629600ba..25f58d9446d8d60 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -149,20 +149,6 @@ _PyThread_cond_after(long long us, struct timespec *abs) *abs = ts; } -int -_PyThread_cond_timedwait(PyCOND_T *cond, PyMUTEX_T *mut, struct timespec *abs) -{ - int r = pthread_cond_timedwait(cond, mut, abs); - if (r == ETIMEDOUT) { - return 1; - } - if (r) { - return -1; - } - return 0; -} - - /* A pthread mutex isn't sufficient to model the Python lock type * because, according to Draft 5 of the docs (P1003.4a/D5), both of the @@ -595,12 +581,15 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, * protocol */ while (success == PY_LOCK_FAILURE) { if (microseconds > 0) { - status = _PyThread_cond_timedwait( + status = pthread_cond_timedwait( &thelock->lock_released, &thelock->mut, &abs); + if (status == 1) { + break; + } if (status == ETIMEDOUT) break; - CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_timedwait"); } else { status = pthread_cond_wait(