From ded1352a80d4c60b6ec65e3868406dab6870c658 Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Sat, 14 Jul 2018 19:49:55 -0700 Subject: [PATCH 1/2] bpo-34060: Report system load when running test suite for Windows --- Lib/test/libregrtest/main.py | 16 +++- .../2018-07-14-19-49-31.bpo-34060.v-z87j.rst | 2 + Modules/_winapi.c | 89 +++++++++++++++++++ Modules/clinic/_winapi.c.h | 41 ++++++++- PCbuild/pythoncore.vcxproj | 4 +- 5 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2018-07-14-19-49-31.bpo-34060.v-z87j.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e262a7a172b931..0dcc8722249c87 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -100,6 +100,18 @@ def __init__(self): self.next_single_test = None self.next_single_filename = None + # use the platform specific APIs available + # for system load averages + self.getloadavg = None + try: + import _winapi + _winapi.InitializeLoadCounter() + self.getloadavg = _winapi.GetLoadAvg + except ImportError: + pass + if hasattr(os, 'getloadavg'): + self.getloadavg = lambda: os.getloadavg()[0] + def accumulate_result(self, test, result): ok, test_time = result if ok not in (CHILD_ERROR, INTERRUPTED): @@ -130,8 +142,8 @@ def display_progress(self, test_index, test): line = f"[{line}] {test}" # add the system load prefix: "load avg: 1.80 " - if hasattr(os, 'getloadavg'): - load_avg_1min = os.getloadavg()[0] + if self.getloadavg: + load_avg_1min = self.getloadavg() line = f"load avg: {load_avg_1min:.2f} {line}" # add the timestamp prefix: "0:01:05 " diff --git a/Misc/NEWS.d/next/Windows/2018-07-14-19-49-31.bpo-34060.v-z87j.rst b/Misc/NEWS.d/next/Windows/2018-07-14-19-49-31.bpo-34060.v-z87j.rst new file mode 100644 index 00000000000000..b77d805b7f2aff --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-07-14-19-49-31.bpo-34060.v-z87j.rst @@ -0,0 +1,2 @@ +Report system load when running test suite on Windows. Patch by Ammar Askar. +Based on prior work by Jeremy Kloth. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 75d1f0678effe6..742af92e3058df 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -39,6 +39,7 @@ #define WINDOWS_LEAN_AND_MEAN #include "windows.h" +#include #include #include "winreparse.h" @@ -1699,6 +1700,92 @@ _winapi_GetFileType_impl(PyObject *module, HANDLE handle) return result; } +// We use an exponentially weighted moving average, just like Unix systems do +// https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation +// This constant serves as the damping factor. +#define LOADAVG_FACTOR_1F 0.9200444146293232478931553241 +// The time interval in seconds between taking load counts +#define SAMPLING_INTERVAL 1 + +double load_1m = 0; + +VOID CALLBACK LoadCallback(PVOID hCounter) +{ + PDH_FMT_COUNTERVALUE displayValue; + PdhGetFormattedCounterValue((PDH_HCOUNTER) hCounter, PDH_FMT_DOUBLE, 0, &displayValue); + + double currentLoad = displayValue.doubleValue; + double newLoad = load_1m * LOADAVG_FACTOR_1F + currentLoad * (1.0 - LOADAVG_FACTOR_1F); + load_1m = newLoad; +} + +/*[clinic input] +_winapi.GetLoadAvg + +Gets the 1 minute load average (processor queue length) for the system. + +InitializeLoadCounter must be called before this function to engage the +mechanism that records load values. + +[clinic start generated code]*/ + +static PyObject * +_winapi_GetLoadAvg_impl(PyObject *module) +/*[clinic end generated code: output=e79e25f9f0783a3e input=e874ce4fc553db23]*/ +{ + PyObject* load = PyFloat_FromDouble(load_1m); + return load; +} + +/*[clinic input] +_winapi.InitializeLoadCounter + +Initializes instrumentation code to keep track of system load. + +[clinic start generated code]*/ + +static PyObject * +_winapi_InitializeLoadCounter_impl(PyObject *module) +/*[clinic end generated code: output=943e4187b2a20b39 input=3caa0958b4abcea0]*/ +{ + PDH_STATUS s; + HQUERY hQuery; + HCOUNTER hCounter; + + if ((s = PdhOpenQueryW(NULL, 0, &hQuery)) != ERROR_SUCCESS) + { + goto WMIerror; + } + + WCHAR *szCounterPath = L"\\System\\Processor Queue Length"; + if ((s = PdhAddEnglishCounterW(hQuery, szCounterPath, 0, &hCounter)) != ERROR_SUCCESS) + { + goto WMIerror; + } + + HANDLE event = CreateEventW(NULL, FALSE, FALSE, L"LoadUpdateEvent"); + if (event == NULL) { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + if ((s = PdhCollectQueryDataEx(hQuery, SAMPLING_INTERVAL, event)) != ERROR_SUCCESS) + { + goto WMIerror; + } + + HANDLE waitHandle; + if (RegisterWaitForSingleObject(&waitHandle, event, (WAITORTIMERCALLBACK) LoadCallback, + (PVOID) hCounter, INFINITE, WT_EXECUTEDEFAULT) == 0) { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + Py_RETURN_NONE; +WMIerror: + PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); + return NULL; +} static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF @@ -1727,6 +1814,8 @@ static PyMethodDef winapi_functions[] = { _WINAPI_WRITEFILE_METHODDEF _WINAPI_GETACP_METHODDEF _WINAPI_GETFILETYPE_METHODDEF + _WINAPI_GETLOADAVG_METHODDEF + _WINAPI_INITIALIZELOADCOUNTER_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index b14f087732ef43..b191b879cfc25a 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -941,4 +941,43 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=ec7f36eb7efc9d00 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_winapi_GetLoadAvg__doc__, +"GetLoadAvg($module, /)\n" +"--\n" +"\n" +"Gets the 1 minute load average (processor queue length) for the system.\n" +"\n" +"InitializeLoadCounter must be called before this function to engage the\n" +"mechanism that records load values."); + +#define _WINAPI_GETLOADAVG_METHODDEF \ + {"GetLoadAvg", (PyCFunction)_winapi_GetLoadAvg, METH_NOARGS, _winapi_GetLoadAvg__doc__}, + +static PyObject * +_winapi_GetLoadAvg_impl(PyObject *module); + +static PyObject * +_winapi_GetLoadAvg(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _winapi_GetLoadAvg_impl(module); +} + +PyDoc_STRVAR(_winapi_InitializeLoadCounter__doc__, +"InitializeLoadCounter($module, /)\n" +"--\n" +"\n" +"Initializes instrumentation code to keep track of system load."); + +#define _WINAPI_INITIALIZELOADCOUNTER_METHODDEF \ + {"InitializeLoadCounter", (PyCFunction)_winapi_InitializeLoadCounter, METH_NOARGS, _winapi_InitializeLoadCounter__doc__}, + +static PyObject * +_winapi_InitializeLoadCounter_impl(PyObject *module); + +static PyObject * +_winapi_InitializeLoadCounter(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _winapi_InitializeLoadCounter_impl(module); +} +/*[clinic end generated code: output=922a40402540f154 input=a9049054013a1b77]*/ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 90330faa0cf26f..d3a8edbcf5e967 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -74,7 +74,7 @@ _Py_HAVE_ZLIB;%(PreprocessorDefinitions) - version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;shlwapi.lib;ws2_32.lib;Pdh.lib;%(AdditionalDependencies) @@ -461,4 +461,4 @@ - + \ No newline at end of file From 92d35bb8120285775aaeb124d30782e6c66bdaaa Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Sat, 14 Jul 2018 20:48:44 -0700 Subject: [PATCH 2/2] Change timing interval to 5 --- Modules/_winapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 742af92e3058df..4a7d3abf02479d 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1705,7 +1705,7 @@ _winapi_GetFileType_impl(PyObject *module, HANDLE handle) // This constant serves as the damping factor. #define LOADAVG_FACTOR_1F 0.9200444146293232478931553241 // The time interval in seconds between taking load counts -#define SAMPLING_INTERVAL 1 +#define SAMPLING_INTERVAL 5 double load_1m = 0;