Python源码示例:joblib.effective_n_jobs()
示例1
def _partition_bmus(self, X):
"""Private function used to partition bmus between jobs.
Parameters
----------
X : np.array
List of datapoints
Returns
-------
n_jobs : int
Number of jobs
list of int
List of number of datapoints per job
list of int
List of start values for every job list
"""
n_datapoints = len(X)
n_jobs = min(effective_n_jobs(self.n_jobs), n_datapoints)
n_datapoints_per_job = np.full(
n_jobs, n_datapoints // n_jobs, dtype=np.int)
n_datapoints_per_job[:n_datapoints % n_jobs] += 1
starts = np.cumsum(n_datapoints_per_job)
return n_jobs, n_datapoints_per_job.tolist(), [0] + starts.tolist()
示例2
def effective_n_jobs_with_context(n_jobs=None):
"""Find the effective number of jobs, either specified directly, or from the joblib.parallel_backend context."""
if n_jobs is None:
_, n_jobs_from_context = joblib.parallel.get_active_backend()
n_jobs = n_jobs_from_context
return joblib.effective_n_jobs(n_jobs)
示例3
def _partition_estimators(n_estimators, n_jobs):
"""Private function used to partition estimators between jobs."""
# Compute the number of jobs
n_jobs = min(effective_n_jobs(n_jobs), n_estimators)
# Partition estimators between jobs
n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs,
dtype=np.int)
n_estimators_per_job[:n_estimators % n_jobs] += 1
starts = np.cumsum(n_estimators_per_job)
xdiff = [starts[n] - starts[n - 1] for n in range(1, len(starts))]
print("Split among workers default:", starts, xdiff)
return n_estimators_per_job.tolist(), [0] + starts.tolist(), n_jobs
示例4
def _partition_estimators(n_estimators, n_jobs):
"""Private function used to partition estimators between jobs."""
# Compute the number of jobs
n_jobs = min(effective_n_jobs(n_jobs), n_estimators)
# Partition estimators between jobs
n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs,
dtype=np.int)
n_estimators_per_job[:n_estimators % n_jobs] += 1
starts = np.cumsum(n_estimators_per_job)
xdiff = [starts[n] - starts[n - 1] for n in range(1, len(starts))]
print("Split among workers default:", starts, xdiff)
return n_estimators_per_job.tolist(), [0] + starts.tolist(), n_jobs
示例5
def maybe_cythonize_extensions(top_path, config):
"""Tweaks for building extensions between release and development mode."""
with_openmp = check_openmp_support()
is_release = os.path.exists(os.path.join(top_path, 'PKG-INFO'))
if is_release:
build_from_c_and_cpp_files(config.ext_modules)
else:
message = ('Please install Cython with a version >= {0} in order '
'to build a sktime development version.').format(
CYTHON_MIN_VERSION)
try:
import Cython
if LooseVersion(Cython.__version__) < CYTHON_MIN_VERSION:
message += ' Your version of Cython was {0}.'.format(
Cython.__version__)
raise ValueError(message)
from Cython.Build import cythonize
except ImportError as exc:
exc.args += (message,)
raise
n_jobs = 1
with contextlib.suppress(ImportError):
import joblib
if LooseVersion(joblib.__version__) > LooseVersion("0.13.0"):
# earlier joblib versions don't account for CPU affinity
# constraints, and may over-estimate the number of available
# CPU particularly in CI (cf loky#114)
n_jobs = joblib.effective_n_jobs()
config.ext_modules = cythonize(
config.ext_modules,
nthreads=n_jobs,
compile_time_env={'SKTIME_OPENMP_SUPPORTED': with_openmp},
compiler_directives={'language_level': 3}
)