histograms.py 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129
  1. """
  2. Histogram-related functions
  3. """
  4. import contextlib
  5. import functools
  6. import operator
  7. import warnings
  8. import numpy as np
  9. from numpy.core import overrides
  10. __all__ = ['histogram', 'histogramdd', 'histogram_bin_edges']
  11. array_function_dispatch = functools.partial(
  12. overrides.array_function_dispatch, module='numpy')
  13. # range is a keyword argument to many functions, so save the builtin so they can
  14. # use it.
  15. _range = range
  16. def _ptp(x):
  17. """Peak-to-peak value of x.
  18. This implementation avoids the problem of signed integer arrays having a
  19. peak-to-peak value that cannot be represented with the array's data type.
  20. This function returns an unsigned value for signed integer arrays.
  21. """
  22. return _unsigned_subtract(x.max(), x.min())
  23. def _hist_bin_sqrt(x, range):
  24. """
  25. Square root histogram bin estimator.
  26. Bin width is inversely proportional to the data size. Used by many
  27. programs for its simplicity.
  28. Parameters
  29. ----------
  30. x : array_like
  31. Input data that is to be histogrammed, trimmed to range. May not
  32. be empty.
  33. Returns
  34. -------
  35. h : An estimate of the optimal bin width for the given data.
  36. """
  37. del range # unused
  38. return _ptp(x) / np.sqrt(x.size)
  39. def _hist_bin_sturges(x, range):
  40. """
  41. Sturges histogram bin estimator.
  42. A very simplistic estimator based on the assumption of normality of
  43. the data. This estimator has poor performance for non-normal data,
  44. which becomes especially obvious for large data sets. The estimate
  45. depends only on size of the data.
  46. Parameters
  47. ----------
  48. x : array_like
  49. Input data that is to be histogrammed, trimmed to range. May not
  50. be empty.
  51. Returns
  52. -------
  53. h : An estimate of the optimal bin width for the given data.
  54. """
  55. del range # unused
  56. return _ptp(x) / (np.log2(x.size) + 1.0)
  57. def _hist_bin_rice(x, range):
  58. """
  59. Rice histogram bin estimator.
  60. Another simple estimator with no normality assumption. It has better
  61. performance for large data than Sturges, but tends to overestimate
  62. the number of bins. The number of bins is proportional to the cube
  63. root of data size (asymptotically optimal). The estimate depends
  64. only on size of the data.
  65. Parameters
  66. ----------
  67. x : array_like
  68. Input data that is to be histogrammed, trimmed to range. May not
  69. be empty.
  70. Returns
  71. -------
  72. h : An estimate of the optimal bin width for the given data.
  73. """
  74. del range # unused
  75. return _ptp(x) / (2.0 * x.size ** (1.0 / 3))
  76. def _hist_bin_scott(x, range):
  77. """
  78. Scott histogram bin estimator.
  79. The binwidth is proportional to the standard deviation of the data
  80. and inversely proportional to the cube root of data size
  81. (asymptotically optimal).
  82. Parameters
  83. ----------
  84. x : array_like
  85. Input data that is to be histogrammed, trimmed to range. May not
  86. be empty.
  87. Returns
  88. -------
  89. h : An estimate of the optimal bin width for the given data.
  90. """
  91. del range # unused
  92. return (24.0 * np.pi**0.5 / x.size)**(1.0 / 3.0) * np.std(x)
  93. def _hist_bin_stone(x, range):
  94. """
  95. Histogram bin estimator based on minimizing the estimated integrated squared error (ISE).
  96. The number of bins is chosen by minimizing the estimated ISE against the unknown true distribution.
  97. The ISE is estimated using cross-validation and can be regarded as a generalization of Scott's rule.
  98. https://en.wikipedia.org/wiki/Histogram#Scott.27s_normal_reference_rule
  99. This paper by Stone appears to be the origination of this rule.
  100. http://digitalassets.lib.berkeley.edu/sdtr/ucb/text/34.pdf
  101. Parameters
  102. ----------
  103. x : array_like
  104. Input data that is to be histogrammed, trimmed to range. May not
  105. be empty.
  106. range : (float, float)
  107. The lower and upper range of the bins.
  108. Returns
  109. -------
  110. h : An estimate of the optimal bin width for the given data.
  111. """
  112. n = x.size
  113. ptp_x = _ptp(x)
  114. if n <= 1 or ptp_x == 0:
  115. return 0
  116. def jhat(nbins):
  117. hh = ptp_x / nbins
  118. p_k = np.histogram(x, bins=nbins, range=range)[0] / n
  119. return (2 - (n + 1) * p_k.dot(p_k)) / hh
  120. nbins_upper_bound = max(100, int(np.sqrt(n)))
  121. nbins = min(_range(1, nbins_upper_bound + 1), key=jhat)
  122. if nbins == nbins_upper_bound:
  123. warnings.warn("The number of bins estimated may be suboptimal.",
  124. RuntimeWarning, stacklevel=3)
  125. return ptp_x / nbins
  126. def _hist_bin_doane(x, range):
  127. """
  128. Doane's histogram bin estimator.
  129. Improved version of Sturges' formula which works better for
  130. non-normal data. See
  131. stats.stackexchange.com/questions/55134/doanes-formula-for-histogram-binning
  132. Parameters
  133. ----------
  134. x : array_like
  135. Input data that is to be histogrammed, trimmed to range. May not
  136. be empty.
  137. Returns
  138. -------
  139. h : An estimate of the optimal bin width for the given data.
  140. """
  141. del range # unused
  142. if x.size > 2:
  143. sg1 = np.sqrt(6.0 * (x.size - 2) / ((x.size + 1.0) * (x.size + 3)))
  144. sigma = np.std(x)
  145. if sigma > 0.0:
  146. # These three operations add up to
  147. # g1 = np.mean(((x - np.mean(x)) / sigma)**3)
  148. # but use only one temp array instead of three
  149. temp = x - np.mean(x)
  150. np.true_divide(temp, sigma, temp)
  151. np.power(temp, 3, temp)
  152. g1 = np.mean(temp)
  153. return _ptp(x) / (1.0 + np.log2(x.size) +
  154. np.log2(1.0 + np.absolute(g1) / sg1))
  155. return 0.0
  156. def _hist_bin_fd(x, range):
  157. """
  158. The Freedman-Diaconis histogram bin estimator.
  159. The Freedman-Diaconis rule uses interquartile range (IQR) to
  160. estimate binwidth. It is considered a variation of the Scott rule
  161. with more robustness as the IQR is less affected by outliers than
  162. the standard deviation. However, the IQR depends on fewer points
  163. than the standard deviation, so it is less accurate, especially for
  164. long tailed distributions.
  165. If the IQR is 0, this function returns 0 for the bin width.
  166. Binwidth is inversely proportional to the cube root of data size
  167. (asymptotically optimal).
  168. Parameters
  169. ----------
  170. x : array_like
  171. Input data that is to be histogrammed, trimmed to range. May not
  172. be empty.
  173. Returns
  174. -------
  175. h : An estimate of the optimal bin width for the given data.
  176. """
  177. del range # unused
  178. iqr = np.subtract(*np.percentile(x, [75, 25]))
  179. return 2.0 * iqr * x.size ** (-1.0 / 3.0)
  180. def _hist_bin_auto(x, range):
  181. """
  182. Histogram bin estimator that uses the minimum width of the
  183. Freedman-Diaconis and Sturges estimators if the FD bin width is non-zero.
  184. If the bin width from the FD estimator is 0, the Sturges estimator is used.
  185. The FD estimator is usually the most robust method, but its width
  186. estimate tends to be too large for small `x` and bad for data with limited
  187. variance. The Sturges estimator is quite good for small (<1000) datasets
  188. and is the default in the R language. This method gives good off-the-shelf
  189. behaviour.
  190. .. versionchanged:: 1.15.0
  191. If there is limited variance the IQR can be 0, which results in the
  192. FD bin width being 0 too. This is not a valid bin width, so
  193. ``np.histogram_bin_edges`` chooses 1 bin instead, which may not be optimal.
  194. If the IQR is 0, it's unlikely any variance-based estimators will be of
  195. use, so we revert to the Sturges estimator, which only uses the size of the
  196. dataset in its calculation.
  197. Parameters
  198. ----------
  199. x : array_like
  200. Input data that is to be histogrammed, trimmed to range. May not
  201. be empty.
  202. Returns
  203. -------
  204. h : An estimate of the optimal bin width for the given data.
  205. See Also
  206. --------
  207. _hist_bin_fd, _hist_bin_sturges
  208. """
  209. fd_bw = _hist_bin_fd(x, range)
  210. sturges_bw = _hist_bin_sturges(x, range)
  211. del range # unused
  212. if fd_bw:
  213. return min(fd_bw, sturges_bw)
  214. else:
  215. # limited variance, so we return a len dependent bw estimator
  216. return sturges_bw
  217. # Private dict initialized at module load time
  218. _hist_bin_selectors = {'stone': _hist_bin_stone,
  219. 'auto': _hist_bin_auto,
  220. 'doane': _hist_bin_doane,
  221. 'fd': _hist_bin_fd,
  222. 'rice': _hist_bin_rice,
  223. 'scott': _hist_bin_scott,
  224. 'sqrt': _hist_bin_sqrt,
  225. 'sturges': _hist_bin_sturges}
  226. def _ravel_and_check_weights(a, weights):
  227. """ Check a and weights have matching shapes, and ravel both """
  228. a = np.asarray(a)
  229. # Ensure that the array is a "subtractable" dtype
  230. if a.dtype == np.bool_:
  231. warnings.warn("Converting input from {} to {} for compatibility."
  232. .format(a.dtype, np.uint8),
  233. RuntimeWarning, stacklevel=3)
  234. a = a.astype(np.uint8)
  235. if weights is not None:
  236. weights = np.asarray(weights)
  237. if weights.shape != a.shape:
  238. raise ValueError(
  239. 'weights should have the same shape as a.')
  240. weights = weights.ravel()
  241. a = a.ravel()
  242. return a, weights
  243. def _get_outer_edges(a, range):
  244. """
  245. Determine the outer bin edges to use, from either the data or the range
  246. argument
  247. """
  248. if range is not None:
  249. first_edge, last_edge = range
  250. if first_edge > last_edge:
  251. raise ValueError(
  252. 'max must be larger than min in range parameter.')
  253. if not (np.isfinite(first_edge) and np.isfinite(last_edge)):
  254. raise ValueError(
  255. "supplied range of [{}, {}] is not finite".format(first_edge, last_edge))
  256. elif a.size == 0:
  257. # handle empty arrays. Can't determine range, so use 0-1.
  258. first_edge, last_edge = 0, 1
  259. else:
  260. first_edge, last_edge = a.min(), a.max()
  261. if not (np.isfinite(first_edge) and np.isfinite(last_edge)):
  262. raise ValueError(
  263. "autodetected range of [{}, {}] is not finite".format(first_edge, last_edge))
  264. # expand empty range to avoid divide by zero
  265. if first_edge == last_edge:
  266. first_edge = first_edge - 0.5
  267. last_edge = last_edge + 0.5
  268. return first_edge, last_edge
  269. def _unsigned_subtract(a, b):
  270. """
  271. Subtract two values where a >= b, and produce an unsigned result
  272. This is needed when finding the difference between the upper and lower
  273. bound of an int16 histogram
  274. """
  275. # coerce to a single type
  276. signed_to_unsigned = {
  277. np.byte: np.ubyte,
  278. np.short: np.ushort,
  279. np.intc: np.uintc,
  280. np.int_: np.uint,
  281. np.longlong: np.ulonglong
  282. }
  283. dt = np.result_type(a, b)
  284. try:
  285. dt = signed_to_unsigned[dt.type]
  286. except KeyError:
  287. return np.subtract(a, b, dtype=dt)
  288. else:
  289. # we know the inputs are integers, and we are deliberately casting
  290. # signed to unsigned
  291. return np.subtract(a, b, casting='unsafe', dtype=dt)
  292. def _get_bin_edges(a, bins, range, weights):
  293. """
  294. Computes the bins used internally by `histogram`.
  295. Parameters
  296. ==========
  297. a : ndarray
  298. Ravelled data array
  299. bins, range
  300. Forwarded arguments from `histogram`.
  301. weights : ndarray, optional
  302. Ravelled weights array, or None
  303. Returns
  304. =======
  305. bin_edges : ndarray
  306. Array of bin edges
  307. uniform_bins : (Number, Number, int):
  308. The upper bound, lowerbound, and number of bins, used in the optimized
  309. implementation of `histogram` that works on uniform bins.
  310. """
  311. # parse the overloaded bins argument
  312. n_equal_bins = None
  313. bin_edges = None
  314. if isinstance(bins, str):
  315. bin_name = bins
  316. # if `bins` is a string for an automatic method,
  317. # this will replace it with the number of bins calculated
  318. if bin_name not in _hist_bin_selectors:
  319. raise ValueError(
  320. "{!r} is not a valid estimator for `bins`".format(bin_name))
  321. if weights is not None:
  322. raise TypeError("Automated estimation of the number of "
  323. "bins is not supported for weighted data")
  324. first_edge, last_edge = _get_outer_edges(a, range)
  325. # truncate the range if needed
  326. if range is not None:
  327. keep = (a >= first_edge)
  328. keep &= (a <= last_edge)
  329. if not np.logical_and.reduce(keep):
  330. a = a[keep]
  331. if a.size == 0:
  332. n_equal_bins = 1
  333. else:
  334. # Do not call selectors on empty arrays
  335. width = _hist_bin_selectors[bin_name](a, (first_edge, last_edge))
  336. if width:
  337. n_equal_bins = int(np.ceil(_unsigned_subtract(last_edge, first_edge) / width))
  338. else:
  339. # Width can be zero for some estimators, e.g. FD when
  340. # the IQR of the data is zero.
  341. n_equal_bins = 1
  342. elif np.ndim(bins) == 0:
  343. try:
  344. n_equal_bins = operator.index(bins)
  345. except TypeError as e:
  346. raise TypeError(
  347. '`bins` must be an integer, a string, or an array') from e
  348. if n_equal_bins < 1:
  349. raise ValueError('`bins` must be positive, when an integer')
  350. first_edge, last_edge = _get_outer_edges(a, range)
  351. elif np.ndim(bins) == 1:
  352. bin_edges = np.asarray(bins)
  353. if np.any(bin_edges[:-1] > bin_edges[1:]):
  354. raise ValueError(
  355. '`bins` must increase monotonically, when an array')
  356. else:
  357. raise ValueError('`bins` must be 1d, when an array')
  358. if n_equal_bins is not None:
  359. # gh-10322 means that type resolution rules are dependent on array
  360. # shapes. To avoid this causing problems, we pick a type now and stick
  361. # with it throughout.
  362. bin_type = np.result_type(first_edge, last_edge, a)
  363. if np.issubdtype(bin_type, np.integer):
  364. bin_type = np.result_type(bin_type, float)
  365. # bin edges must be computed
  366. bin_edges = np.linspace(
  367. first_edge, last_edge, n_equal_bins + 1,
  368. endpoint=True, dtype=bin_type)
  369. return bin_edges, (first_edge, last_edge, n_equal_bins)
  370. else:
  371. return bin_edges, None
  372. def _search_sorted_inclusive(a, v):
  373. """
  374. Like `searchsorted`, but where the last item in `v` is placed on the right.
  375. In the context of a histogram, this makes the last bin edge inclusive
  376. """
  377. return np.concatenate((
  378. a.searchsorted(v[:-1], 'left'),
  379. a.searchsorted(v[-1:], 'right')
  380. ))
  381. def _histogram_bin_edges_dispatcher(a, bins=None, range=None, weights=None):
  382. return (a, bins, weights)
  383. @array_function_dispatch(_histogram_bin_edges_dispatcher)
  384. def histogram_bin_edges(a, bins=10, range=None, weights=None):
  385. r"""
  386. Function to calculate only the edges of the bins used by the `histogram`
  387. function.
  388. Parameters
  389. ----------
  390. a : array_like
  391. Input data. The histogram is computed over the flattened array.
  392. bins : int or sequence of scalars or str, optional
  393. If `bins` is an int, it defines the number of equal-width
  394. bins in the given range (10, by default). If `bins` is a
  395. sequence, it defines the bin edges, including the rightmost
  396. edge, allowing for non-uniform bin widths.
  397. If `bins` is a string from the list below, `histogram_bin_edges` will use
  398. the method chosen to calculate the optimal bin width and
  399. consequently the number of bins (see `Notes` for more detail on
  400. the estimators) from the data that falls within the requested
  401. range. While the bin width will be optimal for the actual data
  402. in the range, the number of bins will be computed to fill the
  403. entire range, including the empty portions. For visualisation,
  404. using the 'auto' option is suggested. Weighted data is not
  405. supported for automated bin size selection.
  406. 'auto'
  407. Maximum of the 'sturges' and 'fd' estimators. Provides good
  408. all around performance.
  409. 'fd' (Freedman Diaconis Estimator)
  410. Robust (resilient to outliers) estimator that takes into
  411. account data variability and data size.
  412. 'doane'
  413. An improved version of Sturges' estimator that works better
  414. with non-normal datasets.
  415. 'scott'
  416. Less robust estimator that that takes into account data
  417. variability and data size.
  418. 'stone'
  419. Estimator based on leave-one-out cross-validation estimate of
  420. the integrated squared error. Can be regarded as a generalization
  421. of Scott's rule.
  422. 'rice'
  423. Estimator does not take variability into account, only data
  424. size. Commonly overestimates number of bins required.
  425. 'sturges'
  426. R's default method, only accounts for data size. Only
  427. optimal for gaussian data and underestimates number of bins
  428. for large non-gaussian datasets.
  429. 'sqrt'
  430. Square root (of data size) estimator, used by Excel and
  431. other programs for its speed and simplicity.
  432. range : (float, float), optional
  433. The lower and upper range of the bins. If not provided, range
  434. is simply ``(a.min(), a.max())``. Values outside the range are
  435. ignored. The first element of the range must be less than or
  436. equal to the second. `range` affects the automatic bin
  437. computation as well. While bin width is computed to be optimal
  438. based on the actual data within `range`, the bin count will fill
  439. the entire range including portions containing no data.
  440. weights : array_like, optional
  441. An array of weights, of the same shape as `a`. Each value in
  442. `a` only contributes its associated weight towards the bin count
  443. (instead of 1). This is currently not used by any of the bin estimators,
  444. but may be in the future.
  445. Returns
  446. -------
  447. bin_edges : array of dtype float
  448. The edges to pass into `histogram`
  449. See Also
  450. --------
  451. histogram
  452. Notes
  453. -----
  454. The methods to estimate the optimal number of bins are well founded
  455. in literature, and are inspired by the choices R provides for
  456. histogram visualisation. Note that having the number of bins
  457. proportional to :math:`n^{1/3}` is asymptotically optimal, which is
  458. why it appears in most estimators. These are simply plug-in methods
  459. that give good starting points for number of bins. In the equations
  460. below, :math:`h` is the binwidth and :math:`n_h` is the number of
  461. bins. All estimators that compute bin counts are recast to bin width
  462. using the `ptp` of the data. The final bin count is obtained from
  463. ``np.round(np.ceil(range / h))``. The final bin width is often less
  464. than what is returned by the estimators below.
  465. 'auto' (maximum of the 'sturges' and 'fd' estimators)
  466. A compromise to get a good value. For small datasets the Sturges
  467. value will usually be chosen, while larger datasets will usually
  468. default to FD. Avoids the overly conservative behaviour of FD
  469. and Sturges for small and large datasets respectively.
  470. Switchover point is usually :math:`a.size \approx 1000`.
  471. 'fd' (Freedman Diaconis Estimator)
  472. .. math:: h = 2 \frac{IQR}{n^{1/3}}
  473. The binwidth is proportional to the interquartile range (IQR)
  474. and inversely proportional to cube root of a.size. Can be too
  475. conservative for small datasets, but is quite good for large
  476. datasets. The IQR is very robust to outliers.
  477. 'scott'
  478. .. math:: h = \sigma \sqrt[3]{\frac{24 * \sqrt{\pi}}{n}}
  479. The binwidth is proportional to the standard deviation of the
  480. data and inversely proportional to cube root of ``x.size``. Can
  481. be too conservative for small datasets, but is quite good for
  482. large datasets. The standard deviation is not very robust to
  483. outliers. Values are very similar to the Freedman-Diaconis
  484. estimator in the absence of outliers.
  485. 'rice'
  486. .. math:: n_h = 2n^{1/3}
  487. The number of bins is only proportional to cube root of
  488. ``a.size``. It tends to overestimate the number of bins and it
  489. does not take into account data variability.
  490. 'sturges'
  491. .. math:: n_h = \log _{2}n+1
  492. The number of bins is the base 2 log of ``a.size``. This
  493. estimator assumes normality of data and is too conservative for
  494. larger, non-normal datasets. This is the default method in R's
  495. ``hist`` method.
  496. 'doane'
  497. .. math:: n_h = 1 + \log_{2}(n) +
  498. \log_{2}(1 + \frac{|g_1|}{\sigma_{g_1}})
  499. g_1 = mean[(\frac{x - \mu}{\sigma})^3]
  500. \sigma_{g_1} = \sqrt{\frac{6(n - 2)}{(n + 1)(n + 3)}}
  501. An improved version of Sturges' formula that produces better
  502. estimates for non-normal datasets. This estimator attempts to
  503. account for the skew of the data.
  504. 'sqrt'
  505. .. math:: n_h = \sqrt n
  506. The simplest and fastest estimator. Only takes into account the
  507. data size.
  508. Examples
  509. --------
  510. >>> arr = np.array([0, 0, 0, 1, 2, 3, 3, 4, 5])
  511. >>> np.histogram_bin_edges(arr, bins='auto', range=(0, 1))
  512. array([0. , 0.25, 0.5 , 0.75, 1. ])
  513. >>> np.histogram_bin_edges(arr, bins=2)
  514. array([0. , 2.5, 5. ])
  515. For consistency with histogram, an array of pre-computed bins is
  516. passed through unmodified:
  517. >>> np.histogram_bin_edges(arr, [1, 2])
  518. array([1, 2])
  519. This function allows one set of bins to be computed, and reused across
  520. multiple histograms:
  521. >>> shared_bins = np.histogram_bin_edges(arr, bins='auto')
  522. >>> shared_bins
  523. array([0., 1., 2., 3., 4., 5.])
  524. >>> group_id = np.array([0, 1, 1, 0, 1, 1, 0, 1, 1])
  525. >>> hist_0, _ = np.histogram(arr[group_id == 0], bins=shared_bins)
  526. >>> hist_1, _ = np.histogram(arr[group_id == 1], bins=shared_bins)
  527. >>> hist_0; hist_1
  528. array([1, 1, 0, 1, 0])
  529. array([2, 0, 1, 1, 2])
  530. Which gives more easily comparable results than using separate bins for
  531. each histogram:
  532. >>> hist_0, bins_0 = np.histogram(arr[group_id == 0], bins='auto')
  533. >>> hist_1, bins_1 = np.histogram(arr[group_id == 1], bins='auto')
  534. >>> hist_0; hist_1
  535. array([1, 1, 1])
  536. array([2, 1, 1, 2])
  537. >>> bins_0; bins_1
  538. array([0., 1., 2., 3.])
  539. array([0. , 1.25, 2.5 , 3.75, 5. ])
  540. """
  541. a, weights = _ravel_and_check_weights(a, weights)
  542. bin_edges, _ = _get_bin_edges(a, bins, range, weights)
  543. return bin_edges
  544. def _histogram_dispatcher(
  545. a, bins=None, range=None, normed=None, weights=None, density=None):
  546. return (a, bins, weights)
  547. @array_function_dispatch(_histogram_dispatcher)
  548. def histogram(a, bins=10, range=None, normed=None, weights=None,
  549. density=None):
  550. r"""
  551. Compute the histogram of a dataset.
  552. Parameters
  553. ----------
  554. a : array_like
  555. Input data. The histogram is computed over the flattened array.
  556. bins : int or sequence of scalars or str, optional
  557. If `bins` is an int, it defines the number of equal-width
  558. bins in the given range (10, by default). If `bins` is a
  559. sequence, it defines a monotonically increasing array of bin edges,
  560. including the rightmost edge, allowing for non-uniform bin widths.
  561. .. versionadded:: 1.11.0
  562. If `bins` is a string, it defines the method used to calculate the
  563. optimal bin width, as defined by `histogram_bin_edges`.
  564. range : (float, float), optional
  565. The lower and upper range of the bins. If not provided, range
  566. is simply ``(a.min(), a.max())``. Values outside the range are
  567. ignored. The first element of the range must be less than or
  568. equal to the second. `range` affects the automatic bin
  569. computation as well. While bin width is computed to be optimal
  570. based on the actual data within `range`, the bin count will fill
  571. the entire range including portions containing no data.
  572. normed : bool, optional
  573. .. deprecated:: 1.6.0
  574. This is equivalent to the `density` argument, but produces incorrect
  575. results for unequal bin widths. It should not be used.
  576. .. versionchanged:: 1.15.0
  577. DeprecationWarnings are actually emitted.
  578. weights : array_like, optional
  579. An array of weights, of the same shape as `a`. Each value in
  580. `a` only contributes its associated weight towards the bin count
  581. (instead of 1). If `density` is True, the weights are
  582. normalized, so that the integral of the density over the range
  583. remains 1.
  584. density : bool, optional
  585. If ``False``, the result will contain the number of samples in
  586. each bin. If ``True``, the result is the value of the
  587. probability *density* function at the bin, normalized such that
  588. the *integral* over the range is 1. Note that the sum of the
  589. histogram values will not be equal to 1 unless bins of unity
  590. width are chosen; it is not a probability *mass* function.
  591. Overrides the ``normed`` keyword if given.
  592. Returns
  593. -------
  594. hist : array
  595. The values of the histogram. See `density` and `weights` for a
  596. description of the possible semantics.
  597. bin_edges : array of dtype float
  598. Return the bin edges ``(length(hist)+1)``.
  599. See Also
  600. --------
  601. histogramdd, bincount, searchsorted, digitize, histogram_bin_edges
  602. Notes
  603. -----
  604. All but the last (righthand-most) bin is half-open. In other words,
  605. if `bins` is::
  606. [1, 2, 3, 4]
  607. then the first bin is ``[1, 2)`` (including 1, but excluding 2) and
  608. the second ``[2, 3)``. The last bin, however, is ``[3, 4]``, which
  609. *includes* 4.
  610. Examples
  611. --------
  612. >>> np.histogram([1, 2, 1], bins=[0, 1, 2, 3])
  613. (array([0, 2, 1]), array([0, 1, 2, 3]))
  614. >>> np.histogram(np.arange(4), bins=np.arange(5), density=True)
  615. (array([0.25, 0.25, 0.25, 0.25]), array([0, 1, 2, 3, 4]))
  616. >>> np.histogram([[1, 2, 1], [1, 0, 1]], bins=[0,1,2,3])
  617. (array([1, 4, 1]), array([0, 1, 2, 3]))
  618. >>> a = np.arange(5)
  619. >>> hist, bin_edges = np.histogram(a, density=True)
  620. >>> hist
  621. array([0.5, 0. , 0.5, 0. , 0. , 0.5, 0. , 0.5, 0. , 0.5])
  622. >>> hist.sum()
  623. 2.4999999999999996
  624. >>> np.sum(hist * np.diff(bin_edges))
  625. 1.0
  626. .. versionadded:: 1.11.0
  627. Automated Bin Selection Methods example, using 2 peak random data
  628. with 2000 points:
  629. >>> import matplotlib.pyplot as plt
  630. >>> rng = np.random.RandomState(10) # deterministic random data
  631. >>> a = np.hstack((rng.normal(size=1000),
  632. ... rng.normal(loc=5, scale=2, size=1000)))
  633. >>> _ = plt.hist(a, bins='auto') # arguments are passed to np.histogram
  634. >>> plt.title("Histogram with 'auto' bins")
  635. Text(0.5, 1.0, "Histogram with 'auto' bins")
  636. >>> plt.show()
  637. """
  638. a, weights = _ravel_and_check_weights(a, weights)
  639. bin_edges, uniform_bins = _get_bin_edges(a, bins, range, weights)
  640. # Histogram is an integer or a float array depending on the weights.
  641. if weights is None:
  642. ntype = np.dtype(np.intp)
  643. else:
  644. ntype = weights.dtype
  645. # We set a block size, as this allows us to iterate over chunks when
  646. # computing histograms, to minimize memory usage.
  647. BLOCK = 65536
  648. # The fast path uses bincount, but that only works for certain types
  649. # of weight
  650. simple_weights = (
  651. weights is None or
  652. np.can_cast(weights.dtype, np.double) or
  653. np.can_cast(weights.dtype, complex)
  654. )
  655. if uniform_bins is not None and simple_weights:
  656. # Fast algorithm for equal bins
  657. # We now convert values of a to bin indices, under the assumption of
  658. # equal bin widths (which is valid here).
  659. first_edge, last_edge, n_equal_bins = uniform_bins
  660. # Initialize empty histogram
  661. n = np.zeros(n_equal_bins, ntype)
  662. # Pre-compute histogram scaling factor
  663. norm = n_equal_bins / _unsigned_subtract(last_edge, first_edge)
  664. # We iterate over blocks here for two reasons: the first is that for
  665. # large arrays, it is actually faster (for example for a 10^8 array it
  666. # is 2x as fast) and it results in a memory footprint 3x lower in the
  667. # limit of large arrays.
  668. for i in _range(0, len(a), BLOCK):
  669. tmp_a = a[i:i+BLOCK]
  670. if weights is None:
  671. tmp_w = None
  672. else:
  673. tmp_w = weights[i:i + BLOCK]
  674. # Only include values in the right range
  675. keep = (tmp_a >= first_edge)
  676. keep &= (tmp_a <= last_edge)
  677. if not np.logical_and.reduce(keep):
  678. tmp_a = tmp_a[keep]
  679. if tmp_w is not None:
  680. tmp_w = tmp_w[keep]
  681. # This cast ensures no type promotions occur below, which gh-10322
  682. # make unpredictable. Getting it wrong leads to precision errors
  683. # like gh-8123.
  684. tmp_a = tmp_a.astype(bin_edges.dtype, copy=False)
  685. # Compute the bin indices, and for values that lie exactly on
  686. # last_edge we need to subtract one
  687. f_indices = _unsigned_subtract(tmp_a, first_edge) * norm
  688. indices = f_indices.astype(np.intp)
  689. indices[indices == n_equal_bins] -= 1
  690. # The index computation is not guaranteed to give exactly
  691. # consistent results within ~1 ULP of the bin edges.
  692. decrement = tmp_a < bin_edges[indices]
  693. indices[decrement] -= 1
  694. # The last bin includes the right edge. The other bins do not.
  695. increment = ((tmp_a >= bin_edges[indices + 1])
  696. & (indices != n_equal_bins - 1))
  697. indices[increment] += 1
  698. # We now compute the histogram using bincount
  699. if ntype.kind == 'c':
  700. n.real += np.bincount(indices, weights=tmp_w.real,
  701. minlength=n_equal_bins)
  702. n.imag += np.bincount(indices, weights=tmp_w.imag,
  703. minlength=n_equal_bins)
  704. else:
  705. n += np.bincount(indices, weights=tmp_w,
  706. minlength=n_equal_bins).astype(ntype)
  707. else:
  708. # Compute via cumulative histogram
  709. cum_n = np.zeros(bin_edges.shape, ntype)
  710. if weights is None:
  711. for i in _range(0, len(a), BLOCK):
  712. sa = np.sort(a[i:i+BLOCK])
  713. cum_n += _search_sorted_inclusive(sa, bin_edges)
  714. else:
  715. zero = np.zeros(1, dtype=ntype)
  716. for i in _range(0, len(a), BLOCK):
  717. tmp_a = a[i:i+BLOCK]
  718. tmp_w = weights[i:i+BLOCK]
  719. sorting_index = np.argsort(tmp_a)
  720. sa = tmp_a[sorting_index]
  721. sw = tmp_w[sorting_index]
  722. cw = np.concatenate((zero, sw.cumsum()))
  723. bin_index = _search_sorted_inclusive(sa, bin_edges)
  724. cum_n += cw[bin_index]
  725. n = np.diff(cum_n)
  726. # density overrides the normed keyword
  727. if density is not None:
  728. if normed is not None:
  729. # 2018-06-13, numpy 1.15.0 (this was not noisily deprecated in 1.6)
  730. warnings.warn(
  731. "The normed argument is ignored when density is provided. "
  732. "In future passing both will result in an error.",
  733. DeprecationWarning, stacklevel=3)
  734. normed = None
  735. if density:
  736. db = np.array(np.diff(bin_edges), float)
  737. return n/db/n.sum(), bin_edges
  738. elif normed:
  739. # 2018-06-13, numpy 1.15.0 (this was not noisily deprecated in 1.6)
  740. warnings.warn(
  741. "Passing `normed=True` on non-uniform bins has always been "
  742. "broken, and computes neither the probability density "
  743. "function nor the probability mass function. "
  744. "The result is only correct if the bins are uniform, when "
  745. "density=True will produce the same result anyway. "
  746. "The argument will be removed in a future version of "
  747. "numpy.",
  748. np.VisibleDeprecationWarning, stacklevel=3)
  749. # this normalization is incorrect, but
  750. db = np.array(np.diff(bin_edges), float)
  751. return n/(n*db).sum(), bin_edges
  752. else:
  753. if normed is not None:
  754. # 2018-06-13, numpy 1.15.0 (this was not noisily deprecated in 1.6)
  755. warnings.warn(
  756. "Passing normed=False is deprecated, and has no effect. "
  757. "Consider passing the density argument instead.",
  758. DeprecationWarning, stacklevel=3)
  759. return n, bin_edges
  760. def _histogramdd_dispatcher(sample, bins=None, range=None, normed=None,
  761. weights=None, density=None):
  762. if hasattr(sample, 'shape'): # same condition as used in histogramdd
  763. yield sample
  764. else:
  765. yield from sample
  766. with contextlib.suppress(TypeError):
  767. yield from bins
  768. yield weights
  769. @array_function_dispatch(_histogramdd_dispatcher)
  770. def histogramdd(sample, bins=10, range=None, normed=None, weights=None,
  771. density=None):
  772. """
  773. Compute the multidimensional histogram of some data.
  774. Parameters
  775. ----------
  776. sample : (N, D) array, or (D, N) array_like
  777. The data to be histogrammed.
  778. Note the unusual interpretation of sample when an array_like:
  779. * When an array, each row is a coordinate in a D-dimensional space -
  780. such as ``histogramdd(np.array([p1, p2, p3]))``.
  781. * When an array_like, each element is the list of values for single
  782. coordinate - such as ``histogramdd((X, Y, Z))``.
  783. The first form should be preferred.
  784. bins : sequence or int, optional
  785. The bin specification:
  786. * A sequence of arrays describing the monotonically increasing bin
  787. edges along each dimension.
  788. * The number of bins for each dimension (nx, ny, ... =bins)
  789. * The number of bins for all dimensions (nx=ny=...=bins).
  790. range : sequence, optional
  791. A sequence of length D, each an optional (lower, upper) tuple giving
  792. the outer bin edges to be used if the edges are not given explicitly in
  793. `bins`.
  794. An entry of None in the sequence results in the minimum and maximum
  795. values being used for the corresponding dimension.
  796. The default, None, is equivalent to passing a tuple of D None values.
  797. density : bool, optional
  798. If False, the default, returns the number of samples in each bin.
  799. If True, returns the probability *density* function at the bin,
  800. ``bin_count / sample_count / bin_volume``.
  801. normed : bool, optional
  802. An alias for the density argument that behaves identically. To avoid
  803. confusion with the broken normed argument to `histogram`, `density`
  804. should be preferred.
  805. weights : (N,) array_like, optional
  806. An array of values `w_i` weighing each sample `(x_i, y_i, z_i, ...)`.
  807. Weights are normalized to 1 if normed is True. If normed is False,
  808. the values of the returned histogram are equal to the sum of the
  809. weights belonging to the samples falling into each bin.
  810. Returns
  811. -------
  812. H : ndarray
  813. The multidimensional histogram of sample x. See normed and weights
  814. for the different possible semantics.
  815. edges : list
  816. A list of D arrays describing the bin edges for each dimension.
  817. See Also
  818. --------
  819. histogram: 1-D histogram
  820. histogram2d: 2-D histogram
  821. Examples
  822. --------
  823. >>> r = np.random.randn(100,3)
  824. >>> H, edges = np.histogramdd(r, bins = (5, 8, 4))
  825. >>> H.shape, edges[0].size, edges[1].size, edges[2].size
  826. ((5, 8, 4), 6, 9, 5)
  827. """
  828. try:
  829. # Sample is an ND-array.
  830. N, D = sample.shape
  831. except (AttributeError, ValueError):
  832. # Sample is a sequence of 1D arrays.
  833. sample = np.atleast_2d(sample).T
  834. N, D = sample.shape
  835. nbin = np.empty(D, int)
  836. edges = D*[None]
  837. dedges = D*[None]
  838. if weights is not None:
  839. weights = np.asarray(weights)
  840. try:
  841. M = len(bins)
  842. if M != D:
  843. raise ValueError(
  844. 'The dimension of bins must be equal to the dimension of the '
  845. ' sample x.')
  846. except TypeError:
  847. # bins is an integer
  848. bins = D*[bins]
  849. # normalize the range argument
  850. if range is None:
  851. range = (None,) * D
  852. elif len(range) != D:
  853. raise ValueError('range argument must have one entry per dimension')
  854. # Create edge arrays
  855. for i in _range(D):
  856. if np.ndim(bins[i]) == 0:
  857. if bins[i] < 1:
  858. raise ValueError(
  859. '`bins[{}]` must be positive, when an integer'.format(i))
  860. smin, smax = _get_outer_edges(sample[:,i], range[i])
  861. try:
  862. n = operator.index(bins[i])
  863. except TypeError as e:
  864. raise TypeError(
  865. "`bins[{}]` must be an integer, when a scalar".format(i)
  866. ) from e
  867. edges[i] = np.linspace(smin, smax, n + 1)
  868. elif np.ndim(bins[i]) == 1:
  869. edges[i] = np.asarray(bins[i])
  870. if np.any(edges[i][:-1] > edges[i][1:]):
  871. raise ValueError(
  872. '`bins[{}]` must be monotonically increasing, when an array'
  873. .format(i))
  874. else:
  875. raise ValueError(
  876. '`bins[{}]` must be a scalar or 1d array'.format(i))
  877. nbin[i] = len(edges[i]) + 1 # includes an outlier on each end
  878. dedges[i] = np.diff(edges[i])
  879. # Compute the bin number each sample falls into.
  880. Ncount = tuple(
  881. # avoid np.digitize to work around gh-11022
  882. np.searchsorted(edges[i], sample[:, i], side='right')
  883. for i in _range(D)
  884. )
  885. # Using digitize, values that fall on an edge are put in the right bin.
  886. # For the rightmost bin, we want values equal to the right edge to be
  887. # counted in the last bin, and not as an outlier.
  888. for i in _range(D):
  889. # Find which points are on the rightmost edge.
  890. on_edge = (sample[:, i] == edges[i][-1])
  891. # Shift these points one bin to the left.
  892. Ncount[i][on_edge] -= 1
  893. # Compute the sample indices in the flattened histogram matrix.
  894. # This raises an error if the array is too large.
  895. xy = np.ravel_multi_index(Ncount, nbin)
  896. # Compute the number of repetitions in xy and assign it to the
  897. # flattened histmat.
  898. hist = np.bincount(xy, weights, minlength=nbin.prod())
  899. # Shape into a proper matrix
  900. hist = hist.reshape(nbin)
  901. # This preserves the (bad) behavior observed in gh-7845, for now.
  902. hist = hist.astype(float, casting='safe')
  903. # Remove outliers (indices 0 and -1 for each dimension).
  904. core = D*(slice(1, -1),)
  905. hist = hist[core]
  906. # handle the aliasing normed argument
  907. if normed is None:
  908. if density is None:
  909. density = False
  910. elif density is None:
  911. # an explicit normed argument was passed, alias it to the new name
  912. density = normed
  913. else:
  914. raise TypeError("Cannot specify both 'normed' and 'density'")
  915. if density:
  916. # calculate the probability density function
  917. s = hist.sum()
  918. for i in _range(D):
  919. shape = np.ones(D, int)
  920. shape[i] = nbin[i] - 2
  921. hist = hist / dedges[i].reshape(shape)
  922. hist /= s
  923. if (hist.shape != nbin - 2).any():
  924. raise RuntimeError(
  925. "Internal Shape Error")
  926. return hist, edges