Nav2 Navigation Stack - lyrical  lyrical
ROS 2 Navigation Stack
robin_hood.h
1 // Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com>
2 // ______ _____ ______ _________
3 // ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ /
4 // __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ /
5 // _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ /
6 // /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
7 // _/_____/
8 //
9 // Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
10 // https://github.com/martinus/robin-hood-hashing
11 //
12 // Licensed under the MIT License <http://opensource.org/licenses/MIT>.
13 // SPDX-License-Identifier: MIT
14 // Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com>
15 //
16 // Permission is hereby granted, free of charge, to any person obtaining a copy
17 // of this software and associated documentation files (the "Software"), to deal
18 // in the Software without restriction, including without limitation the rights
19 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
20 // copies of the Software, and to permit persons to whom the Software is
21 // furnished to do so, subject to the following conditions:
22 //
23 // The above copyright notice and this permission notice shall be included in all
24 // copies or substantial portions of the Software.
25 //
26 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
31 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 // SOFTWARE.
33 
34 #ifndef NAV2_SMAC_PLANNER__THIRDPARTY__ROBIN_HOOD_H_
35 #define NAV2_SMAC_PLANNER__THIRDPARTY__ROBIN_HOOD_H_
36 
37 // see https://semver.org/
38 #define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes
39 #define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner
40 #define ROBIN_HOOD_VERSION_PATCH 5 // for backwards-compatible bug fixes
41 
42 #include <algorithm>
43 #include <cstdlib>
44 #include <cstring>
45 #include <functional>
46 #include <limits>
47 #include <memory> // only to support hash of smart pointers
48 #include <stdexcept>
49 #include <string>
50 #include <type_traits>
51 #include <tuple>
52 #include <utility>
53 #if __cplusplus >= 201703L
54 # include <string_view>
55 #endif
56 /* *INDENT-OFF* */
57 
58 // #define ROBIN_HOOD_LOG_ENABLED
59 #ifdef ROBIN_HOOD_LOG_ENABLED
60 # include <iostream>
61 # define ROBIN_HOOD_LOG(...) \
62  std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
63 #else
64 # define ROBIN_HOOD_LOG(x)
65 #endif
66 
67 // #define ROBIN_HOOD_TRACE_ENABLED
68 #ifdef ROBIN_HOOD_TRACE_ENABLED
69 # include <iostream>
70 # define ROBIN_HOOD_TRACE(...) \
71  std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
72 #else
73 # define ROBIN_HOOD_TRACE(x)
74 #endif
75 
76 // #define ROBIN_HOOD_COUNT_ENABLED
77 #ifdef ROBIN_HOOD_COUNT_ENABLED
78 # include <iostream>
79 # define ROBIN_HOOD_COUNT(x) ++counts().x;
80 namespace robin_hood {
81 struct Counts {
82  uint64_t shiftUp{};
83  uint64_t shiftDown{};
84 };
85 inline std::ostream& operator<<(std::ostream& os, Counts const& c) {
86  return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
87 }
88 
89 static Counts& counts() {
90  static Counts counts{};
91  return counts;
92 }
93 } // namespace robin_hood
94 #else
95 # define ROBIN_HOOD_COUNT(x)
96 #endif
97 
98 // all non-argument macros should use this facility. See
99 // https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
100 #define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()
101 
102 // mark unused members with this macro
103 #define ROBIN_HOOD_UNUSED(identifier)
104 
105 // bitness
106 #if SIZE_MAX == UINT32_MAX
107 # define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
108 #elif SIZE_MAX == UINT64_MAX
109 # define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
110 #else
111 # error Unsupported bitness
112 #endif
113 
114 // endianness
115 #ifdef _MSC_VER
116 # define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
117 # define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
118 #else
119 # define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \
120  (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
121 # define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
122 #endif
123 
124 // inline
125 #ifdef _MSC_VER
126 # define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
127 #else
128 # define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
129 #endif
130 
131 // exceptions
132 #if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
133 # define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
134 #else
135 # define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
136 #endif
137 
138 // count leading/trailing bits
139 #if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
140 # ifdef _MSC_VER
141 # if ROBIN_HOOD(BITNESS) == 32
142 # define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
143 # else
144 # define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
145 # endif
146 # include <intrin.h>
147 # pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
148 # define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \
149  [](size_t mask) noexcept -> int { \
150  /* NOLINTNEXTLINE(runtime/int) */ \
151  unsigned long index; \
152  return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \
153  : ROBIN_HOOD(BITNESS); \
154  }(x)
155 # else
156 # if ROBIN_HOOD(BITNESS) == 32
157 # define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
158 # define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
159 # else
160 # define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
161 # define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
162 # endif
163 # define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
164 # define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
165 # endif
166 #endif
167 
168 // fallthrough
169 #ifndef __has_cpp_attribute // For backwards compatibility
170 # define __has_cpp_attribute(x) 0
171 #endif
172 #if __has_cpp_attribute(clang::fallthrough)
173 # define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
174 #elif __has_cpp_attribute(gnu::fallthrough)
175 # define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
176 #else
177 # define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
178 #endif
179 
180 // likely/unlikely
181 #ifdef _MSC_VER
182 # define ROBIN_HOOD_LIKELY(condition) condition
183 # define ROBIN_HOOD_UNLIKELY(condition) condition
184 #else
185 # define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
186 # define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
187 #endif
188 
189 // detect if native wchar_t type is available in MSVC
190 #ifdef _MSC_VER
191 # ifdef _NATIVE_WCHAR_T_DEFINED
192 # define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
193 # else
194 # define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
195 # endif
196 #else
197 # define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
198 #endif
199 
200 // detect if MSVC supports the pair(std::piecewise_construct_t,...) constructor being constexpr
201 #ifdef _MSC_VER
202 # if _MSC_VER <= 1900
203 # define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1
204 # else
205 # define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
206 # endif
207 #else
208 # define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
209 #endif
210 
211 // workaround missing "is_trivially_copyable" in g++ < 5.0
212 // See https://stackoverflow.com/a/31798726/48181
213 #if defined(__GNUC__) && __GNUC__ < 5
214 # define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
215 #else
216 # define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
217 #endif
218 
219 // helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
220 #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
221 #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
222 #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
223 #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
224 #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L
225 
226 #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
227 # define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
228 #else
229 # define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
230 #endif
231 
232 namespace robin_hood {
233 
234 #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
235 # define ROBIN_HOOD_STD std
236 #else
237 
238 // c++11 compatibility layer
239 namespace ROBIN_HOOD_STD {
240 template <class T>
242  : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};
243 
244 template <class T, T... Ints>
246 public:
247  using value_type = T;
248  static_assert(std::is_integral<value_type>::value, "not integral type");
249  static constexpr std::size_t size() noexcept {
250  return sizeof...(Ints);
251  }
252 };
253 template <std::size_t... Inds>
254 using index_sequence = integer_sequence<std::size_t, Inds...>;
255 
256 namespace detail_ {
257 template <class T, T Begin, T End, bool>
258 struct IntSeqImpl {
259  using TValue = T;
260  static_assert(std::is_integral<TValue>::value, "not integral type");
261  static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");
262 
263  template <class, class>
265 
266  template <TValue... Inds0, TValue... Inds1>
267  struct IntSeqCombiner<integer_sequence<TValue, Inds0...>, integer_sequence<TValue, Inds1...>> {
268  using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
269  };
270 
271  using TResult =
272  typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2,
273  (End - Begin) / 2 == 1>::TResult,
274  typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End,
275  (End - Begin + 1) / 2 == 1>::TResult>::TResult;
276 };
277 
278 template <class T, T Begin>
279 struct IntSeqImpl<T, Begin, Begin, false> {
280  using TValue = T;
281  static_assert(std::is_integral<TValue>::value, "not integral type");
282  static_assert(Begin >= 0, "unexpected argument (Begin<0)");
284 };
285 
286 template <class T, T Begin, T End>
287 struct IntSeqImpl<T, Begin, End, true> {
288  using TValue = T;
289  static_assert(std::is_integral<TValue>::value, "not integral type");
290  static_assert(Begin >= 0, "unexpected argument (Begin<0)");
292 };
293 } // namespace detail_
294 
295 template <class T, T N>
296 using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;
297 
298 template <std::size_t N>
299 using make_index_sequence = make_integer_sequence<std::size_t, N>;
300 
301 template <class... T>
302 using index_sequence_for = make_index_sequence<sizeof...(T)>;
303 
304 } // namespace ROBIN_HOOD_STD
305 
306 #endif
307 
308 namespace detail {
309 
310 // make sure we static_cast to the correct type for hash_int
311 #if ROBIN_HOOD(BITNESS) == 64
312 using SizeT = uint64_t;
313 #else
314 using SizeT = uint32_t;
315 #endif
316 
317 template <typename T>
318 T rotr(T x, unsigned k) {
319  return (x >> k) | (x << (8U * sizeof(T) - k));
320 }
321 
322 // This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
323 // 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
324 // care!
325 template <typename T>
326 inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept {
327  return reinterpret_cast<T>(ptr);
328 }
329 
330 template <typename T>
331 inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept {
332  return reinterpret_cast<T>(ptr);
333 }
334 
335 // make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
336 // inlinings more difficult. Throws are also generally the slow path.
337 template <typename E, typename... Args>
338 [[noreturn]] ROBIN_HOOD(NOINLINE)
339 #if ROBIN_HOOD(HAS_EXCEPTIONS)
340  void doThrow(Args&&... args) {
341  throw E(std::forward<Args>(args)...);
342 }
343 #else
344  void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) {
345  abort();
346 }
347 #endif
348 
349 template <typename E, typename T, typename... Args>
350 T* assertNotNull(T* t, Args&&... args) {
351  if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
352  doThrow<E>(std::forward<Args>(args)...);
353  }
354  return t;
355 }
356 
357 template <typename T>
358 inline T unaligned_load(void const* ptr) noexcept {
359  // using memcpy so we don't get into unaligned load problems.
360  // compiler should optimize this very well anyways.
361  T t;
362  std::memcpy(&t, ptr, sizeof(T));
363  return t;
364 }
365 
366 // Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
367 // and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
368 // pointer.
369 template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
371 public:
372  BulkPoolAllocator() noexcept = default;
373 
374  // does not copy anything, just creates a new allocator.
375  BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept
376  : mHead(nullptr)
377  , mListForFree(nullptr) {}
378 
380  : mHead(o.mHead)
381  , mListForFree(o.mListForFree) {
382  o.mListForFree = nullptr;
383  o.mHead = nullptr;
384  }
385 
386  BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept {
387  reset();
388  mHead = o.mHead;
389  mListForFree = o.mListForFree;
390  o.mListForFree = nullptr;
391  o.mHead = nullptr;
392  return *this;
393  }
394 
396  operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
397  // does not do anything
398  return *this;
399  }
400 
401  ~BulkPoolAllocator() noexcept {
402  reset();
403  }
404 
405  // Deallocates all allocated memory.
406  void reset() noexcept {
407  while (mListForFree) {
408  T* tmp = *mListForFree;
409  ROBIN_HOOD_LOG("std::free")
410  std::free(mListForFree);
411  mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp);
412  }
413  mHead = nullptr;
414  }
415 
416  // allocates, but does NOT initialize. Use in-place new constructor, e.g.
417  // T* obj = pool.allocate();
418  // ::new (static_cast<void*>(obj)) T();
419  T* allocate() {
420  T* tmp = mHead;
421  if (!tmp) {
422  tmp = performAllocation();
423  }
424 
425  mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp);
426  return tmp;
427  }
428 
429  // does not actually deallocate but puts it in store.
430  // make sure you have already called the destructor! e.g. with
431  // obj->~T();
432  // pool.deallocate(obj);
433  void deallocate(T* obj) noexcept {
434  *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead;
435  mHead = obj;
436  }
437 
438  // Adds an already allocated block of memory to the allocator. This allocator is from now on
439  // responsible for freeing the data (with free()). If the provided data is not large enough to
440  // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
441  void addOrFree(void* ptr, const size_t numBytes) noexcept {
442  // calculate number of available elements in ptr
443  if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
444  // not enough data for at least one element. Free and return.
445  ROBIN_HOOD_LOG("std::free")
446  std::free(ptr);
447  } else {
448  ROBIN_HOOD_LOG("add to buffer")
449  add(ptr, numBytes);
450  }
451  }
452 
453  void swap(BulkPoolAllocator<T, MinNumAllocs, MaxNumAllocs>& other) noexcept {
454  using std::swap;
455  swap(mHead, other.mHead);
456  swap(mListForFree, other.mListForFree);
457  }
458 
459 private:
460  // iterates the list of allocated memory to calculate how many to alloc next.
461  // Recalculating this each time saves us a size_t member.
462  // This ignores the fact that memory blocks might have been added manually with addOrFree. In
463  // practice, this should not matter much.
464  ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
465  auto tmp = mListForFree;
466  size_t numAllocs = MinNumAllocs;
467 
468  while (numAllocs * 2 <= MaxNumAllocs && tmp) {
469  auto x = reinterpret_cast<T***>(tmp); // NOLINT
470  tmp = *x;
471  numAllocs *= 2;
472  }
473 
474  return numAllocs;
475  }
476 
477  // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
478  void add(void* ptr, const size_t numBytes) noexcept {
479  const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;
480  auto data = reinterpret_cast<T**>(ptr);
481 
482  // link free list
483  auto x = reinterpret_cast<T***>(data);
484  *x = mListForFree;
485  mListForFree = data;
486 
487  // create linked list for newly allocated data
488  auto* const headT =
489  reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT);
490 
491  auto* const head = reinterpret_cast<char*>(headT);
492 
493  // Visual Studio compiler automatically unrolls this loop, which is pretty cool
494  for (size_t i = 0; i < numElements; ++i) {
495  *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) =
496  head + (i + 1) * ALIGNED_SIZE;
497  }
498 
499  // last one points to 0
500  *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) =
501  mHead;
502  mHead = headT;
503  }
504 
505  // Called when no memory is available (mHead == 0).
506  // Don't inline this slow path.
507  ROBIN_HOOD(NOINLINE) T* performAllocation() {
508  size_t const numElementsToAlloc = calcNumElementsToAlloc();
509 
510  // alloc new memory: [prev |T, T, ... T]
511  size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
512  ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE
513  << " * " << numElementsToAlloc)
514  add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes);
515  return mHead;
516  }
517 
518  // enforce byte alignment of the T's
519 #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
520  static constexpr size_t ALIGNMENT =
521  (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value);
522 #else
523  static const size_t ALIGNMENT =
526  : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround
527 #endif
528 
529  static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;
530 
531  static_assert(MinNumAllocs >= 1, "MinNumAllocs");
532  static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
533  static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE");
534  static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod");
535  static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT");
536 
537  T* mHead{nullptr};
538  T** mListForFree{nullptr};
539 };
540 
541 template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
543 
544 // dummy allocator that does nothing
545 template <typename T, size_t MinSize, size_t MaxSize>
546 struct NodeAllocator<T, MinSize, MaxSize, true> {
547  // we are not using the data, so just free it.
548  void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes)/*unused*/) noexcept {
549  ROBIN_HOOD_LOG("std::free")
550  std::free(ptr);
551  }
552 };
553 
554 template <typename T, size_t MinSize, size_t MaxSize>
555 struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};
556 
557 // c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
558 // my own here.
559 namespace swappable {
560 #if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
561 using std::swap;
562 template <typename T>
563 struct nothrow {
564  static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>()));
565 };
566 #else
567 template <typename T>
568 struct nothrow {
569  static const bool value = std::is_nothrow_swappable<T>::value;
570 };
571 #endif
572 } // namespace swappable
573 
574 } // namespace detail
575 
577 
578 // A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
579 // which means it would not be allowed to be used in std::memcpy. This struct is copiable, which is
580 // also tested.
581 template <typename T1, typename T2>
582 struct pair {
583  using first_type = T1;
584  using second_type = T2;
585 
586  template <typename U1 = T1, typename U2 = T2,
587  typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
588  std::is_default_constructible<U2>::value>::type>
589  constexpr pair() noexcept(noexcept(U1()) && noexcept(U2()))
590  : first()
591  , second() {}
592 
593  // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
594  explicit constexpr pair(std::pair<T1, T2> const& o) noexcept(
595  noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>())))
596  : first(o.first)
597  , second(o.second) {}
598 
599  // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
600  explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept(
601  T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
602  : first(std::move(o.first))
603  , second(std::move(o.second)) {}
604 
605  constexpr pair(T1&& a, T2&& b) noexcept(noexcept(
606  T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
607  : first(std::move(a))
608  , second(std::move(b)) {}
609 
610  template <typename U1, typename U2>
611  constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>(
612  std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>()))))
613  : first(std::forward<U1>(a))
614  , second(std::forward<U2>(b)) {}
615 
616  template <typename... U1, typename... U2>
617  // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members"
618  // if this constructor is constexpr
619 #if !ROBIN_HOOD(BROKEN_CONSTEXPR)
620  constexpr
621 #endif
622  pair(std::piecewise_construct_t /*unused*/, std::tuple<U1...> a,
623  std::tuple<U2...>
624  b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(),
625  std::declval<std::tuple<U2...>&>(),
626  ROBIN_HOOD_STD::index_sequence_for<U1...>(),
627  ROBIN_HOOD_STD::index_sequence_for<U2...>())))
628  : pair(a, b, ROBIN_HOOD_STD::index_sequence_for<U1...>(),
629  ROBIN_HOOD_STD::index_sequence_for<U2...>()) {
630  }
631 
632  // constructor called from the std::piecewise_construct_t ctor
633  template <typename... U1, size_t... I1, typename... U2, size_t... I2>
634  pair(
635  std::tuple<U1...>& a, std::tuple<U2...>& b,
637  ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept(
638  noexcept(T1(std::forward<U1>(std::get<I1>(
639  std::declval<std::tuple<
640  U1...>&>()))...)) && noexcept(T2(std::
641  forward<U2>(std::get<I2>(
642  std::declval<std::tuple<U2...>&>()))...)))
643  : first(std::forward<U1>(std::get<I1>(a))...)
644  , second(std::forward<U2>(std::get<I2>(b))...) {
645  // make visual studio compiler happy about warning about unused a & b.
646  // Visual studio's pair implementation disables warning 4100.
647  (void)a;
648  (void)b;
649  }
650 
651  void swap(pair<T1, T2>& o) noexcept((detail::swappable::nothrow<T1>::value) &&
653  using std::swap;
654  swap(first, o.first);
655  swap(second, o.second);
656  }
657 
658  T1 first; // NOLINT
659  T2 second; // NOLINT
660 };
661 
662 template <typename A, typename B>
663 inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept(
664  noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) {
665  a.swap(b);
666 }
667 
668 template <typename A, typename B>
669 inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) {
670  return (x.first == y.first) && (x.second == y.second);
671 }
672 template <typename A, typename B>
673 inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) {
674  return !(x == y);
675 }
676 template <typename A, typename B>
677 inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept(
678  std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() <
679  std::declval<B const&>())) {
680  return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
681 }
682 template <typename A, typename B>
683 inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) {
684  return y < x;
685 }
686 template <typename A, typename B>
687 inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) {
688  return !(x > y);
689 }
690 template <typename A, typename B>
691 inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) {
692  return !(x < y);
693 }
694 
695 inline size_t hash_bytes(void const* ptr, size_t len) noexcept {
696  static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
697  static constexpr uint64_t seed = UINT64_C(0xe17a1465);
698  static constexpr unsigned int r = 47;
699 
700  auto const* const data64 = static_cast<uint64_t const*>(ptr);
701  uint64_t h = seed ^ (len * m);
702 
703  size_t const n_blocks = len / 8;
704  for (size_t i = 0; i < n_blocks; ++i) {
705  auto k = detail::unaligned_load<uint64_t>(data64 + i);
706 
707  k *= m;
708  k ^= k >> r;
709  k *= m;
710 
711  h ^= k;
712  h *= m;
713  }
714 
715  auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks);
716  switch (len & 7U) {
717  case 7:
718  h ^= static_cast<uint64_t>(data8[6]) << 48U;
719  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
720  case 6:
721  h ^= static_cast<uint64_t>(data8[5]) << 40U;
722  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
723  case 5:
724  h ^= static_cast<uint64_t>(data8[4]) << 32U;
725  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
726  case 4:
727  h ^= static_cast<uint64_t>(data8[3]) << 24U;
728  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
729  case 3:
730  h ^= static_cast<uint64_t>(data8[2]) << 16U;
731  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
732  case 2:
733  h ^= static_cast<uint64_t>(data8[1]) << 8U;
734  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
735  case 1:
736  h ^= static_cast<uint64_t>(data8[0]);
737  h *= m;
738  ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
739  default:
740  break;
741  }
742 
743  h ^= h >> r;
744 
745  // not doing the final step here, because this will be done by keyToIdx anyways
746  // h *= m;
747  // h ^= h >> r;
748  return static_cast<size_t>(h);
749 }
750 
751 inline size_t hash_int(uint64_t x) noexcept {
752  // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested,
753  // and doesn't need any special 128bit operations.
754  x ^= x >> 33U;
755  x *= UINT64_C(0xff51afd7ed558ccd);
756  x ^= x >> 33U;
757 
758  // not doing the final step here, because this will be done by keyToIdx anyways
759  // x *= UINT64_C(0xc4ceb9fe1a85ec53);
760  // x ^= x >> 33U;
761  return static_cast<size_t>(x);
762 }
763 
764 // A thin wrapper around std::hash, performing an additional simple mixing step of the result.
765 template <typename T, typename Enable = void>
766 struct hash : public std::hash<T> {
767  size_t operator()(T const& obj) const
768  noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) {
769  // call base hash
770  auto result = std::hash<T>::operator()(obj);
771  // return mixed of that, to be save against identity has
772  return hash_int(static_cast<detail::SizeT>(result));
773  }
774 };
775 
776 template <typename CharT>
777 struct hash<std::basic_string<CharT>> {
778  size_t operator()(std::basic_string<CharT> const& str) const noexcept {
779  return hash_bytes(str.data(), sizeof(CharT) * str.size());
780  }
781 };
782 
783 #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
784 template <typename CharT>
785 struct hash<std::basic_string_view<CharT>> {
786  size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept {
787  return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
788  }
789 };
790 #endif
791 
792 template <class T>
793 struct hash<T*> {
794  size_t operator()(T* ptr) const noexcept {
795  return hash_int(reinterpret_cast<detail::SizeT>(ptr));
796  }
797 };
798 
799 template <class T>
800 struct hash<std::unique_ptr<T>> {
801  size_t operator()(std::unique_ptr<T> const& ptr) const noexcept {
802  return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
803  }
804 };
805 
806 template <class T>
807 struct hash<std::shared_ptr<T>> {
808  size_t operator()(std::shared_ptr<T> const& ptr) const noexcept {
809  return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
810  }
811 };
812 
813 template <typename Enum>
814 struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
815  size_t operator()(Enum e) const noexcept {
816  using Underlying = typename std::underlying_type<Enum>::type;
817  return hash<Underlying>{}(static_cast<Underlying>(e));
818  }
819 };
820 
821 #define ROBIN_HOOD_HASH_INT(T) \
822  template <> \
823  struct hash<T> { \
824  size_t operator()(T const& obj) const noexcept { \
825  return hash_int(static_cast<uint64_t>(obj)); \
826  } \
827  }
828 
829 #if defined(__GNUC__) && !defined(__clang__)
830 # pragma GCC diagnostic push
831 # pragma GCC diagnostic ignored "-Wuseless-cast"
832 #endif
833 // see https://en.cppreference.com/w/cpp/utility/hash
834 ROBIN_HOOD_HASH_INT(bool);
835 ROBIN_HOOD_HASH_INT(char);
836 ROBIN_HOOD_HASH_INT(signed char);
837 ROBIN_HOOD_HASH_INT(unsigned char);
838 ROBIN_HOOD_HASH_INT(char16_t);
839 ROBIN_HOOD_HASH_INT(char32_t);
840 #if ROBIN_HOOD(HAS_NATIVE_WCHART)
841 ROBIN_HOOD_HASH_INT(wchar_t);
842 #endif
843 ROBIN_HOOD_HASH_INT(short); // NOLINT
844 ROBIN_HOOD_HASH_INT(unsigned short); // NOLINT
845 ROBIN_HOOD_HASH_INT(int);
846 ROBIN_HOOD_HASH_INT(unsigned int);
847 ROBIN_HOOD_HASH_INT(long); // NOLINT
848 ROBIN_HOOD_HASH_INT(long long); // NOLINT
849 ROBIN_HOOD_HASH_INT(unsigned long); // NOLINT
850 ROBIN_HOOD_HASH_INT(unsigned long long); // NOLINT
851 #if defined(__GNUC__) && !defined(__clang__)
852 # pragma GCC diagnostic pop
853 #endif
854 namespace detail {
855 
856 template <typename T>
857 struct void_type {
858  using type = void;
859 };
860 
861 template <typename T, typename = void>
862 struct has_is_transparent : public std::false_type {};
863 
864 template <typename T>
865 struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type>
866  : public std::true_type {};
867 
868 // using wrapper classes for hash and key_equal prevents the diamond problem when the same type
869 // is used. see https://stackoverflow.com/a/28771920/48181
870 template <typename T>
871 struct WrapHash : public T {
872  WrapHash() = default;
873  explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
874  : T(o) {}
875 };
876 
877 template <typename T>
878 struct WrapKeyEqual : public T {
879  WrapKeyEqual() = default;
880  explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
881  : T(o) {}
882 };
883 
884 // A highly optimized hashmap implementation, using the Robin Hood algorithm.
885 //
886 // In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
887 // be about 2x faster in most cases and require much less allocations.
888 //
889 // This implementation uses the following memory layout:
890 //
891 // [Node, Node, ... Node | info, info, ... infoSentinel ]
892 //
893 // * Node: either a DataNode that directly has the std::pair<key, val> as member,
894 // or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
895 // depends on how fast the swap() operation is. Heuristically, this is automatically chosen
896 // based on sizeof(). there are always 2^n Nodes.
897 //
898 // * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
899 // Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
900 // corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
901 // actually belongs to the previous position and was pushed out because that place is already
902 // taken.
903 //
904 // * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
905 // need for a idx variable.
906 //
907 // According to STL, order of templates has effect on throughput. That's why I've moved the
908 // boolean to the front.
909 // https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
910 template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash,
911  typename KeyEqual>
912 class Table
913  : public WrapHash<Hash>,
914  public WrapKeyEqual<KeyEqual>,
916  typename std::conditional<
917  std::is_void<T>::value, Key,
918  robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
919  4, 16384, IsFlat> {
920 public:
921  static constexpr bool is_flat = IsFlat;
922  static constexpr bool is_map = !std::is_void<T>::value;
923  static constexpr bool is_set = !is_map;
924  static constexpr bool is_transparent =
926 
927  using key_type = Key;
928  using mapped_type = T;
929  using value_type = typename std::conditional<
930  is_set, Key,
932  using size_type = size_t;
933  using hasher = Hash;
934  using key_equal = KeyEqual;
936 
937 private:
938  static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100,
939  "MaxLoadFactor100 needs to be >10 && < 100");
940 
941  using WHash = WrapHash<Hash>;
943 
944  // configuration defaults
945 
946  // make sure we have 8 elements, needed to quickly rehash mInfo
947  static constexpr size_t InitialNumElements = sizeof(uint64_t);
948  static constexpr uint32_t InitialInfoNumBits = 5;
949  static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
950  static constexpr size_t InfoMask = InitialInfoInc - 1U;
951  static constexpr uint8_t InitialInfoHashShift = 0;
953 
954  // type needs to be wider than uint8_t.
955  using InfoType = uint32_t;
956 
957  // DataNode ////////////////////////////////////////////////////////
958 
959  // Primary template for the data node. We have special implementations for small and big
960  // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
961  // on the heap so swap merely swaps a pointer.
962  template <typename M, bool>
963  class DataNode {};
964 
965  // Small: just allocate on the stack.
966  template <typename M>
967  class DataNode<M, true> final {
968  public:
969  template <typename... Args>
970  explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept(
971  noexcept(value_type(std::forward<Args>(args)...)))
972  : mData(std::forward<Args>(args)...) {}
973 
974  DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept(
975  std::is_nothrow_move_constructible<value_type>::value)
976  : mData(std::move(n.mData)) {}
977 
978  // doesn't do anything
979  void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
980  void destroyDoNotDeallocate() noexcept {}
981 
982  value_type const* operator->() const noexcept {
983  return &mData;
984  }
985  value_type* operator->() noexcept {
986  return &mData;
987  }
988 
989  const value_type& operator*() const noexcept {
990  return mData;
991  }
992 
993  value_type& operator*() noexcept {
994  return mData;
995  }
996 
997  template <typename VT = value_type>
998  ROBIN_HOOD(NODISCARD)
999  typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
1000  return mData.first;
1001  }
1002  template <typename VT = value_type>
1003  ROBIN_HOOD(NODISCARD)
1004  typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
1005  return mData;
1006  }
1007 
1008  template <typename VT = value_type>
1009  ROBIN_HOOD(NODISCARD)
1010  typename std::enable_if<is_map, typename VT::first_type const&>::type
1011  getFirst() const noexcept {
1012  return mData.first;
1013  }
1014  template <typename VT = value_type>
1015  ROBIN_HOOD(NODISCARD)
1016  typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
1017  return mData;
1018  }
1019 
1020  template <typename MT = mapped_type>
1021  ROBIN_HOOD(NODISCARD)
1022  typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
1023  return mData.second;
1024  }
1025 
1026  template <typename MT = mapped_type>
1027  ROBIN_HOOD(NODISCARD)
1028  typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept {
1029  return mData.second;
1030  }
1031 
1032  void swap(DataNode<M, true>& o) noexcept(
1033  noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
1034  mData.swap(o.mData);
1035  }
1036 
1037  private:
1038  value_type mData;
1039  };
1040 
1041  // big object: allocate on heap.
1042  template <typename M>
1043  class DataNode<M, false> {
1044  public:
1045  template <typename... Args>
1046  explicit DataNode(M& map, Args&&... args)
1047  : mData(map.allocate()) {
1048  ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...);
1049  }
1050 
1051  DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept
1052  : mData(std::move(n.mData)) {}
1053 
1054  void destroy(M& map) noexcept {
1055  // don't deallocate, just put it into list of datapool.
1056  mData->~value_type();
1057  map.deallocate(mData);
1058  }
1059 
1060  void destroyDoNotDeallocate() noexcept {
1061  mData->~value_type();
1062  }
1063 
1064  value_type const* operator->() const noexcept {
1065  return mData;
1066  }
1067 
1068  value_type* operator->() noexcept {
1069  return mData;
1070  }
1071 
1072  const value_type& operator*() const {
1073  return *mData;
1074  }
1075 
1076  value_type& operator*() {
1077  return *mData;
1078  }
1079 
1080  template <typename VT = value_type>
1081  ROBIN_HOOD(NODISCARD)
1082  typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
1083  return mData->first;
1084  }
1085  template <typename VT = value_type>
1086  ROBIN_HOOD(NODISCARD)
1087  typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
1088  return *mData;
1089  }
1090 
1091  template <typename VT = value_type>
1092  ROBIN_HOOD(NODISCARD)
1093  typename std::enable_if<is_map, typename VT::first_type const&>::type
1094  getFirst() const noexcept {
1095  return mData->first;
1096  }
1097  template <typename VT = value_type>
1098  ROBIN_HOOD(NODISCARD)
1099  typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
1100  return *mData;
1101  }
1102 
1103  template <typename MT = mapped_type>
1104  ROBIN_HOOD(NODISCARD)
1105  typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
1106  return mData->second;
1107  }
1108 
1109  template <typename MT = mapped_type>
1110  ROBIN_HOOD(NODISCARD)
1111  typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept {
1112  return mData->second;
1113  }
1114 
1115  void swap(DataNode<M, false>& o) noexcept {
1116  using std::swap;
1117  swap(mData, o.mData);
1118  }
1119 
1120  private:
1121  value_type* mData;
1122  };
1123 
1124  using Node = DataNode<Self, IsFlat>;
1125 
1126  // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required)
1127  ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept {
1128  return n.getFirst();
1129  }
1130 
1131  // in case we have void mapped_type, we are not using a pair, thus we just route k through.
1132  // No need to disable this because it's just not used if not applicable.
1133  ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept {
1134  return k;
1135  }
1136 
1137  // in case we have non-void mapped_type, we have a standard robin_hood::pair
1138  template <typename Q = mapped_type>
1139  ROBIN_HOOD(NODISCARD)
1140  typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type
1141  getFirstConst(value_type const& vt) const noexcept {
1142  return vt.first;
1143  }
1144 
1145  // Cloner //////////////////////////////////////////////////////////
1146 
1147  template <typename M, bool UseMemcpy>
1148  struct Cloner;
1149 
1150  // fast path: Just copy data, without allocating anything.
1151  template <typename M>
1152  struct Cloner<M, true> {
1153  void operator()(M const& source, M& target) const {
1154  auto const* const src = reinterpret_cast<char const*>(source.mKeyVals);
1155  auto* tgt = reinterpret_cast<char*>(target.mKeyVals);
1156  auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
1157  std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
1158  }
1159  };
1160 
1161  template <typename M>
1162  struct Cloner<M, false> {
1163  void operator()(M const& s, M& t) const {
1164  auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
1165  std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);
1166 
1167  for (size_t i = 0; i < numElementsWithBuffer; ++i) {
1168  if (t.mInfo[i]) {
1169  ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
1170  }
1171  }
1172  }
1173  };
1174 
1175  // Destroyer ///////////////////////////////////////////////////////
1176 
1177  template <typename M, bool IsFlatAndTrivial>
1178  struct Destroyer {};
1179 
1180  template <typename M>
1181  struct Destroyer<M, true> {
1182  void nodes(M& m) const noexcept {
1183  m.mNumElements = 0;
1184  }
1185 
1186  void nodesDoNotDeallocate(M& m) const noexcept {
1187  m.mNumElements = 0;
1188  }
1189  };
1190 
1191  template <typename M>
1192  struct Destroyer<M, false> {
1193  void nodes(M& m) const noexcept {
1194  m.mNumElements = 0;
1195  // clear also resets mInfo to 0, that's sometimes not necessary.
1196  auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
1197 
1198  for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
1199  if (0 != m.mInfo[idx]) {
1200  Node& n = m.mKeyVals[idx];
1201  n.destroy(m);
1202  n.~Node();
1203  }
1204  }
1205  }
1206 
1207  void nodesDoNotDeallocate(M& m) const noexcept {
1208  m.mNumElements = 0;
1209  // clear also resets mInfo to 0, that's sometimes not necessary.
1210  auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
1211  for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
1212  if (0 != m.mInfo[idx]) {
1213  Node& n = m.mKeyVals[idx];
1214  n.destroyDoNotDeallocate();
1215  n.~Node();
1216  }
1217  }
1218  }
1219  };
1220 
1221  // Iter ////////////////////////////////////////////////////////////
1222 
1223  struct fast_forward_tag {};
1224 
1225  // generic iterator for both const_iterator and iterator.
1226  template <bool IsConst>
1227  class Iter {
1228  private:
1229  using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type;
1230 
1231  public:
1232  using difference_type = std::ptrdiff_t;
1233  using value_type = typename Self::value_type;
1234  using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type;
1235  using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type;
1236  using iterator_category = std::forward_iterator_tag;
1237 
1238  // default constructed iterator can be compared to itself, but WON'T return true when
1239  // compared to end().
1240  Iter() = default;
1241 
1242  // Rule of zero: nothing specified. The conversion constructor is only enabled for
1243  // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.
1244 
1245  // Conversion constructor from iterator to const_iterator.
1246  template <bool OtherIsConst,
1247  typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
1248  Iter(Iter<OtherIsConst> const& other) noexcept
1249  : mKeyVals(other.mKeyVals)
1250  , mInfo(other.mInfo) {}
1251 
1252  Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept
1253  : mKeyVals(valPtr)
1254  , mInfo(infoPtr) {}
1255 
1256  Iter(NodePtr valPtr, uint8_t const* infoPtr,
1257  fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
1258  : mKeyVals(valPtr)
1259  , mInfo(infoPtr) {
1260  fastForward();
1261  }
1262 
1263  template <bool OtherIsConst,
1264  typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
1265  Iter& operator=(Iter<OtherIsConst> const& other) noexcept {
1266  mKeyVals = other.mKeyVals;
1267  mInfo = other.mInfo;
1268  return *this;
1269  }
1270 
1271  // prefix increment. Undefined behavior if we are at end()!
1272  Iter& operator++() noexcept {
1273  mInfo++;
1274  mKeyVals++;
1275  fastForward();
1276  return *this;
1277  }
1278 
1279  Iter operator++(int) noexcept {
1280  Iter tmp = *this;
1281  ++(*this);
1282  return tmp;
1283  }
1284 
1285  reference operator*() const {
1286  return **mKeyVals;
1287  }
1288 
1289  pointer operator->() const {
1290  return &**mKeyVals;
1291  }
1292 
1293  template <bool O>
1294  bool operator==(Iter<O> const& o) const noexcept {
1295  return mKeyVals == o.mKeyVals;
1296  }
1297 
1298  template <bool O>
1299  bool operator!=(Iter<O> const& o) const noexcept {
1300  return mKeyVals != o.mKeyVals;
1301  }
1302 
1303  private:
1304  // fast forward to the next non-free info byte
1305  // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
1306  // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
1307  void fastForward() noexcept {
1308  size_t n = 0;
1309  while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
1310  mInfo += sizeof(size_t);
1311  mKeyVals += sizeof(size_t);
1312  }
1313 #if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
1314  // we know for certain that within the next 8 bytes we'll find a non-zero one.
1315  if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
1316  mInfo += 4;
1317  mKeyVals += 4;
1318  }
1319  if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
1320  mInfo += 2;
1321  mKeyVals += 2;
1322  }
1323  if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
1324  mInfo += 1;
1325  mKeyVals += 1;
1326  }
1327 #else
1328 # if ROBIN_HOOD(LITTLE_ENDIAN)
1329  auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
1330 # else
1331  auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
1332 # endif
1333  mInfo += inc;
1334  mKeyVals += inc;
1335 #endif
1336  }
1337 
1338  friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
1339  NodePtr mKeyVals{nullptr};
1340  uint8_t const* mInfo{nullptr};
1341  };
1342 
1344 
1345  // highly performance relevant code.
1346  // Lower bits are used for indexing into the array (2^n size)
1347  // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
1348  template <typename HashKey>
1349  void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const {
1350  // In addition to whatever hash is used, add another mul & shift so we get better hashing.
1351  // This serves as a bad hash prevention, if the given data is
1352  // badly mixed.
1353  auto h = static_cast<uint64_t>(WHash::operator()(key));
1354 
1355  h *= mHashMultiplier;
1356  h ^= h >> 33U;
1357 
1358  // the lower InitialInfoNumBits are reserved for info.
1359  *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
1360  *idx = (static_cast<size_t>(h) >> InitialInfoNumBits) & mMask;
1361  }
1362 
1363  // forwards the index by one, wrapping around at the end
1364  void next(InfoType* info, size_t* idx) const noexcept {
1365  *idx = *idx + 1;
1366  *info += mInfoInc;
1367  }
1368 
1369  void nextWhileLess(InfoType* info, size_t* idx) const noexcept {
1370  // unrolling this by hand did not bring any speedups.
1371  while (*info < mInfo[*idx]) {
1372  next(info, idx);
1373  }
1374  }
1375 
1376  // Shift everything up by one element. Tries to move stuff around.
1377  void
1378  shiftUp(size_t startIdx,
1379  size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
1380  auto idx = startIdx;
1381  ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
1382  while (--idx != insertion_idx) {
1383  mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
1384  }
1385 
1386  idx = startIdx;
1387  while (idx != insertion_idx) {
1388  ROBIN_HOOD_COUNT(shiftUp)
1389  mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
1390  if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
1391  mMaxNumElementsAllowed = 0;
1392  }
1393  --idx;
1394  }
1395  }
1396 
1397  void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
1398  // until we find one that is either empty or has zero offset.
1399  // TODO(martinus) we don't need to move everything, just the last one for the same
1400  // bucket.
1401  mKeyVals[idx].destroy(*this);
1402 
1403  // until we find one that is either empty or has zero offset.
1404  while (mInfo[idx + 1] >= 2 * mInfoInc) {
1405  ROBIN_HOOD_COUNT(shiftDown)
1406  mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
1407  mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
1408  ++idx;
1409  }
1410 
1411  mInfo[idx] = 0;
1412  // don't destroy, we've moved it
1413  // mKeyVals[idx].destroy(*this);
1414  mKeyVals[idx].~Node();
1415  }
1416 
1417  // copy of find(), except that it returns iterator instead of const_iterator.
1418  template <typename Other>
1419  ROBIN_HOOD(NODISCARD)
1420  size_t findIdx(Other const& key) const {
1421  size_t idx{};
1422  InfoType info{};
1423  keyToIdx(key, &idx, &info);
1424 
1425  do {
1426  // unrolling this twice gives a bit of a speedup. More unrolling did not help.
1427  if (info == mInfo[idx] &&
1428  ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
1429  return idx;
1430  }
1431  next(&info, &idx);
1432  if (info == mInfo[idx] &&
1433  ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
1434  return idx;
1435  }
1436  next(&info, &idx);
1437  } while (info <= mInfo[idx]);
1438 
1439  // nothing found!
1440  return mMask == 0 ? 0
1441  : static_cast<size_t>(std::distance(
1442  mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo)));
1443  }
1444 
1445  void cloneData(const Table& o) {
1446  Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this);
1447  }
1448 
1449  // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
1450  // @return True on success, false if something went wrong
1451  void insert_move(Node&& keyval) {
1452  // we don't retry, fail if overflowing
1453  // don't need to check max num elements
1454  if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
1455  throwOverflowError();
1456  }
1457 
1458  size_t idx{};
1459  InfoType info{};
1460  keyToIdx(keyval.getFirst(), &idx, &info);
1461 
1462  // skip forward. Use <= because we are certain that the element is not there.
1463  while (info <= mInfo[idx]) {
1464  idx = idx + 1;
1465  info += mInfoInc;
1466  }
1467 
1468  // key not found, so we are now exactly where we want to insert it.
1469  auto const insertion_idx = idx;
1470  auto const insertion_info = static_cast<uint8_t>(info);
1471  if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
1472  mMaxNumElementsAllowed = 0;
1473  }
1474 
1475  // find an empty spot
1476  while (0 != mInfo[idx]) {
1477  next(&info, &idx);
1478  }
1479 
1480  auto& l = mKeyVals[insertion_idx];
1481  if (idx == insertion_idx) {
1482  ::new (static_cast<void*>(&l)) Node(std::move(keyval));
1483  } else {
1484  shiftUp(idx, insertion_idx);
1485  l = std::move(keyval);
1486  }
1487 
1488  // put at empty spot
1489  mInfo[insertion_idx] = insertion_info;
1490 
1491  ++mNumElements;
1492  }
1493 
1494 public:
1495  using iterator = Iter<false>;
1496  using const_iterator = Iter<true>;
1497 
1498  Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual()))
1499  : WHash()
1500  , WKeyEqual() {
1501  ROBIN_HOOD_TRACE(this)
1502  }
1503 
1504  // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
1505  // This tremendously speeds up ctor & dtor of a map that never receives an element. The
1506  // penalty is paid at the first insert, and not before. Lookup of this empty map works
1507  // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
1508  // standard, but we can ignore it.
1509  explicit Table(
1510  size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{},
1511  const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal)))
1512  : WHash(h)
1513  , WKeyEqual(equal) {
1514  ROBIN_HOOD_TRACE(this)
1515  }
1516 
1517  template <typename Iter>
1518  Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
1519  const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{})
1520  : WHash(h)
1521  , WKeyEqual(equal) {
1522  ROBIN_HOOD_TRACE(this)
1523  insert(first, last);
1524  }
1525 
1526  Table(std::initializer_list<value_type> initlist,
1527  size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{},
1528  const KeyEqual& equal = KeyEqual{})
1529  : WHash(h)
1530  , WKeyEqual(equal) {
1531  ROBIN_HOOD_TRACE(this)
1532  insert(initlist.begin(), initlist.end());
1533  }
1534 
1535  Table(Table&& o) noexcept
1536  : WHash(std::move(static_cast<WHash&>(o)))
1537  , WKeyEqual(std::move(static_cast<WKeyEqual&>(o)))
1538  , DataPool(std::move(static_cast<DataPool&>(o))) {
1539  ROBIN_HOOD_TRACE(this)
1540  if (o.mMask) {
1541  mHashMultiplier = std::move(o.mHashMultiplier);
1542  mKeyVals = std::move(o.mKeyVals);
1543  mInfo = std::move(o.mInfo);
1544  mNumElements = std::move(o.mNumElements);
1545  mMask = std::move(o.mMask);
1546  mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
1547  mInfoInc = std::move(o.mInfoInc);
1548  mInfoHashShift = std::move(o.mInfoHashShift);
1549  // set other's mask to 0 so its destructor won't do anything
1550  o.init();
1551  }
1552  }
1553 
1554  Table& operator=(Table&& o) noexcept {
1555  ROBIN_HOOD_TRACE(this)
1556  if (&o != this) {
1557  if (o.mMask) {
1558  // only move stuff if the other map actually has some data
1559  destroy();
1560  mHashMultiplier = std::move(o.mHashMultiplier);
1561  mKeyVals = std::move(o.mKeyVals);
1562  mInfo = std::move(o.mInfo);
1563  mNumElements = std::move(o.mNumElements);
1564  mMask = std::move(o.mMask);
1565  mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
1566  mInfoInc = std::move(o.mInfoInc);
1567  mInfoHashShift = std::move(o.mInfoHashShift);
1568  WHash::operator=(std::move(static_cast<WHash&>(o)));
1569  WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o)));
1570  DataPool::operator=(std::move(static_cast<DataPool&>(o)));
1571 
1572  o.init();
1573 
1574  } else {
1575  // nothing in the other map => just clear us.
1576  clear();
1577  }
1578  }
1579  return *this;
1580  }
1581 
1582  Table(const Table& o)
1583  : WHash(static_cast<const WHash&>(o))
1584  , WKeyEqual(static_cast<const WKeyEqual&>(o))
1585  , DataPool(static_cast<const DataPool&>(o)) {
1586  ROBIN_HOOD_TRACE(this)
1587  if (!o.empty()) {
1588  // not empty: create an exact copy. it is also possible to just iterate through all
1589  // elements and insert them, but copying is probably faster.
1590 
1591  auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
1592  auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
1593 
1594  ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
1595  << numElementsWithBuffer << ")")
1596  mHashMultiplier = o.mHashMultiplier;
1597  mKeyVals = static_cast<Node*>(
1598  detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
1599  // no need for calloc because clonData does memcpy
1600  mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
1601  mNumElements = o.mNumElements;
1602  mMask = o.mMask;
1603  mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
1604  mInfoInc = o.mInfoInc;
1605  mInfoHashShift = o.mInfoHashShift;
1606  cloneData(o);
1607  }
1608  }
1609 
1610  // Creates a copy of the given map. Copy constructor of each entry is used.
1611  // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
1612  Table& operator=(Table const& o) {
1613  ROBIN_HOOD_TRACE(this)
1614  if (&o == this) {
1615  // prevent assigning of itself
1616  return *this;
1617  }
1618 
1619  // we keep using the old allocator and not assign the new one, because we want to keep
1620  // the memory available. when it is the same size.
1621  if (o.empty()) {
1622  if (0 == mMask) {
1623  // nothing to do, we are empty too
1624  return *this;
1625  }
1626 
1627  // not empty: destroy what we have there
1628  // clear also resets mInfo to 0, that's sometimes not necessary.
1629  destroy();
1630  init();
1631  WHash::operator=(static_cast<const WHash&>(o));
1632  WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
1633  DataPool::operator=(static_cast<DataPool const&>(o));
1634 
1635  return *this;
1636  }
1637 
1638  // clean up old stuff
1639  Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
1640 
1641  if (mMask != o.mMask) {
1642  // no luck: we don't have the same array size allocated, so we need to realloc.
1643  if (0 != mMask) {
1644  // only deallocate if we actually have data!
1645  ROBIN_HOOD_LOG("std::free")
1646  std::free(mKeyVals);
1647  }
1648 
1649  auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
1650  auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
1651  ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
1652  << numElementsWithBuffer << ")")
1653  mKeyVals = static_cast<Node*>(
1654  detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
1655 
1656  // no need for calloc here because cloneData performs a memcpy.
1657  mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
1658  // sentinel is set in cloneData
1659  }
1660  WHash::operator=(static_cast<const WHash&>(o));
1661  WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
1662  DataPool::operator=(static_cast<DataPool const&>(o));
1663  mHashMultiplier = o.mHashMultiplier;
1664  mNumElements = o.mNumElements;
1665  mMask = o.mMask;
1666  mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
1667  mInfoInc = o.mInfoInc;
1668  mInfoHashShift = o.mInfoHashShift;
1669  cloneData(o);
1670 
1671  return *this;
1672  }
1673 
1674  // Swaps everything between the two maps.
1675  void swap(Table& o) {
1676  ROBIN_HOOD_TRACE(this)
1677  using std::swap;
1678  swap(o, *this);
1679  }
1680 
1681  // Clears all data, without resizing.
1682  void clear() {
1683  ROBIN_HOOD_TRACE(this)
1684  if (empty()) {
1685  // don't do anything! also important because we don't want to write to
1686  // DummyInfoByte::b, even though we would just write 0 to it.
1687  return;
1688  }
1689 
1690  Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
1691 
1692  auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
1693  // clear everything, then set the sentinel again
1694  uint8_t const z = 0;
1695  std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
1696  mInfo[numElementsWithBuffer] = 1;
1697 
1698  mInfoInc = InitialInfoInc;
1699  mInfoHashShift = InitialInfoHashShift;
1700  }
1701 
1702  // Destroys the map and all it's contents.
1703  ~Table() {
1704  ROBIN_HOOD_TRACE(this)
1705  destroy();
1706  }
1707 
1708  // Checks if both tables contain the same entries. Order is irrelevant.
1709  bool operator==(const Table& other) const {
1710  ROBIN_HOOD_TRACE(this)
1711  if (other.size() != size()) {
1712  return false;
1713  }
1714  for (auto const& otherEntry : other) {
1715  if (!has(otherEntry)) {
1716  return false;
1717  }
1718  }
1719 
1720  return true;
1721  }
1722 
1723  bool operator!=(const Table& other) const {
1724  ROBIN_HOOD_TRACE(this)
1725  return !operator==(other);
1726  }
1727 
1728  template <typename Q = mapped_type>
1729  typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) {
1730  ROBIN_HOOD_TRACE(this)
1731  auto idxAndState = insertKeyPrepareEmptySpot(key);
1732  switch (idxAndState.second) {
1733  case InsertionState::key_found:
1734  break;
1735 
1736  case InsertionState::new_node:
1737  ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
1738  Node(*this, std::piecewise_construct, std::forward_as_tuple(key),
1739  std::forward_as_tuple());
1740  break;
1741 
1742  case InsertionState::overwrite_node:
1743  mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
1744  std::forward_as_tuple(key), std::forward_as_tuple());
1745  break;
1746 
1747  case InsertionState::overflow_error:
1748  throwOverflowError();
1749  }
1750 
1751  return mKeyVals[idxAndState.first].getSecond();
1752  }
1753 
1754  template <typename Q = mapped_type>
1755  typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) {
1756  ROBIN_HOOD_TRACE(this)
1757  auto idxAndState = insertKeyPrepareEmptySpot(key);
1758  switch (idxAndState.second) {
1759  case InsertionState::key_found:
1760  break;
1761 
1762  case InsertionState::new_node:
1763  ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
1764  Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
1765  std::forward_as_tuple());
1766  break;
1767 
1768  case InsertionState::overwrite_node:
1769  mKeyVals[idxAndState.first] =
1770  Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
1771  std::forward_as_tuple());
1772  break;
1773 
1774  case InsertionState::overflow_error:
1775  throwOverflowError();
1776  }
1777 
1778  return mKeyVals[idxAndState.first].getSecond();
1779  }
1780 
1781  template <typename Iter>
1782  void insert(Iter first, Iter last) {
1783  for (; first != last; ++first) {
1784  // value_type ctor needed because this might be called with std::pair's
1785  insert(value_type(*first));
1786  }
1787  }
1788 
1789  void insert(std::initializer_list<value_type> ilist) {
1790  for (auto&& vt : ilist) {
1791  insert(std::move(vt));
1792  }
1793  }
1794 
1795  template <typename... Args>
1796  std::pair<iterator, bool> emplace(Args&&... args) {
1797  ROBIN_HOOD_TRACE(this)
1798  Node n{*this, std::forward<Args>(args)...};
1799  auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n));
1800  switch (idxAndState.second) {
1801  case InsertionState::key_found:
1802  n.destroy(*this);
1803  break;
1804 
1805  case InsertionState::new_node:
1806  ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(*this, std::move(n));
1807  break;
1808 
1809  case InsertionState::overwrite_node:
1810  mKeyVals[idxAndState.first] = std::move(n);
1811  break;
1812 
1813  case InsertionState::overflow_error:
1814  n.destroy(*this);
1815  throwOverflowError();
1816  break;
1817  }
1818 
1819  return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
1820  InsertionState::key_found != idxAndState.second);
1821  }
1822 
1823  template <typename... Args>
1824  iterator emplace_hint(const_iterator position, Args&&... args) {
1825  (void)position;
1826  return emplace(std::forward<Args>(args)...).first;
1827  }
1828 
1829  template <typename... Args>
1830  std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
1831  return try_emplace_impl(key, std::forward<Args>(args)...);
1832  }
1833 
1834  template <typename... Args>
1835  std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
1836  return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
1837  }
1838 
1839  template <typename... Args>
1840  iterator try_emplace(const_iterator hint, const key_type& key, Args&&... args) {
1841  (void)hint;
1842  return try_emplace_impl(key, std::forward<Args>(args)...).first;
1843  }
1844 
1845  template <typename... Args>
1846  iterator try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
1847  (void)hint;
1848  return try_emplace_impl(std::move(key), std::forward<Args>(args)...).first;
1849  }
1850 
1851  template <typename Mapped>
1852  std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) {
1853  return insertOrAssignImpl(key, std::forward<Mapped>(obj));
1854  }
1855 
1856  template <typename Mapped>
1857  std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) {
1858  return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj));
1859  }
1860 
1861  template <typename Mapped>
1862  iterator insert_or_assign(const_iterator hint, const key_type& key, Mapped&& obj) {
1863  (void)hint;
1864  return insertOrAssignImpl(key, std::forward<Mapped>(obj)).first;
1865  }
1866 
1867  template <typename Mapped>
1868  iterator insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) {
1869  (void)hint;
1870  return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj)).first;
1871  }
1872 
1873  std::pair<iterator, bool> insert(const value_type& keyval) {
1874  ROBIN_HOOD_TRACE(this)
1875  return emplace(keyval);
1876  }
1877 
1878  iterator insert(const_iterator hint, const value_type& keyval) {
1879  (void)hint;
1880  return emplace(keyval).first;
1881  }
1882 
1883  std::pair<iterator, bool> insert(value_type&& keyval) {
1884  return emplace(std::move(keyval));
1885  }
1886 
1887  iterator insert(const_iterator hint, value_type&& keyval) {
1888  (void)hint;
1889  return emplace(std::move(keyval)).first;
1890  }
1891 
1892  // Returns 1 if key is found, 0 otherwise.
1893  size_t count(const key_type& key) const { // NOLINT
1894  ROBIN_HOOD_TRACE(this)
1895  auto kv = mKeyVals + findIdx(key);
1896  if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1897  return 1;
1898  }
1899  return 0;
1900  }
1901 
1902  template <typename OtherKey, typename Self_ = Self>
1903  typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const {
1904  ROBIN_HOOD_TRACE(this)
1905  auto kv = mKeyVals + findIdx(key);
1906  if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1907  return 1;
1908  }
1909  return 0;
1910  }
1911 
1912  bool contains(const key_type& key) const { // NOLINT
1913  return 1U == count(key);
1914  }
1915 
1916  template <typename OtherKey, typename Self_ = Self>
1917  typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const {
1918  return 1U == count(key);
1919  }
1920 
1921  // Returns a reference to the value found for key.
1922  // Throws std::out_of_range if element cannot be found
1923  template <typename Q = mapped_type>
1924  typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) {
1925  ROBIN_HOOD_TRACE(this)
1926  auto kv = mKeyVals + findIdx(key);
1927  if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1928  doThrow<std::out_of_range>("key not found");
1929  }
1930  return kv->getSecond();
1931  }
1932 
1933  // Returns a reference to the value found for key.
1934  // Throws std::out_of_range if element cannot be found
1935  template <typename Q = mapped_type>
1936  typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const {
1937  ROBIN_HOOD_TRACE(this)
1938  auto kv = mKeyVals + findIdx(key);
1939  if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1940  doThrow<std::out_of_range>("key not found");
1941  }
1942  return kv->getSecond();
1943  }
1944 
1945  const_iterator find(const key_type& key) const { // NOLINT
1946  ROBIN_HOOD_TRACE(this)
1947  const size_t idx = findIdx(key);
1948  return const_iterator{mKeyVals + idx, mInfo + idx};
1949  }
1950 
1951  template <typename OtherKey>
1952  const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const {
1953  ROBIN_HOOD_TRACE(this)
1954  const size_t idx = findIdx(key);
1955  return const_iterator{mKeyVals + idx, mInfo + idx};
1956  }
1957 
1958  template <typename OtherKey, typename Self_ = Self>
1959  typename std::enable_if<Self_::is_transparent, // NOLINT
1960  const_iterator>::type // NOLINT
1961  find(const OtherKey& key) const { // NOLINT
1962  ROBIN_HOOD_TRACE(this)
1963  const size_t idx = findIdx(key);
1964  return const_iterator{mKeyVals + idx, mInfo + idx};
1965  }
1966 
1967  iterator find(const key_type& key) {
1968  ROBIN_HOOD_TRACE(this)
1969  const size_t idx = findIdx(key);
1970  return iterator{mKeyVals + idx, mInfo + idx};
1971  }
1972 
1973  template <typename OtherKey>
1974  iterator find(const OtherKey& key, is_transparent_tag/*unused*/) {
1975  ROBIN_HOOD_TRACE(this)
1976  const size_t idx = findIdx(key);
1977  return iterator{mKeyVals + idx, mInfo + idx};
1978  }
1979 
1980  template <typename OtherKey, typename Self_ = Self>
1981  typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) {
1982  ROBIN_HOOD_TRACE(this)
1983  const size_t idx = findIdx(key);
1984  return iterator{mKeyVals + idx, mInfo + idx};
1985  }
1986 
1987  iterator begin() {
1988  ROBIN_HOOD_TRACE(this)
1989  if (empty()) {
1990  return end();
1991  }
1992  return iterator(mKeyVals, mInfo, fast_forward_tag{});
1993  }
1994  const_iterator begin() const { // NOLINT
1995  ROBIN_HOOD_TRACE(this)
1996  return cbegin();
1997  }
1998  const_iterator cbegin() const { // NOLINT
1999  ROBIN_HOOD_TRACE(this)
2000  if (empty()) {
2001  return cend();
2002  }
2003  return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
2004  }
2005 
2006  iterator end() {
2007  ROBIN_HOOD_TRACE(this)
2008  // no need to supply valid info pointer: end() must not be dereferenced, and only node
2009  // pointer is compared.
2010  return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
2011  }
2012  const_iterator end() const { // NOLINT
2013  ROBIN_HOOD_TRACE(this)
2014  return cend();
2015  }
2016  const_iterator cend() const { // NOLINT
2017  ROBIN_HOOD_TRACE(this)
2018  return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
2019  }
2020 
2021  iterator erase(const_iterator pos) {
2022  ROBIN_HOOD_TRACE(this)
2023  // its safe to perform const cast here
2024  return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)});
2025  }
2026 
2027  // Erases element at pos, returns iterator to the next element.
2028  iterator erase(iterator pos) {
2029  ROBIN_HOOD_TRACE(this)
2030  // we assume that pos always points to a valid entry, and not end().
2031  auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);
2032 
2033  shiftDown(idx);
2034  --mNumElements;
2035 
2036  if (*pos.mInfo) {
2037  // we've backward shifted, return this again
2038  return pos;
2039  }
2040 
2041  // no backward shift, return next element
2042  return ++pos;
2043  }
2044 
2045  size_t erase(const key_type& key) {
2046  ROBIN_HOOD_TRACE(this)
2047  size_t idx{};
2048  InfoType info{};
2049  keyToIdx(key, &idx, &info);
2050 
2051  // check while info matches with the source idx
2052  do {
2053  if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
2054  shiftDown(idx);
2055  --mNumElements;
2056  return 1;
2057  }
2058  next(&info, &idx);
2059  } while (info <= mInfo[idx]);
2060 
2061  // nothing found to delete
2062  return 0;
2063  }
2064 
2065  // reserves space for the specified number of elements. Makes sure the old data fits.
2066  // exactly the same as reserve(c).
2067  void rehash(size_t c) {
2068  // forces a reserve
2069  reserve(c, true);
2070  }
2071 
2072  // reserves space for the specified number of elements. Makes sure the old data fits.
2073  // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
2074  void reserve(size_t c) {
2075  // reserve, but don't force rehash
2076  reserve(c, false);
2077  }
2078 
2079  // If possible reallocates the map to a smaller one. This frees the underlying table.
2080  // Does not do anything if load_factor is too large for decreasing the table's size.
2081  void compact() {
2082  ROBIN_HOOD_TRACE(this)
2083  auto newSize = InitialNumElements;
2084  while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
2085  newSize *= 2;
2086  }
2087  if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
2088  throwOverflowError();
2089  }
2090 
2091  ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
2092 
2093  // only actually do anything when the new size is bigger than the old one. This prevents to
2094  // continuously allocate for each reserve() call.
2095  if (newSize < mMask + 1) {
2096  rehashPowerOfTwo(newSize, true);
2097  }
2098  }
2099 
2100  size_type size() const noexcept { // NOLINT
2101  ROBIN_HOOD_TRACE(this)
2102  return mNumElements;
2103  }
2104 
2105  size_type max_size() const noexcept { // NOLINT
2106  ROBIN_HOOD_TRACE(this)
2107  return static_cast<size_type>(-1);
2108  }
2109 
2110  ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
2111  ROBIN_HOOD_TRACE(this)
2112  return 0 == mNumElements;
2113  }
2114 
2115  float max_load_factor() const noexcept { // NOLINT
2116  ROBIN_HOOD_TRACE(this)
2117  return MaxLoadFactor100 / 100.0F;
2118  }
2119 
2120  // Average number of elements per bucket. Since we allow only 1 per bucket
2121  float load_factor() const noexcept { // NOLINT
2122  ROBIN_HOOD_TRACE(this)
2123  return static_cast<float>(size()) / static_cast<float>(mMask + 1);
2124  }
2125 
2126  ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
2127  ROBIN_HOOD_TRACE(this)
2128  return mMask;
2129  }
2130 
2131  ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
2132  if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
2133  return maxElements * MaxLoadFactor100 / 100;
2134  }
2135 
2136  // we might be a bit imprecise, but since maxElements is quite large that doesn't matter
2137  return (maxElements / 100) * MaxLoadFactor100;
2138  }
2139 
2140  ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
2141  // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
2142  // 64bit types.
2143  return numElements + sizeof(uint64_t);
2144  }
2145 
2146  ROBIN_HOOD(NODISCARD)
2147  size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
2148  auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
2149  return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
2150  }
2151 
2152  // calculation only allowed for 2^n values
2153  ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
2154 #if ROBIN_HOOD(BITNESS) == 64
2155  return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
2156 #else
2157  // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
2158  auto const ne = static_cast<uint64_t>(numElements);
2159  auto const s = static_cast<uint64_t>(sizeof(Node));
2160  auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));
2161 
2162  auto const total64 = ne * s + infos;
2163  auto const total = static_cast<size_t>(total64);
2164 
2165  if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
2166  throwOverflowError();
2167  }
2168  return total;
2169 #endif
2170  }
2171 
2172 private:
2173  template <typename Q = mapped_type>
2174  ROBIN_HOOD(NODISCARD)
2175  typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const {
2176  ROBIN_HOOD_TRACE(this)
2177  auto it = find(e.first);
2178  return it != end() && it->second == e.second;
2179  }
2180 
2181  template <typename Q = mapped_type>
2182  ROBIN_HOOD(NODISCARD)
2183  typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const {
2184  ROBIN_HOOD_TRACE(this)
2185  return find(e) != end();
2186  }
2187 
2188  void reserve(size_t c, bool forceRehash) {
2189  ROBIN_HOOD_TRACE(this)
2190  auto const minElementsAllowed = (std::max)(c, mNumElements);
2191  auto newSize = InitialNumElements;
2192  while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
2193  newSize *= 2;
2194  }
2195  if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
2196  throwOverflowError();
2197  }
2198 
2199  ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
2200 
2201  // only actually do anything when the new size is bigger than the old one. This prevents to
2202  // continuously allocate for each reserve() call.
2203  if (forceRehash || newSize > mMask + 1) {
2204  rehashPowerOfTwo(newSize, false);
2205  }
2206  }
2207 
2208  // reserves space for at least the specified number of elements.
2209  // only works if numBuckets if power of two
2210  // True on success, false otherwise
2211  void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
2212  ROBIN_HOOD_TRACE(this)
2213 
2214  Node* const oldKeyVals = mKeyVals;
2215  uint8_t const* const oldInfo = mInfo;
2216 
2217  const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
2218 
2219  // resize operation: move stuff
2220  initData(numBuckets);
2221  if (oldMaxElementsWithBuffer > 1) {
2222  for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
2223  if (oldInfo[i] != 0) {
2224  // might throw an exception, which is really bad since we are in the middle of
2225  // moving stuff.
2226  insert_move(std::move(oldKeyVals[i]));
2227  // destroy the node but DON'T destroy the data.
2228  oldKeyVals[i].~Node();
2229  }
2230  }
2231 
2232  // this check is not necessary as it's guarded by the previous if, but it helps
2233  // silence g++'s overeager "attempt to free a non-heap object 'map'
2234  // [-Werror=free-nonheap-object]" warning.
2235  if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
2236  // don't destroy old data: put it into the pool instead
2237  if (forceFree) {
2238  std::free(oldKeyVals);
2239  } else {
2240  DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
2241  }
2242  }
2243  }
2244  }
2245 
2246  ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
2247 #if ROBIN_HOOD(HAS_EXCEPTIONS)
2248  throw std::overflow_error("robin_hood::map overflow");
2249 #else
2250  abort();
2251 #endif
2252  }
2253 
2254  template <typename OtherKey, typename... Args>
2255  std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) {
2256  ROBIN_HOOD_TRACE(this)
2257  auto idxAndState = insertKeyPrepareEmptySpot(key);
2258  switch (idxAndState.second) {
2259  case InsertionState::key_found:
2260  break;
2261 
2262  case InsertionState::new_node:
2263  ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
2264  *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
2265  std::forward_as_tuple(std::forward<Args>(args)...));
2266  break;
2267 
2268  case InsertionState::overwrite_node:
2269  mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
2270  std::forward_as_tuple(std::forward<OtherKey>(key)),
2271  std::forward_as_tuple(std::forward<Args>(args)...));
2272  break;
2273 
2274  case InsertionState::overflow_error:
2275  throwOverflowError();
2276  break;
2277  }
2278 
2279  return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
2280  InsertionState::key_found != idxAndState.second);
2281  }
2282 
2283  template <typename OtherKey, typename Mapped>
2284  std::pair<iterator, bool> insertOrAssignImpl(OtherKey&& key, Mapped&& obj) {
2285  ROBIN_HOOD_TRACE(this)
2286  auto idxAndState = insertKeyPrepareEmptySpot(key);
2287  switch (idxAndState.second) {
2288  case InsertionState::key_found:
2289  mKeyVals[idxAndState.first].getSecond() = std::forward<Mapped>(obj);
2290  break;
2291 
2292  case InsertionState::new_node:
2293  ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
2294  *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
2295  std::forward_as_tuple(std::forward<Mapped>(obj)));
2296  break;
2297 
2298  case InsertionState::overwrite_node:
2299  mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
2300  std::forward_as_tuple(std::forward<OtherKey>(key)),
2301  std::forward_as_tuple(std::forward<Mapped>(obj)));
2302  break;
2303 
2304  case InsertionState::overflow_error:
2305  throwOverflowError();
2306  break;
2307  }
2308 
2309  return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
2310  InsertionState::key_found != idxAndState.second);
2311  }
2312 
2313  void initData(size_t max_elements) {
2314  mNumElements = 0;
2315  mMask = max_elements - 1;
2316  mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);
2317 
2318  auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);
2319 
2320  // malloc & zero mInfo. Faster than calloc everything.
2321  auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
2322  ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal("
2323  << numElementsWithBuffer << ")")
2324  mKeyVals = reinterpret_cast<Node*>(
2325  detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
2326  mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
2327  std::memset(mInfo, 0, numBytesTotal - numElementsWithBuffer * sizeof(Node));
2328 
2329  // set sentinel
2330  mInfo[numElementsWithBuffer] = 1;
2331 
2332  mInfoInc = InitialInfoInc;
2333  mInfoHashShift = InitialInfoHashShift;
2334  }
2335 
2336  enum class InsertionState { overflow_error, key_found, new_node, overwrite_node };
2337 
2338  // Finds key, and if not already present prepares a spot where to pot the key & value.
2339  // This potentially shifts nodes out of the way, updates mInfo and number of inserted
2340  // elements, so the only operation left to do is create/assign a new node at that spot.
2341  template <typename OtherKey>
2342  std::pair<size_t, InsertionState> insertKeyPrepareEmptySpot(OtherKey&& key) {
2343  for (int i = 0; i < 256; ++i) {
2344  size_t idx{};
2345  InfoType info{};
2346  keyToIdx(key, &idx, &info);
2347  nextWhileLess(&info, &idx);
2348 
2349  // while we potentially have a match
2350  while (info == mInfo[idx]) {
2351  if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
2352  // key already exists, do NOT insert.
2353  // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
2354  return std::make_pair(idx, InsertionState::key_found);
2355  }
2356  next(&info, &idx);
2357  }
2358 
2359  // unlikely that this evaluates to true
2360  if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
2361  if (!increase_size()) {
2362  return std::make_pair(size_t(0), InsertionState::overflow_error);
2363  }
2364  continue;
2365  }
2366 
2367  // key not found, so we are now exactly where we want to insert it.
2368  auto const insertion_idx = idx;
2369  auto const insertion_info = info;
2370  if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
2371  mMaxNumElementsAllowed = 0;
2372  }
2373 
2374  // find an empty spot
2375  while (0 != mInfo[idx]) {
2376  next(&info, &idx);
2377  }
2378 
2379  if (idx != insertion_idx) {
2380  shiftUp(idx, insertion_idx);
2381  }
2382  // put at empty spot
2383  mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
2384  ++mNumElements;
2385  return std::make_pair(insertion_idx, idx == insertion_idx
2386  ? InsertionState::new_node
2387  : InsertionState::overwrite_node);
2388  }
2389 
2390  // enough attempts failed, so finally give up.
2391  return std::make_pair(size_t(0), InsertionState::overflow_error);
2392  }
2393 
2394  bool try_increase_info() {
2395  ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
2396  << ", maxNumElementsAllowed="
2397  << calcMaxNumElementsAllowed(mMask + 1))
2398  if (mInfoInc <= 2) {
2399  // need to be > 2 so that shift works (otherwise undefined behavior!)
2400  return false;
2401  }
2402  // we got space left, try to make info smaller
2403  mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);
2404 
2405  // remove one bit of the hash, leaving more space for the distance info.
2406  // This is extremely fast because we can operate on 8 bytes at once.
2407  ++mInfoHashShift;
2408  auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
2409 
2410  for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
2411  auto val = unaligned_load<uint64_t>(mInfo + i);
2412  val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
2413  std::memcpy(mInfo + i, &val, sizeof(val));
2414  }
2415  // update sentinel, which might have been cleared out!
2416  mInfo[numElementsWithBuffer] = 1;
2417 
2418  mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
2419  return true;
2420  }
2421 
2422  // True if resize was possible, false otherwise
2423  bool increase_size() {
2424  // nothing allocated yet? just allocate InitialNumElements
2425  if (0 == mMask) {
2426  initData(InitialNumElements);
2427  return true;
2428  }
2429 
2430  auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
2431  if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
2432  return true;
2433  }
2434 
2435  ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed="
2436  << maxNumElementsAllowed << ", load="
2437  << (static_cast<double>(mNumElements) * 100.0 /
2438  (static_cast<double>(mMask) + 1)))
2439 
2440  if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
2441  // we have to resize, even though there would still be plenty of space left!
2442  // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case
2443  // we have to rehash a few times
2444  nextHashMultiplier();
2445  rehashPowerOfTwo(mMask + 1, true);
2446  } else {
2447  // we've reached the capacity of the map, so the hash seems to work nice. Keep using it.
2448  rehashPowerOfTwo((mMask + 1) * 2, false);
2449  }
2450  return true;
2451  }
2452 
2453  void nextHashMultiplier() {
2454  // adding an *even* number, so that the multiplier will always stay odd. This is necessary
2455  // so that the hash stays a mixing function (and thus doesn't have any information loss).
2456  mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54);
2457  }
2458 
2459  void destroy() {
2460  if (0 == mMask) {
2461  // don't deallocate!
2462  return;
2463  }
2464 
2465  Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}
2466  .nodesDoNotDeallocate(*this);
2467 
2468  // This protection against not deleting mMask shouldn't be needed as it's sufficiently
2469  // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
2470  // reports a compile error: attempt to free a non-heap object 'fm'
2471  // [-Werror=free-nonheap-object]
2472  if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
2473  ROBIN_HOOD_LOG("std::free")
2474  std::free(mKeyVals);
2475  }
2476  }
2477 
2478  void init() noexcept {
2479  mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask);
2480  mInfo = reinterpret_cast<uint8_t*>(&mMask);
2481  mNumElements = 0;
2482  mMask = 0;
2483  mMaxNumElementsAllowed = 0;
2484  mInfoInc = InitialInfoInc;
2485  mInfoHashShift = InitialInfoHashShift;
2486  }
2487 
2488  // members are sorted so no padding occurs
2489  uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8
2490  Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 16
2491  uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 24
2492  size_t mNumElements = 0; // 8 byte 32
2493  size_t mMask = 0; // 8 byte 40
2494  size_t mMaxNumElementsAllowed = 0; // 8 byte 48
2495  InfoType mInfoInc = InitialInfoInc; // 4 byte 52
2496  InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56
2497  // 16 byte 56 if NodeAllocator
2498 };
2499 
2500 } // namespace detail
2501 
2502 // map
2503 
2504 template <typename Key, typename T, typename Hash = hash<Key>,
2505  typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2507 
2508 template <typename Key, typename T, typename Hash = hash<Key>,
2509  typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2511 
2512 template <typename Key, typename T, typename Hash = hash<Key>,
2513  typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2514 using unordered_map =
2515  detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
2516  std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
2517  std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
2518  MaxLoadFactor100, Key, T, Hash, KeyEqual>;
2519 
2520 // set
2521 
2522 template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2523  size_t MaxLoadFactor100 = 80>
2525 
2526 template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2527  size_t MaxLoadFactor100 = 80>
2529 
2530 template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2531  size_t MaxLoadFactor100 = 80>
2532 using unordered_set = detail::Table < sizeof(Key) <= sizeof(size_t) * 6 &&
2533  std::is_nothrow_move_constructible<Key>::value &&
2534  std::is_nothrow_move_assignable<Key>::value,
2535  MaxLoadFactor100, Key, void, Hash, KeyEqual>;
2536 
2537 } // namespace robin_hood
2538 /* *INDENT-ON* */
2539 
2540 #endif // NAV2_SMAC_PLANNER__THIRDPARTY__ROBIN_HOOD_H_