BitMagic-C++
bmfunc.h
Go to the documentation of this file.
1#ifndef BMFUNC__H__INCLUDED__
2#define BMFUNC__H__INCLUDED__
3/*
4Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com)
5
6Licensed under the Apache License, Version 2.0 (the "License");
7you may not use this file except in compliance with the License.
8You may obtain a copy of the License at
9
10 http://www.apache.org/licenses/LICENSE-2.0
11
12Unless required by applicable law or agreed to in writing, software
13distributed under the License is distributed on an "AS IS" BASIS,
14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15See the License for the specific language governing permissions and
16limitations under the License.
17
18For more information please visit: http://bitmagic.io
19*/
20
21/*! \file bmfunc.h
22 \brief Bit manipulation primitives (internal)
23*/
24
25#include <memory.h>
26
27#include "bmdef.h"
28#include "bmutil.h"
29
30
31#ifdef _MSC_VER
32# pragma warning( disable: 4146 )
33#endif
34
35namespace bm
36{
37
38
39inline
41 bm::word_t left,
43
44inline
46 bm::word_t left,
48
49/*!
50 @brief Structure with statistical information about memory
51 allocation footprint, serialization projection, number of vectors
52 @ingroup bvector
53*/
55{
56 size_t bit_blocks; ///< Number of bit blocks
57 size_t gap_blocks; ///< Number of GAP blocks
58 size_t ptr_sub_blocks; ///< Number of sub-blocks
59 size_t bv_count; ///< Number of bit-vectors
60 size_t max_serialize_mem; ///< estimated maximum memory for serialization
61 size_t memory_used; ///< memory usage for all blocks and service tables
62 size_t gap_cap_overhead; ///< gap memory overhead between length and capacity
63 gap_word_t gap_levels[bm::gap_levels]; ///< GAP block lengths in the bvect
64 unsigned long long gaps_by_level[bm::gap_levels]; ///< number of GAP blocks at each level
65
66 /// cound bit block
68 {
69 ++bit_blocks;
70 size_t mem_used = sizeof(bm::word_t) * bm::set_block_size;
71 memory_used += mem_used;
72 max_serialize_mem += mem_used;
73 }
74
75 /// count gap block
76 void add_gap_block(unsigned capacity, unsigned length) BMNOEXCEPT
77 {
78 ++gap_blocks;
79 size_t mem_used = (capacity * sizeof(gap_word_t));
80 memory_used += mem_used;
81 max_serialize_mem += (unsigned)(length * sizeof(gap_word_t));
82 BM_ASSERT(length <= capacity);
83 gap_cap_overhead += (capacity - length) * sizeof(gap_word_t);
84 for (unsigned i = 0; i < bm::gap_levels; ++i)
85 {
86 if (capacity == gap_levels[i])
87 {
88 gaps_by_level[i]++;
89 return;
90 }
91 }
92 BM_ASSERT(0);
93 }
94
95 /// Reset statisctics
97 {
100 for (unsigned i = 0; i < bm::gap_levels; ++i)
101 gaps_by_level[i] = 0ull;
102 }
103
104 /// Sum data from another sttructure
106 {
107 bit_blocks += st.bit_blocks;
108 gap_blocks += st.gap_blocks;
109 ptr_sub_blocks += st.ptr_sub_blocks;
110 bv_count += st.bv_count;
111 max_serialize_mem += st.max_serialize_mem + 8;
112 memory_used += st.memory_used;
113 gap_cap_overhead += st.gap_cap_overhead;
114 }
115};
116
117/**
118 @brief Pair type
119*/
120template<typename First, typename Second>
121struct pair
122{
123 First first;
124 Second second;
125
126 pair(First f, Second s) : first(f), second(s) {}
127};
128
129/**
130 \brief bit-decode cache structure
131*/
133{
134 unsigned short bits[65]; //< decoded bits
135 unsigned bcnt; //< length of bits array
136 bm::id64_t cvalue; //< cache decoded value
137
139};
140
141
142/**
143 \brief Recalc linear bvector block index into 2D matrix coordinates
144 \internal
145*/
146template<typename BI_TYPE>
147void get_block_coord(BI_TYPE nb, unsigned& i, unsigned& j) BMNOEXCEPT
148{
149 i = unsigned(nb >> bm::set_array_shift); // top block address
150 j = unsigned(nb & bm::set_array_mask); // address in sub-block
151}
152
153/**
154 Compute bit address of the first bit in a superblock
155 \internal
156*/
157template<typename RTYPE>
159{
160 return RTYPE(i) * bm::set_sub_total_bits;
161}
162
163/**
164 Compute bit address of the first bit in a block
165 \internal
166*/
167template<typename RTYPE>
168BMFORCEINLINE RTYPE get_block_start(unsigned i, unsigned j) BMNOEXCEPT
169{
171 idx += (j) * bm::gap_max_bits;
172 return idx;
173}
174
175/*!
176 @defgroup gapfunc GAP functions
177 GAP functions implement different opereations on GAP compressed blocks (internals)
178 and serve as a minimal building blocks.
179 \internal
180 @ingroup bvector
181 */
182
183/*!
184 @defgroup bitfunc BIT functions
185 Bit functions implement different opereations on bit blocks (internals)
186 and serve as a minimal building blocks.
187 \internal
188 @ingroup bvector
189 */
190
191
192/*!
193 Returns bit count
194 @ingroup bitfunc
195*/
198{
199#if defined(BMSSE42OPT) || defined(BMAVX2OPT)
200 return bm::id_t(_mm_popcnt_u32(w));
201#else
202 #if defined(BM_USE_GCC_BUILD)
203 return (bm::id_t)__builtin_popcount(w);
204 #else
205 return
206 bm::bit_count_table<true>::_count[(unsigned char)(w)] +
207 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 8)] +
208 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 16)] +
209 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 24)];
210 #endif
211#endif
212}
213
214inline
216{
217 unsigned int tmp;
218
219 tmp = n - ((n >> 1) & 033333333333)
220 - ((n >> 2) & 011111111111);
221 return ((tmp + (tmp >> 3)) & 030707070707) % 63;
222}
223
224
225/*!
226 Function calculates number of 1 bits in 64-bit word.
227 @ingroup bitfunc
228*/
231{
232#if defined(BMSSE42OPT) || defined(BMAVX2OPT)
233#if defined(BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
234 return unsigned(_mm_popcnt_u64(x));
235#else
236 return _mm_popcnt_u32(x >> 32) + _mm_popcnt_u32((unsigned)x);
237#endif
238#else
239 #if defined(BM_USE_GCC_BUILD)
240 return (unsigned)__builtin_popcountll(x);
241 #else
242 x = x - ((x >> 1) & 0x5555555555555555);
243 x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
244 x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0F;
245 x = x + (x >> 8);
246 x = x + (x >> 16);
247 x = x + (x >> 32);
248 return x & 0xFF;
249 #endif
250#endif
251}
252
253inline
256{
257 const bm::id64_t m1 = 0x5555555555555555U;
258 const bm::id64_t m2 = 0x3333333333333333U;
259 const bm::id64_t m3 = 0x0F0F0F0F0F0F0F0FU;
260 const bm::id64_t m4 = 0x000000FF000000FFU;
261
262 x = x - ((x >> 1) & m1);
263 y = y - ((y >> 1) & m1);
264 u = u - ((u >> 1) & m1);
265 v = v - ((v >> 1) & m1);
266 x = (x & m2) + ((x >> 2) & m2);
267 y = (y & m2) + ((y >> 2) & m2);
268 u = (u & m2) + ((u >> 2) & m2);
269 v = (v & m2) + ((v >> 2) & m2);
270 x = x + y;
271 u = u + v;
272 x = (x & m3) + ((x >> 4) & m3);
273 u = (u & m3) + ((u >> 4) & m3);
274 x = x + u;
275 x = x + (x >> 8);
276 x = x + (x >> 16);
277 x = x & m4;
278 x = x + (x >> 32);
279 return x & 0x000001FFU;
280}
281
282
283// --------------------------------------------------------------
284// Functions for bit-scanning
285// --------------------------------------------------------------
286
287/*!
288 \brief Templated algorithm to unpacks octet based word into list of ON bit indexes
289 \param w - value
290 \param func - bit functor
291
292 @ingroup bitfunc
293*/
294template<typename T, typename F>
295void bit_for_each_4(T w, F& func)
296{
297 for (unsigned sub_octet = 0; w != 0; w >>= 4, sub_octet += 4)
298 {
299 switch (w & 15) // 1111
300 {
301 case 0: // 0000
302 break;
303 case 1: // 0001
304 func(sub_octet);
305 break;
306 case 2: // 0010
307 func(sub_octet + 1);
308 break;
309 case 3: // 0011
310 func(sub_octet, sub_octet + 1);
311 break;
312 case 4: // 0100
313 func(sub_octet + 2);
314 break;
315 case 5: // 0101
316 func(sub_octet, sub_octet + 2);
317 break;
318 case 6: // 0110
319 func(sub_octet + 1, sub_octet + 2);
320 break;
321 case 7: // 0111
322 func(sub_octet, sub_octet + 1, sub_octet + 2);
323 break;
324 case 8: // 1000
325 func(sub_octet + 3);
326 break;
327 case 9: // 1001
328 func(sub_octet, sub_octet + 3);
329 break;
330 case 10: // 1010
331 func(sub_octet + 1, sub_octet + 3);
332 break;
333 case 11: // 1011
334 func(sub_octet, sub_octet + 1, sub_octet + 3);
335 break;
336 case 12: // 1100
337 func(sub_octet + 2, sub_octet + 3);
338 break;
339 case 13: // 1101
340 func(sub_octet, sub_octet + 2, sub_octet + 3);
341 break;
342 case 14: // 1110
343 func(sub_octet + 1, sub_octet + 2, sub_octet + 3);
344 break;
345 case 15: // 1111
346 func(sub_octet, sub_octet + 1, sub_octet + 2, sub_octet + 3);
347 break;
348 default:
349 BM_ASSERT(0);
350 break;
351 }
352
353 } // for
354}
355
356
357/*!
358 \brief Templated algorithm to unpacks word into list of ON bit indexes
359 \param w - value
360 \param func - bit functor
361
362 @ingroup bitfunc
363*/
364template<typename T, typename F>
365void bit_for_each(T w, F& func)
366{
367 // Note: 4-bit table method works slower than plain check approach
368 for (unsigned octet = 0; w != 0; w >>= 8, octet += 8)
369 {
370 if (w & 1) func(octet + 0);
371 if (w & 2) func(octet + 1);
372 if (w & 4) func(octet + 2);
373 if (w & 8) func(octet + 3);
374 if (w & 16) func(octet + 4);
375 if (w & 32) func(octet + 5);
376 if (w & 64) func(octet + 6);
377 if (w & 128) func(octet + 7);
378
379 } // for
380}
381
382/*! @brief Adaptor to copy 1 bits to array
383 @internal
384*/
385template<typename B>
387{
388public:
389 copy_to_array_functor(B* bits): bp_(bits)
390 {}
391
392 B* ptr() { return bp_; }
393
394 void operator()(unsigned bit_idx) BMNOEXCEPT { *bp_++ = (B)bit_idx; }
395
396 void operator()(unsigned bit_idx0,
397 unsigned bit_idx1) BMNOEXCEPT
398 {
399 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
400 bp_+=2;
401 }
402
403 void operator()(unsigned bit_idx0,
404 unsigned bit_idx1,
405 unsigned bit_idx2) BMNOEXCEPT
406 {
407 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1; bp_[2] = (B)bit_idx2;
408 bp_+=3;
409 }
410
411 void operator()(unsigned bit_idx0,
412 unsigned bit_idx1,
413 unsigned bit_idx2,
414 unsigned bit_idx3) BMNOEXCEPT
415 {
416 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
417 bp_[2] = (B)bit_idx2; bp_[3] = (B)bit_idx3;
418 bp_+=4;
419 }
420
421private:
424private:
425 B* bp_;
426};
427
428
429/*!
430 \brief Unpacks word into list of ON bit indexes
431 \param w - value
432 \param bits - pointer on the result array
433 \return number of bits in the list
434
435 @ingroup bitfunc
436*/
437template<typename T,typename B>
438unsigned bit_list(T w, B* bits) BMNOEXCEPT
439{
440 copy_to_array_functor<B> func(bits);
441 bit_for_each(w, func);
442 return (unsigned)(func.ptr() - bits);
443}
444
445
446
447/*!
448 \brief Unpacks word into list of ON bit indexes (quad-bit based)
449 \param w - value
450 \param bits - pointer on the result array
451 \return number of bits in the list
452
453 @ingroup bitfunc
454*/
455template<typename T,typename B>
456unsigned bit_list_4(T w, B* bits) BMNOEXCEPT
457{
458 copy_to_array_functor<B> func(bits);
459 bit_for_each_4(w, func);
460 return (unsigned)(func.ptr() - bits);
461}
462
463/*!
464 \brief Unpacks word into list of ON bit indexes using popcnt method
465 \param w - value
466 \param bits - pointer on the result array
467 \param offs - value to add to bit position (programmed shift)
468 \return number of bits in the list
469
470 @ingroup bitfunc
471 @internal
472*/
473template<typename B>
474unsigned short
475bitscan_popcnt(bm::id_t w, B* bits, unsigned short offs) BMNOEXCEPT
476{
477 unsigned pos = 0;
478 while (w)
479 {
480 bm::id_t t = w & -w;
481 bits[pos++] = (B)(bm::word_bitcount(t - 1) + offs);
482 w &= w - 1;
483 }
484 return (unsigned short)pos;
485}
486
487/*!
488 \brief Unpacks word into list of ON bit indexes using popcnt method
489 \param w - value
490 \param bits - pointer on the result array
491 \return number of bits in the list
492
493 @ingroup bitfunc
494 @internal
495*/
496template<typename B>
497unsigned short bitscan_popcnt(bm::id_t w, B* bits) BMNOEXCEPT
498{
499 unsigned pos = 0;
500 while (w)
501 {
502 bm::id_t t = w & -w;
503 bits[pos++] = (B)(bm::word_bitcount(t - 1));
504 w &= w - 1;
505 }
506 return (unsigned short)pos;
507}
508
509
510/*!
511 \brief Unpacks 64-bit word into list of ON bit indexes using popcnt method
512 \param w - value
513 \param bits - pointer on the result array
514 \return number of bits in the list
515 @ingroup bitfunc
516*/
517template<typename B>
518unsigned short bitscan_popcnt64(bm::id64_t w, B* bits) BMNOEXCEPT
519{
520 unsigned short pos = 0;
521 while (w)
522 {
523 bm::id64_t t = bmi_blsi_u64(w); // w & -w;
524 bits[pos++] = (B) bm::word_bitcount64(t - 1);
525 w = bmi_bslr_u64(w); // w &= w - 1;
526 }
527 return pos;
528}
529
530/*!
531 \brief Unpacks 64-bit word into list of ON bit indexes using popcnt method
532 \param w - value
533 \param bits - pointer on the result array
534 \param offs - value to add to bit position (programmed shift)
535 \return number of bits in the list
536 @ingroup bitfunc
537*/
538template<typename B>
539unsigned short
540bitscan_popcnt64(bm::id64_t w, B* bits, unsigned short offs) BMNOEXCEPT
541{
542 unsigned short pos = 0;
543 while (w)
544 {
545 bm::id64_t t = bmi_blsi_u64(w); // w & -w;
546 bits[pos++] = B(bm::word_bitcount64(t - 1) + offs);
547 w = bmi_bslr_u64(w); // w &= w - 1;
548 }
549 return pos;
550}
551
552
553template<typename V, typename B>
554unsigned short bitscan(V w, B* bits) BMNOEXCEPT
555{
556 if (bm::conditional<sizeof(V) == 8>::test())
557 return bm::bitscan_popcnt64(w, bits);
558 else
559 return bm::bitscan_popcnt((bm::word_t)w, bits);
560}
561
562// --------------------------------------------------------------
563// Functions for select
564// --------------------------------------------------------------
565
566/**
567 \brief word find index of the rank-th bit set by bit-testing
568 \param w - 64-bit work to search
569 \param rank - rank to select (should be > 0)
570
571 \return selected value (inxed of bit set)
572*/
573inline
575{
576 BM_ASSERT(w);
577 BM_ASSERT(rank);
578
579 for (unsigned count = 0; w; w >>=1ull, ++count)
580 {
581 rank -= unsigned(w & 1ull);
582 if (!rank)
583 return count;
584 }
585 BM_ASSERT(0); // shoud not be here if rank is achievable
586 return ~0u;
587}
588
589/**
590 \brief word find index of the rank-th bit set by bit-testing
591 \param w - 64-bit work to search
592 \param rank - rank to select (should be > 0)
593
594 \return selected value (inxed of bit set)
595*/
596inline
598{
599 BM_ASSERT(w);
600 BM_ASSERT(rank);
601 BM_ASSERT(rank <= bm::word_bitcount64(w));
602
603 do
604 {
605 --rank;
606 if (!rank)
607 break;
608 w &= w - 1;
609 } while (1);
610
611 bm::id64_t t = w & -w;
612 unsigned count = bm::word_bitcount64(t - 1);
613 return count;
614}
615
616/**
617 \brief word find index of the rank-th bit set by bit-testing
618 \param w - 64-bit work to search
619 \param rank - rank to select (should be > 0)
620
621 \return selected value (inxed of bit set)
622*/
623inline
624unsigned word_select64(bm::id64_t w, unsigned rank) BMNOEXCEPT
625{
626#if defined(BMI2_SELECT64)
627 return BMI2_SELECT64(w, rank);
628#else
629 #if defined(BMI1_SELECT64)
630 return BMI2_SELECT64(w, rank);
631 #else
632 return bm::word_select64_bitscan(w, rank);
633 #endif
634#endif
635}
636
637// --------------------------------------------------------------
638// Functions for bit-block digest calculation
639// --------------------------------------------------------------
640
641/*!
642 \brief Compute digest mask for word address in block
643
644 @return digest bit-mask
645
646 @ingroup bitfunc
647 @internal
648*/
651{
652 bm::id64_t mask(1ull);
653 return mask << (w_idx / bm::set_block_digest_wave_size);
654}
655
656/**
657 \brief Compute digest mask for [from..to] positions
658 \param from - range from (in bit-block coordinates)
659 \param to - range to (in bit-block coordinates)
660
661 @ingroup bitfunc
662 @internal
663*/
665bm::id64_t digest_mask(unsigned from, unsigned to) BMNOEXCEPT
666{
667 BM_ASSERT(from <= to);
668
669 bm::id64_t digest_from = from >> bm::set_block_digest_pos_shift;
671 const bm::id64_t maskFF(~0ull);
672 return ((maskFF) >> (63 - (digest_to - digest_from))) << digest_from;
673}
674
675/*!
676 \brief check if all digest bits for the range [from..to] are 0
677
678 \param digest - current digest
679 \param bitpos_from - range from (in bit-block coordinates)
680 \param bitpos_to - range to (in bit-block coordinates)
681
682 @return true if all range is zero
683
684 @ingroup bitfunc
685 @internal
686*/
687inline
689 unsigned bitpos_from, unsigned bitpos_to) BMNOEXCEPT
690{
691 bm::id64_t mask = bm::digest_mask(bitpos_from, bitpos_to);
692 return !(digest & mask);
693}
694
695/*!
696 \brief Init block with 000111000 pattren based on digest
697 \param block - Bit block [out]
698 \param digest - digest (used for block initialization)
699
700 @ingroup bitfunc
701 @internal
702*/
703inline
705{
706 unsigned off;
707 for (unsigned i = 0; i < 64; ++i)
708 {
710 bm::word_t mask = (digest & 1) ? ~0u : 0u;
711#if defined(VECT_BLOCK_SET_DIGEST)
712 VECT_BLOCK_SET_DIGEST(&block[off], mask);
713#else
714 for (unsigned j = 0; j < bm::set_block_digest_wave_size; j+=4)
715 {
716 block[off+j+0] = block[off+j+1] =
717 block[off+j+2] = block[off+j+3] = mask;
718 } // for j
719#endif
720 digest >>= 1ull;
721 } // for
722}
723
724/*!
725 \brief Compute digest for 64 non-zero areas
726 \param block - Bit block
727
728 @return digest bit-mask (0 means all block is empty)
729
730 @ingroup bitfunc
731 @internal
732*/
733inline
735{
736 bm::id64_t digest0 = 0;
737 unsigned off;
738
739 for (unsigned i = 0; i < 64; ++i)
740 {
742 #if defined(VECT_IS_DIGEST_ZERO)
743 bool all_zero = VECT_IS_DIGEST_ZERO(&block[off]);
744 digest0 |= bm::id64_t(!all_zero) << i;
745 #else
746 const bm::id64_t mask(1ull);
747 for (unsigned j = 0; j < bm::set_block_digest_wave_size; j+=4)
748 {
749 bm::word_t w =
750 block[off+j+0] | block[off+j+1] | block[off+j+2] | block[off+j+3];
751 if (w)
752 {
753 digest0 |= (mask << i);
754 break;
755 }
756 } // for j
757 #endif
758
759 } // for i
760 return digest0;
761}
762
763/*!
764 \brief Compute digest for 64 non-zero areas based on existing digest
765 (function revalidates zero areas)
766 \param block - bit block
767 \param digest - start digest
768
769 @return digest bit-mask (0 means all block is empty)
770
771 @ingroup bitfunc
772 @internal
773*/
774inline
777{
778 const bm::id64_t mask(1ull);
779 bm::id64_t d = digest;
780
781 while (d)
782 {
783 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
784
785 unsigned wave = bm::word_bitcount64(t - 1);
786 unsigned off = wave * bm::set_block_digest_wave_size;
787
788 #if defined(VECT_IS_DIGEST_ZERO)
789 bool all_zero = VECT_IS_DIGEST_ZERO(&block[off]);
790 digest &= all_zero ? ~(mask << wave) : digest;
791 #else
793 (const bm::bit_block_t::bunion_t*)(&block[off]);
794 bm::id64_t w64 = 0;
795 unsigned j = 0;
796 do
797 {
798 w64 |=
799 src_u->w64[j+0] | src_u->w64[j+1] | src_u->w64[j+2] | src_u->w64[j+3];
800 j += 4;
801 } while ((j < bm::set_block_digest_wave_size/2) & !w64);
802 digest &= w64 ? digest : ~(mask << wave);
803 #endif
804
805 d = bm::bmi_bslr_u64(d); // d &= d - 1;
806 } // while
807
808 BM_ASSERT(bm::calc_block_digest0(block) == digest);
809 return digest;
810}
811
812// --------------------------------------------------------------
813
814
815/// Returns true if set operation is constant (bitcount)
816inline
818{
819 return (int(op) >= int(set_COUNT));
820}
821
822/**
823 Convert set operation to operation
824*/
825inline
827{
828 BM_ASSERT(op == set_AND ||
829 op == set_OR ||
830 op == set_SUB ||
831 op == set_XOR);
832 return (bm::operation) op;
833}
834
835//---------------------------------------------------------------------
836
837/**
838 Structure carries pointer on bit block with all bits 1
839 @ingroup bitfunc
840 @internal
841*/
842template<bool T> struct all_set
843{
845 {
849
851 {
852 ::memset(_p, 0xFF, sizeof(_p)); // set FULL BLOCK content (all 1s)
853 if (bm::conditional<sizeof(void*) == 8>::test())
854 {
855 const unsigned long long magic_mask = 0xFFFFfffeFFFFfffe;
856 ::memcpy(&_p_fullp, &magic_mask, sizeof(magic_mask));
857 for (unsigned i = 0; i < bm::set_sub_array_size; ++i)
858 _s[i] = reinterpret_cast<bm::word_t*>(magic_mask);
859 }
860 else
861 {
862 const unsigned magic_mask = 0xFFFFfffe;
863 ::memcpy(&_p_fullp, &magic_mask, sizeof(magic_mask));
864 for (unsigned i = 0; i < bm::set_sub_array_size; ++i)
865 _s[i] = reinterpret_cast<bm::word_t*>(magic_mask);
866 }
867 }
868 };
869
870 // version with minimal branching, super-scalar friendly
871 //
872 inline
874 {
875 bm::id64_t type;
876 if (bm::conditional<sizeof(void*) == 8>::test())
877 {
878 bm::id64_t w = reinterpret_cast<unsigned long long>(bp);
879 type = (w & 3) | // FULL BLOCK or GAP
880 ((bp == _block._p) << 1);
881 type = type ? type : w;
882 }
883 else
884 {
885 unsigned w = reinterpret_cast<unsigned long>(bp);
886 type = (w & 3) | // FULL BLOCK or GAP
887 ((bp == _block._p) << 1);
888 type = type ? type : w;
889 }
890 return type;
891 }
892
894 static bool is_full_block(const bm::word_t* bp) BMNOEXCEPT
895 { return (bp == _block._p || bp == _block._p_fullp); }
896
899 { return (bp && !(bp == _block._p || bp == _block._p_fullp)); }
900
902};
903
904
905template<bool T> typename all_set<T>::all_set_block all_set<T>::_block;
906
907/// XOR swap two scalar variables
908template<typename W>
909void xor_swap(W& x, W& y) BMNOEXCEPT
910{
911 BM_ASSERT(&x != &y);
912 x ^= y;
913 y ^= x;
914 x ^= y;
915}
916
917/*!
918 Fini not NULL position
919 @return index of not NULL pointer
920 @internal
921*/
922template<typename N>
923bool find_not_null_ptr(bm::word_t*** arr, N start, N size, N* pos) BMNOEXCEPT
924{
925 BM_ASSERT(pos);
926// BM_ASSERT(start < size);
927//#if defined(BM64_AVX2) || defined(BM64_AVX512)
928// TODO: optimization for SIMD based next ptr scan
929#if 0
930 const unsigned unroll_factor = 4;
931 const unsigned len = (size - start);
932 const unsigned len_unr = len - (len % unroll_factor);
933 unsigned k;
934
935 arr += start;
936 for (k = 0; k < len_unr; k+=unroll_factor)
937 {
938 if (!avx2_test_all_zero_wave(arr+k))
939 {
940 if (arr[k])
941 {
942 *pos = k + start;
943 return true;
944 }
945 if (arr[k+1])
946 {
947 *pos = k + start + 1;
948 return true;
949 }
950 if (arr[k+2])
951 {
952 *pos = k + start + 2;
953 return true;
954 }
955 if (arr[k+3])
956 {
957 *pos = k + start + 3;
958 return true;
959 }
960 }
961 } // for k
962
963 for (; k < len; ++k)
964 {
965 if (arr[k])
966 {
967 *pos = k + start;
968 return true;
969 }
970 } // for k
971#else
972 for (; start < size; ++start)
973 {
974 if (arr[start])
975 {
976 *pos = start;
977 return true;
978 }
979 } // for i
980#endif
981 return false;
982}
983
984
985
986
987//---------------------------------------------------------------------
988
989/*!
990 \brief Lexicographical comparison of two words as bit strings (reference)
991 Auxiliary implementation for testing and reference purposes.
992 \param w1 - First word.
993 \param w2 - Second word.
994 \return <0 - less, =0 - equal, >0 - greater.
995
996 @ingroup bitfunc
997*/
998template<typename T> int wordcmp0(T w1, T w2)
999{
1000 while (w1 != w2)
1001 {
1002 int res = (w1 & 1) - (w2 & 1);
1003 if (res != 0) return res;
1004 w1 >>= 1;
1005 w2 >>= 1;
1006 }
1007 return 0;
1008}
1009
1010
1011/*
1012template<typename T> int wordcmp(T w1, T w2)
1013{
1014 T diff = w1 ^ w2;
1015 return diff ? ((w1 & diff & (diff ^ (diff - 1)))? 1 : -1) : 0;
1016}
1017*/
1018/*!
1019 \brief Lexicographical comparison of two words as bit strings.
1020 Auxiliary implementation for testing and reference purposes.
1021 \param a - First word.
1022 \param b - Second word.
1023 \return <0 - less, =0 - equal, >0 - greater.
1024
1025 @ingroup bitfunc
1026*/
1027template<typename T> int wordcmp(T a, T b)
1028{
1029 T diff = a ^ b;
1030 return diff? ( (a & diff & -diff)? 1 : -1 ) : 0;
1031}
1032
1033
1034// Low bit extraction
1035// x & (x ^ (x-1))
1036
1037
1038// ----------------------------------------------------------------------
1039
1040
1041/*! @brief Returns "true" if all bits in the block are 0
1042 @ingroup bitfunc
1043*/
1044inline
1046{
1047#if defined(VECT_IS_ZERO_BLOCK)
1048 return VECT_IS_ZERO_BLOCK(start);
1049#else
1050 const bm::wordop_t* BMRESTRICT blk = (bm::wordop_t*) (start);
1051 const bm::wordop_t* BMRESTRICT blk_end = (bm::wordop_t*)(start + bm::set_block_size);
1052 do
1053 {
1054 if (blk[0] | blk[1] | blk[2] | blk[3])
1055 return false;
1056 blk += 4;
1057 } while (blk < blk_end);
1058 return true;
1059#endif
1060}
1061
1062// ----------------------------------------------------------------------
1063
1064/*!
1065 \brief Checks if GAP block is all-zero.
1066 \param buf - GAP buffer pointer.
1067 \returns true if all-zero.
1068
1069 @ingroup gapfunc
1070*/
1073{
1074 // (almost) branchless variant:
1075 return (!(*buf & 1u)) & (!(bm::gap_max_bits - 1 - buf[1]));
1076}
1077
1078/*!
1079 \brief Checks if GAP block is all-one.
1080 \param buf - GAP buffer pointer.
1081 \returns true if all-one.
1082 @ingroup gapfunc
1083*/
1086{
1087 return ((*buf & 1u) && (buf[1] == bm::gap_max_bits - 1));
1088}
1089
1090/*!
1091 \brief Returs GAP block length.
1092 \param buf - GAP buffer pointer.
1093 \returns GAP block length.
1094
1095 @ingroup gapfunc
1096*/
1099{
1100 return (bm::gap_word_t)((*buf >> 3) + 1);
1101}
1102
1103
1104/*!
1105 \brief Returs GAP block capacity
1106 \param buf - GAP buffer pointer
1107 \param glevel_len - pointer on GAP header word
1108 \returns GAP block capacity.
1109
1110 @ingroup gapfunc
1111*/
1112template<typename T>
1113unsigned
1114gap_capacity(const T* BMRESTRICT buf, const T* BMRESTRICT glevel_len) BMNOEXCEPT
1115{
1116 return glevel_len[(*buf >> 1) & 3];
1117}
1118
1119
1120/*!
1121 \brief Returs GAP block capacity limit.
1122 \param buf - GAP buffer pointer.
1123 \param glevel_len - GAP lengths table (gap_len_table)
1124 \returns GAP block limit.
1125
1126 @ingroup gapfunc
1127*/
1128template<typename T>
1129unsigned
1130gap_limit(const T* BMRESTRICT buf, const T* BMRESTRICT glevel_len) BMNOEXCEPT
1131{
1132 return glevel_len[(*buf >> 1) & 3]-4;
1133}
1134
1135
1136/*!
1137 \brief Returs GAP blocks capacity level.
1138 \param buf - GAP buffer pointer.
1139 \returns GAP block capacity level.
1140
1141 @ingroup gapfunc
1142*/
1143template<typename T>
1145{
1146 return T((*buf >> 1) & 3u);
1147}
1148
1149
1150/*!
1151 \brief GAP block find the last set bit
1152
1153 \param buf - GAP buffer pointer.
1154 \param last - index of the last 1 bit
1155
1156 \return 0 if 1 bit was NOT found
1157
1158 @ingroup gapfunc
1159*/
1160template<typename T>
1161unsigned
1162gap_find_last(const T* BMRESTRICT buf, unsigned* BMRESTRICT last) BMNOEXCEPT
1163{
1164 BM_ASSERT(last);
1165
1166 T is_set = (*buf) & 1u;
1167 T end = T((*buf) >> 3u);
1168
1169 BM_ASSERT(buf[end] == bm::gap_max_bits - 1);
1170
1171 is_set ^= T((end-1) & 1u);
1172 if (is_set)
1173 {
1174 *last = buf[end];
1175 return is_set;
1176 }
1177 *last = buf[--end];
1178 return end;
1179}
1180
1181/*!
1182 \brief GAP block find the first set bit
1183
1184 \param buf - GAP buffer pointer.
1185 \param first - index of the first 1 bit
1186
1187 \return 0 if 1 bit was NOT found
1188
1189 @ingroup gapfunc
1190*/
1191template<typename T>
1192unsigned
1193gap_find_first(const T* BMRESTRICT buf, unsigned* BMRESTRICT first) BMNOEXCEPT
1194{
1195 BM_ASSERT(first);
1196
1197 T is_set = (*buf) & 1u;
1198 if (is_set)
1199 {
1200 *first = 0;
1201 return is_set;
1202 }
1203 if (buf[1] == bm::gap_max_bits - 1)
1204 return 0;
1205 *first = buf[1] + 1;
1206 return 1;
1207}
1208
1209
1210
1211/*
1212 \brief Binary search for the block where bit = pos located.
1213 \param buf - GAP buffer pointer.
1214 \param pos - index of the element.
1215 \param is_set - output. GAP value (0 or 1).
1216 \return GAP index.
1217 @ingroup gapfunc
1218*/
1219template<typename T>
1220unsigned gap_bfind(const T* BMRESTRICT buf,
1221 unsigned pos, unsigned* BMRESTRICT is_set) BMNOEXCEPT
1222{
1224 #undef VECT_GAP_BFIND // TODO: VECTOR bfind causes performance degradation
1225 #ifdef VECT_GAP_BFIND
1226 return VECT_GAP_BFIND(buf, pos, is_set);
1227 #else
1228 *is_set = (*buf) & 1;
1229
1230 unsigned start = 1;
1231 unsigned end = 1 + ((*buf) >> 3);
1232
1233 while ( start != end )
1234 {
1235 unsigned curr = (start + end) >> 1;
1236 if ( buf[curr] < pos )
1237 start = curr + 1;
1238 else
1239 end = curr;
1240 }
1241 *is_set ^= ((start-1) & 1);
1242 return start;
1243 #endif
1244}
1245
1246
1247/*!
1248 \brief Tests if bit = pos is true.
1249 \param buf - GAP buffer pointer.
1250 \param pos - index of the element.
1251 \return true if position is in "1" gap
1252 @ingroup gapfunc
1253*/
1254template<typename T>
1255unsigned gap_test(const T* BMRESTRICT buf, unsigned pos) BMNOEXCEPT
1256{
1258
1259 unsigned start = 1;
1260 unsigned end = 1 + ((*buf) >> 3);
1261 if (end - start < 10)
1262 {
1263 unsigned sv = *buf & 1;
1264 unsigned sv1= sv ^ 1;
1265 if (buf[1] >= pos) return sv;
1266 if (buf[2] >= pos) return sv1;
1267 if (buf[3] >= pos) return sv;
1268 if (buf[4] >= pos) return sv1;
1269 if (buf[5] >= pos) return sv;
1270 if (buf[6] >= pos) return sv1;
1271 if (buf[7] >= pos) return sv;
1272 if (buf[8] >= pos) return sv1;
1273 if (buf[9] >= pos) return sv;
1274 BM_ASSERT(0);
1275 }
1276 else
1277 {
1278 while (start != end)
1279 {
1280 unsigned curr = (start + end) >> 1;
1281 if (buf[curr] < pos)
1282 start = curr + 1;
1283 else
1284 end = curr;
1285 }
1286 }
1287 return ((*buf) & 1) ^ ((--start) & 1);
1288}
1289
1290/*!
1291 \brief Tests if bit = pos is true. Analog of bm::gap_test with SIMD unrolling.
1292 \param buf - GAP buffer pointer.
1293 \param pos - index of the element.
1294 \return true if position is in "1" gap
1295 @ingroup gapfunc
1296*/
1297template<typename T>
1298unsigned gap_test_unr(const T* BMRESTRICT buf, const unsigned pos) BMNOEXCEPT
1299{
1301
1302 if (pos == 0) // quick answer possible
1303 {
1304 return (*buf) & 1;
1305 }
1306#if defined(BMSSE2OPT)
1307 unsigned res = bm::sse2_gap_test(buf, pos);
1308 BM_ASSERT(res == bm::gap_test(buf, pos));
1309#elif defined(BMSSE42OPT)
1310 unsigned res = bm::sse42_gap_test(buf, pos);
1311 BM_ASSERT(res == bm::gap_test(buf, pos));
1312#elif defined(BMAVX2OPT)
1313 unsigned res = bm::avx2_gap_test(buf, pos);
1314 BM_ASSERT(res == bm::gap_test(buf, pos));
1315#else
1316 unsigned res = bm::gap_test(buf, pos);
1317#endif
1318 return res;
1319}
1320
1321/*! For each non-zero block in [from, to] executes supplied functor
1322 \internal
1323*/
1324template<typename T, typename N, typename F>
1326 N top_size, N nb_from, N nb_to, F& f) BMNOEXCEPT
1327{
1328 BM_ASSERT(top_size);
1329 if (nb_from > nb_to)
1330 return;
1331 unsigned i_from = unsigned(nb_from >> bm::set_array_shift);
1332 unsigned j_from = unsigned(nb_from & bm::set_array_mask);
1333 unsigned i_to = unsigned(nb_to >> bm::set_array_shift);
1334 unsigned j_to = unsigned(nb_to & bm::set_array_mask);
1335
1336 if (i_from >= top_size)
1337 return;
1338 if (i_to >= top_size)
1339 {
1340 i_to = unsigned(top_size-1);
1341 j_to = bm::set_sub_array_size-1;
1342 }
1343
1344 for (unsigned i = i_from; i <= i_to; ++i)
1345 {
1346 T** blk_blk = root[i];
1347 if (!blk_blk)
1348 continue;
1349 if ((bm::word_t*)blk_blk == FULL_BLOCK_FAKE_ADDR)
1350 {
1351 unsigned j = (i == i_from) ? j_from : 0;
1352 if (!j && (i != i_to)) // full sub-block
1353 f.add_full(bm::set_sub_total_bits);
1354 else
1355 {
1356 do
1357 {
1358 f.add_full(bm::gap_max_bits);
1359 if ((i == i_to) && (j == j_to))
1360 return;
1361 } while (++j < bm::set_sub_array_size);
1362 }
1363 }
1364 else
1365 {
1366 unsigned j = (i == i_from) ? j_from : 0;
1367 do
1368 {
1369 if (blk_blk[j])
1370 f(blk_blk[j]);
1371 if ((i == i_to) && (j == j_to))
1372 return;
1373 } while (++j < bm::set_sub_array_size);
1374 }
1375 } // for i
1376}
1377
1378/*! For each non-zero block executes supplied function.
1379 \internal
1380*/
1381template<class T, class F>
1382void for_each_nzblock(T*** root, unsigned size1, F& f)
1383{
1384 typedef typename F::size_type size_type;
1385 for (unsigned i = 0; i < size1; ++i)
1386 {
1387 T** blk_blk = root[i];
1388 if (!blk_blk)
1389 {
1390 f.on_empty_top(i);
1391 continue;
1392 }
1393 f.on_non_empty_top(i);
1394 if ((bm::word_t*)blk_blk == FULL_BLOCK_FAKE_ADDR)
1395 {
1396 size_type r = i * bm::set_sub_array_size;
1397 unsigned j = 0;
1398 do
1399 {
1400 f(FULL_BLOCK_FAKE_ADDR, r + j);
1401 } while (++j < bm::set_sub_array_size);
1402 continue;
1403 }
1404
1405 unsigned non_empty_top = 0;
1406 size_type r = i * bm::set_sub_array_size;
1407 unsigned j = 0;
1408 do
1409 {
1410#if defined(BM64_AVX2) || defined(BM64_AVX512)
1411 if (!avx2_test_all_zero_wave(blk_blk + j))
1412 {
1413 non_empty_top = 1;
1414 T* blk0 = blk_blk[j + 0];
1415 T* blk1 = blk_blk[j + 1];
1416 T* blk2 = blk_blk[j + 2];
1417 T* blk3 = blk_blk[j + 3];
1418
1419 size_type block_idx = r + j + 0;
1420 if (blk0)
1421 f(blk0, block_idx);
1422 else
1423 f.on_empty_block(block_idx);
1424
1425 if (blk1)
1426 f(blk1, block_idx + 1);
1427 else
1428 f.on_empty_block(block_idx + 1);
1429
1430 if (blk2)
1431 f(blk2, block_idx + 2);
1432 else
1433 f.on_empty_block(block_idx + 2);
1434
1435 if (blk3)
1436 f(blk3, block_idx + 3);
1437 else
1438 f.on_empty_block(block_idx + 3);
1439 }
1440 else
1441 {
1442 f.on_empty_block(r + j + 0); f.on_empty_block(r + j + 1);
1443 f.on_empty_block(r + j + 2); f.on_empty_block(r + j + 3);
1444 }
1445 j += 4;
1446#elif defined(BM64_SSE4)
1447 if (!sse42_test_all_zero_wave((blk_blk + j)))
1448 {
1449 non_empty_top = 1;
1450 T* blk0 = blk_blk[j + 0];
1451 T* blk1 = blk_blk[j + 1];
1452
1453 size_type block_idx = r + j + 0;
1454 if (blk0)
1455 f(blk0, block_idx);
1456 else
1457 f.on_empty_block(block_idx);
1458
1459 ++block_idx;
1460 if (blk1)
1461 f(blk1, block_idx);
1462 else
1463 f.on_empty_block(block_idx);
1464 }
1465 else
1466 {
1467 f.on_empty_block(r + j + 0);
1468 f.on_empty_block(r + j + 1);
1469 }
1470 j += 2;
1471#else
1472 if (blk_blk[j])
1473 {
1474 f(blk_blk[j], r + j);
1475 non_empty_top = 1;
1476 }
1477 else
1478 f.on_empty_block(r + j);
1479 ++j;
1480#endif
1481 } while (j < bm::set_sub_array_size);
1482
1483 if (non_empty_top == 0)
1484 f.on_empty_top(i);
1485 } // for i
1486}
1487
1488/*! For each non-zero block executes supplied function.
1489*/
1490template<class T, class F>
1491void for_each_nzblock2(T*** root, unsigned size1, F& f)
1492{
1493#ifdef BM64_SSE4
1494 for (unsigned i = 0; i < size1; ++i)
1495 {
1496 T** blk_blk;
1497 if ((blk_blk = root[i])!=0)
1498 {
1499 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1500 {
1501 for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1502 {
1504 }
1505 continue;
1506 }
1507 {
1508 __m128i w0;
1509 for (unsigned j = 0; j < bm::set_sub_array_size; j+=4)
1510 {
1511 w0 = _mm_loadu_si128((__m128i*)(blk_blk + j));
1512 if (!_mm_testz_si128(w0, w0))
1513 {
1514 T* blk0 = blk_blk[j + 0];
1515 T* blk1 = blk_blk[j + 1];
1516
1517 if (blk0)
1518 f(blk0);
1519 if (blk1)
1520 f(blk1);
1521 }
1522 w0 = _mm_loadu_si128((__m128i*)(blk_blk + j + 2));
1523 if (!_mm_testz_si128(w0, w0))
1524 {
1525 T* blk0 = blk_blk[j + 2];
1526 T* blk1 = blk_blk[j + 3];
1527 if (blk0)
1528 f(blk0);
1529 if (blk1)
1530 f(blk1);
1531 }
1532 } // for j
1533 }
1534 }
1535 } // for i
1536#elif defined(BM64_AVX2) || defined(BM64_AVX512)
1537 for (unsigned i = 0; i < size1; ++i)
1538 {
1539 T** blk_blk;
1540 if ((blk_blk = root[i]) != 0)
1541 {
1542 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1543 {
1544 for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1545 {
1547 }
1548 continue;
1549 }
1550 {
1551 for (unsigned j = 0; j < bm::set_sub_array_size; j += 4)
1552 {
1553 __m256i w0 = _mm256_loadu_si256((__m256i*)(blk_blk + j));
1554 if (!_mm256_testz_si256(w0, w0))
1555 {
1556 // as a variant could use: blk0 = (T*)_mm256_extract_epi64(w0, 0);
1557 // but it measures marginally slower
1558 T* blk0 = blk_blk[j + 0];
1559 T* blk1 = blk_blk[j + 1];
1560 T* blk2 = blk_blk[j + 2];
1561 T* blk3 = blk_blk[j + 3];
1562 if (blk0)
1563 f(blk0);
1564 if (blk1)
1565 f(blk1);
1566 if (blk2)
1567 f(blk2);
1568 if (blk3)
1569 f(blk3);
1570 }
1571 } // for j
1572 }
1573 }
1574 } // for i
1575
1576#else // non-SIMD mode
1577 for (unsigned i = 0; i < size1; ++i)
1578 {
1579 T** blk_blk;
1580 if ((blk_blk = root[i])!=0)
1581 {
1582 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1583 {
1584 for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1585 {
1587 }
1588 continue;
1589 }
1590 unsigned j = 0;
1591 do
1592 {
1593 if (blk_blk[j])
1594 f(blk_blk[j]);
1595 if (blk_blk[j+1])
1596 f(blk_blk[j+1]);
1597 if (blk_blk[j+2])
1598 f(blk_blk[j+2]);
1599 if (blk_blk[j+3])
1600 f(blk_blk[j+3]);
1601 j += 4;
1602 } while (j < bm::set_sub_array_size);
1603 }
1604 } // for i
1605#endif
1606}
1607
1608
1609/*! For each non-zero block executes supplied function-predicate.
1610 Function returns if function-predicate returns true
1611*/
1612template<typename T, typename BI, typename F>
1613bool for_each_nzblock_if(T*** root, BI size1, F& f) BMNOEXCEPT
1614{
1615 BI block_idx = 0;
1616 for (BI i = 0; i < size1; ++i)
1617 {
1618 T** blk_blk = root[i];
1619 if (!blk_blk)
1620 {
1621 block_idx += bm::set_sub_array_size;
1622 continue;
1623 }
1624 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1625 {
1626 for (unsigned j = 0; j < bm::set_sub_array_size; ++j, ++block_idx)
1627 {
1628 if (f(FULL_BLOCK_FAKE_ADDR, block_idx))
1629 return true;
1630 } // for j
1631 continue;
1632 }
1633
1634 for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1635 {
1636 if (blk_blk[j])
1637 if (f(blk_blk[j], block_idx))
1638 return true;
1639 } // for j
1640 } // for i
1641 return false;
1642}
1643
1644/*! For each block executes supplied function.
1645*/
1646template<class T, class F, typename BLOCK_IDX>
1647void for_each_block(T*** root, unsigned size1, F& f, BLOCK_IDX start)
1648{
1649 BLOCK_IDX block_idx = start;
1650 for (unsigned i = 0; i < size1; ++i)
1651 {
1652 T** blk_blk = root[i];
1653 if (blk_blk)
1654 {
1655 if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1656 {
1657 for (unsigned j = 0; j < bm::set_sub_array_size; ++j, ++block_idx)
1658 {
1659 f(FULL_BLOCK_FAKE_ADDR, block_idx);
1660 }
1661 continue;
1662 }
1663 for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1664 {
1665 f(blk_blk[j], block_idx);
1666 }
1667 }
1668 else
1669 {
1670 for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1671 {
1672 f(0, block_idx);
1673 }
1674 }
1675 }
1676}
1677
1678
1679
1680/*! Special BM optimized analog of STL for_each
1681*/
1682template<class T, class F> F bmfor_each(T first, T last, F f)
1683{
1684 do
1685 {
1686 f(*first);
1687 ++first;
1688 } while (first < last);
1689 return f;
1690}
1691
1692/*! Computes SUM of all elements of the sequence
1693*/
1694template<typename T>
1695bm::id64_t sum_arr(const T* first, const T* last) BMNOEXCEPT
1696{
1697 bm::id64_t sum = 0;
1698 for (;first < last; ++first)
1699 sum += *first;
1700 return sum;
1701}
1702
1703/*!
1704 Extract short (len=1) exceptions from the GAP block
1705 \param buf - GAP buffer to split
1706 \param arr0 - [OUT] list of isolates 0 positions (clear list)
1707 \param arr1 - [OUT] list of isolated 1 positions (set list)
1708 \param arr0_cnt - [OUT] arr0 size
1709 \param arr1_cnt -
1710 @ingroup gapfunc
1711*/
1712template<typename T>
1713void gap_split(const T* buf,
1714 T* arr0, T* arr1, T& arr0_cnt, T& arr1_cnt) BMNOEXCEPT
1715{
1716 const T* pcurr = buf;
1717 unsigned len = (*pcurr >> 3);
1718 const T* pend = pcurr + len;
1719
1720 T cnt0, cnt1;
1721 cnt0 = cnt1 = 0;
1722 unsigned is_set = (*buf & 1);
1723
1724 if (*pcurr == 0)
1725 {
1726 if (is_set)
1727 {
1728 arr1[cnt1] = *pcurr;
1729 ++cnt1;
1730 }
1731 else
1732 {
1733 arr0[cnt0] = *pcurr;
1734 ++cnt0;
1735 }
1736 }
1737 T prev = *pcurr;
1738 ++pcurr;
1739
1740 while (pcurr <= pend)
1741 {
1742 is_set ^= 1;
1743 T delta = *pcurr - prev;
1744 if (delta == 1)
1745 {
1746 if (is_set)
1747 {
1748 arr1[cnt1] = prev;
1749 ++cnt1;
1750 }
1751 else
1752 {
1753 arr0[cnt0] = prev;
1754 ++cnt0;
1755 }
1756 }
1757 prev = *pcurr++;
1758 } // while
1759
1760 arr0_cnt = cnt0;
1761 arr1_cnt = cnt1;
1762}
1763
1764
1765/*!
1766 \brief Calculates number of bits ON in GAP buffer.
1767 \param buf - GAP buffer pointer.
1768 \param dsize - buffer size
1769 \return Number of non-zero bits.
1770 @ingroup gapfunc
1771*/
1772template<typename T>
1773unsigned gap_bit_count(const T* buf, unsigned dsize=0) BMNOEXCEPT
1774{
1775 const T* pcurr = buf;
1776 if (dsize == 0)
1777 dsize = (*pcurr >> 3);
1778
1779 const T* pend = pcurr + dsize;
1780
1781 unsigned bits_counter = 0;
1782 ++pcurr;
1783
1784 if (*buf & 1)
1785 {
1786 bits_counter += *pcurr + 1;
1787 ++pcurr;
1788 }
1789 for (++pcurr; pcurr <= pend; pcurr += 2)
1790 bits_counter += *pcurr - *(pcurr-1);
1791 return bits_counter;
1792}
1793
1794/*!
1795 \brief Calculates number of bits ON in GAP buffer. Loop unrolled version.
1796 \param buf - GAP buffer pointer.
1797 \return Number of non-zero bits.
1798 @ingroup gapfunc
1799*/
1800template<typename T>
1801unsigned gap_bit_count_unr(const T* buf) BMNOEXCEPT
1802{
1803 const T* pcurr = buf;
1804 unsigned dsize = (*pcurr >> 3);
1805
1806 unsigned cnt = 0;
1807 pcurr = buf + 1; // set up start position
1808 T first_one = *buf & 1;
1809 if (first_one)
1810 {
1811 cnt += *pcurr + 1;
1812 ++pcurr;
1813 }
1814 ++pcurr; // set GAP to 1
1815
1816 #if defined(BMAVX2OPT) || defined(BMAVX512OPT)
1817 if (dsize > 34)
1818 {
1819 const unsigned unr_factor = 32;
1820 unsigned waves = (dsize-2) / unr_factor;
1821 pcurr = avx2_gap_sum_arr(pcurr, waves, &cnt);
1822 }
1823 #elif defined(BMSSE42OPT) || defined(BMSSE2OPT)
1824 if (dsize > 18)
1825 {
1826 const unsigned unr_factor = 16;
1827 unsigned waves = (dsize - 2) / unr_factor;
1828 pcurr = sse2_gap_sum_arr(pcurr, waves, &cnt);
1829 }
1830 #else
1831 if (dsize > 10)
1832 {
1833 const unsigned unr_factor = 8;
1834 unsigned waves = (dsize - 2) / unr_factor;
1835 for (unsigned i = 0; i < waves; i += unr_factor)
1836 {
1837 cnt += pcurr[0] - pcurr[0 - 1];
1838 cnt += pcurr[2] - pcurr[2 - 1];
1839 cnt += pcurr[4] - pcurr[4 - 1];
1840 cnt += pcurr[6] - pcurr[6 - 1];
1841
1842 pcurr += unr_factor;
1843 } // for
1844 }
1845 #endif
1846
1847 const T* pend = buf + dsize;
1848 for ( ; pcurr <= pend ; pcurr+=2)
1849 {
1850 cnt += *pcurr - *(pcurr - 1);
1851 }
1852 BM_ASSERT(cnt == bm::gap_bit_count(buf));
1853 return cnt;
1854}
1855
1856
1857
1858/*!
1859 \brief Counts 1 bits in GAP buffer in the closed [left, right] range.
1860 \param buf - GAP buffer pointer.
1861 \param left - leftmost bit index to start from
1862 \param right- rightmost bit index
1863 \return Number of non-zero bits.
1864 @ingroup gapfunc
1865*/
1866template<typename T>
1867unsigned gap_bit_count_range(const T* const buf,
1868 unsigned left, unsigned right) BMNOEXCEPT
1869{
1870 BM_ASSERT(left <= right);
1871 BM_ASSERT(right < bm::gap_max_bits);
1872
1873 const T* pcurr = buf;
1874 const T* pend = pcurr + (*pcurr >> 3);
1875
1876 unsigned bits_counter = 0;
1877 unsigned is_set;
1878 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1879 is_set = ~(is_set - 1u); // 0xFFF.. if true (mask for branchless code)
1880
1881 pcurr = buf + start_pos;
1882 if (right <= *pcurr) // we are in the target gap right now
1883 {
1884 bits_counter = unsigned(right - left + 1u) & is_set; // & is_set == if(is_set)
1885 return bits_counter;
1886 }
1887 bits_counter += unsigned(*pcurr - left + 1u) & is_set;
1888
1889 unsigned prev_gap = *pcurr++;
1890 for (is_set ^= ~0u; right > *pcurr; is_set ^= ~0u)
1891 {
1892 bits_counter += (*pcurr - prev_gap) & is_set;
1893 if (pcurr == pend)
1894 return bits_counter;
1895 prev_gap = *pcurr++;
1896 }
1897 bits_counter += unsigned(right - prev_gap) & is_set;
1898 return bits_counter;
1899}
1900
1901/*!
1902 \brief Test if all bits are 1 in GAP buffer in the [left, right] range.
1903 \param buf - GAP buffer pointer.
1904 \param left - leftmost bit index to start from
1905 \param right- rightmost bit index
1906 \return true if all bits are "11111"
1907 @ingroup gapfunc
1908*/
1909template<typename T>
1910bool gap_is_all_one_range(const T* const BMRESTRICT buf,
1911 unsigned left, unsigned right) BMNOEXCEPT
1912{
1913 BM_ASSERT(left <= right);
1914 BM_ASSERT(right < bm::gap_max_bits);
1915
1916 unsigned is_set;
1917 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1918 if (!is_set) // GAP is 0
1919 return false;
1920 const T* const pcurr = buf + start_pos;
1921 return (right <= *pcurr);
1922}
1923
1924/*!
1925 \brief Test if any bits are 1 in GAP buffer in the [left, right] range.
1926 \param buf - GAP buffer pointer.
1927 \param left - leftmost bit index to start from
1928 \param right- rightmost bit index
1929 \return true if at least 1 "00010"
1930 @ingroup gapfunc
1931*/
1932template<typename T>
1933bool gap_any_range(const T* const BMRESTRICT buf,
1934 unsigned left, unsigned right) BMNOEXCEPT
1935{
1936 BM_ASSERT(left <= right);
1937 BM_ASSERT(right < bm::gap_max_bits);
1938
1939 unsigned is_set;
1940 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1941 const T* const pcurr = buf + start_pos;
1942
1943 if (!is_set) // start GAP is 0 ...
1944 {
1945 if (right <= *pcurr) // ...bit if the interval goes into at least 1 blk
1946 return false; // .. nope
1947 return true;
1948 }
1949 return true;
1950}
1951
1952/*!
1953 \brief Test if any bits are 1 in GAP buffer in the [left, right] range
1954 and flanked with 0s
1955 \param buf - GAP buffer pointer.
1956 \param left - leftmost bit index to start from
1957 \param right- rightmost bit index
1958 \return true if "011110"
1959 @ingroup gapfunc
1960*/
1961template<typename T>
1962bool gap_is_interval(const T* const BMRESTRICT buf,
1963 unsigned left, unsigned right) BMNOEXCEPT
1964{
1965 BM_ASSERT(left <= right);
1966 BM_ASSERT(left > 0); // cannot check left-1 otherwise
1967 BM_ASSERT(right < bm::gap_max_bits-1); // cannot check right+1 otherwise
1968
1969 unsigned is_set;
1970 unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1971
1972 const T* pcurr = buf + start_pos;
1973 if (!is_set || (right != *pcurr) || (start_pos <= 1))
1974 return false;
1975 --pcurr;
1976 if (*pcurr != left-1)
1977 return false;
1978 return true;
1979}
1980
1981/**
1982 \brief Searches for the last 1 bit in the 111 interval of a GAP block
1983 \param buf - BIT block buffer
1984 \param nbit - bit index to start checking from
1985 \param pos - [out] found value
1986
1987 \return false if not found
1988 @ingroup gapfunc
1989*/
1990template<typename T>
1991bool gap_find_interval_end(const T* const BMRESTRICT buf,
1992 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
1993{
1994 BM_ASSERT(pos);
1996
1997 unsigned is_set;
1998 unsigned start_pos = bm::gap_bfind(buf, nbit, &is_set);
1999 if (!is_set)
2000 return false;
2001 *pos = buf[start_pos];
2002 return true;
2003}
2004
2005
2006/**
2007 \brief Searches for the first 1 bit in the 111 interval of a GAP block
2008 \param buf - BIT block buffer
2009 \param nbit - bit index to start checking from
2010 \param pos - [out] found value
2011
2012 \return false if not found
2013 @ingroup gapfunc
2014*/
2015template<typename T>
2016bool gap_find_interval_start(const T* const BMRESTRICT buf,
2017 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
2018{
2019 BM_ASSERT(pos);
2021
2022 unsigned is_set;
2023 unsigned start_pos = bm::gap_bfind(buf, nbit, &is_set);
2024 if (!is_set)
2025 return false;
2026 --start_pos;
2027 if (!start_pos)
2028 *pos = 0;
2029 else
2030 *pos = buf[start_pos]+1;
2031 return true;
2032}
2033
2034
2035/*!
2036 \brief GAP block find position for the rank
2037
2038 \param block - bit block buffer pointer
2039 \param rank - rank to find (must be > 0)
2040 \param nbit_from - start bit position in block
2041 \param nbit_pos - found position
2042
2043 \return 0 if position with rank was found, or
2044 the remaining rank (rank - population count)
2045
2046 @ingroup gapfunc
2047*/
2048template<typename T, typename SIZE_TYPE>
2049SIZE_TYPE gap_find_rank(const T* const block,
2050 SIZE_TYPE rank,
2051 unsigned nbit_from,
2052 unsigned& nbit_pos) BMNOEXCEPT
2053{
2054 BM_ASSERT(block);
2055 BM_ASSERT(rank);
2056
2057 const T* pcurr = block;
2058 const T* pend = pcurr + (*pcurr >> 3);
2059
2060 unsigned bits_counter = 0;
2061 unsigned is_set;
2062 unsigned start_pos = bm::gap_bfind(block, nbit_from, &is_set);
2063 is_set = ~(is_set - 1u); // 0xFFF.. if true (mask for branchless code)
2064
2065 pcurr = block + start_pos;
2066 bits_counter += unsigned(*pcurr - nbit_from + 1u) & is_set;
2067 if (bits_counter >= rank) // found!
2068 {
2069 nbit_pos = nbit_from + unsigned(rank) - 1u;
2070 return 0;
2071 }
2072 rank -= bits_counter;
2073 unsigned prev_gap = *pcurr++;
2074 for (is_set ^= ~0u; pcurr <= pend; is_set ^= ~0u)
2075 {
2076 bits_counter = (*pcurr - prev_gap) & is_set;
2077 if (bits_counter >= rank) // found!
2078 {
2079 nbit_pos = prev_gap + unsigned(rank);
2080 return 0;
2081 }
2082 rank -= bits_counter;
2083 prev_gap = *pcurr++;
2084 } // for
2085
2086 return rank;
2087}
2088
2089
2090
2091/*!
2092 \brief Counts 1 bits in GAP buffer in the closed [0, right] range.
2093 \param buf - GAP buffer pointer.
2094 \param right- rightmost bit index
2095 \param is_corrected - if true the result will be rank corrected
2096 if right bit == true count=count-1
2097 \return Number of non-zero bits
2098 @ingroup gapfunc
2099*/
2100template<typename T>
2101unsigned gap_bit_count_to(const T* const buf, T right,
2102 bool is_corrected=false) BMNOEXCEPT
2103{
2104 const T* pcurr = buf;
2105 const T* pend = pcurr + (*pcurr >> 3);
2106
2107 unsigned bits_counter = 0;
2108 unsigned is_set = ~((unsigned(*buf) & 1u) - 1u); // 0xFFF.. if true (mask for branchless code)
2109 BM_ASSERT(is_set == 0u || is_set == ~0u);
2110 pcurr = buf + 1;
2111
2112 if (right <= *pcurr) // we are in the target block right now
2113 {
2114 bits_counter = (right + 1u) & is_set; // & is_set == if (is_set)
2115 bits_counter -= (is_set & is_corrected);
2116 return bits_counter;
2117 }
2118 bits_counter += (*pcurr + 1u) & is_set;
2119
2120 unsigned prev_gap = *pcurr++;
2121 for (is_set ^= ~0u; right > *pcurr; is_set ^= ~0u)
2122 {
2123 bits_counter += (*pcurr - prev_gap) & is_set;
2124 if (pcurr == pend)
2125 {
2126 bits_counter -= (is_set & is_corrected);
2127 return bits_counter;
2128 }
2129 prev_gap = *pcurr++;
2130 }
2131 bits_counter += (right - prev_gap) & is_set;
2132 bits_counter -= (is_set & is_corrected);
2133 return bits_counter;
2134}
2135
2136
2137/*!
2138 D-GAP block for_each algorithm
2139
2140 D-Gap Functor is called for each element but last one.
2141
2142 \param gap_buf - GAP buffer
2143 \param func - functor object
2144
2145*/
2146template<class T, class Func>
2147void for_each_dgap(const T* gap_buf, Func& func)
2148{
2149 const T* pcurr = gap_buf;
2150 const T* pend = pcurr + (*pcurr >> 3);
2151 ++pcurr;
2152
2153 T prev = *pcurr;
2154 func((T)(prev + 1)); // first element incremented to avoid 0
2155 ++pcurr;
2156 do
2157 {
2158 func((T)(*pcurr - prev)); // all others are [N] - [N-1]
2159 prev = *pcurr;
2160 } while (++pcurr < pend);
2161}
2162
2163/** d-Gap copy functor
2164 @internal
2165*/
2166template<typename T> struct d_copy_func
2167{
2168 d_copy_func(T* dg_buf) : dgap_buf_(dg_buf) {}
2169 void operator()(T dgap) { *dgap_buf_++ = dgap; }
2170
2172};
2173
2174/*!
2175 \brief Convert GAP buffer into D-GAP buffer
2176
2177 Delta GAP representation is DGAP[N] = GAP[N] - GAP[N-1]
2178
2179 \param gap_buf - GAP buffer
2180 \param dgap_buf - Delta-GAP buffer
2181 \param copy_head - flag to copy GAP header
2182
2183 \internal
2184
2185 @ingroup gapfunc
2186*/
2187template<typename T>
2188T* gap_2_dgap(const T* BMRESTRICT gap_buf,
2189 T* BMRESTRICT dgap_buf, bool copy_head=true) BMNOEXCEPT
2190{
2191 if (copy_head) // copy GAP header
2192 {
2193 *dgap_buf++ = *gap_buf;
2194 }
2195
2196 d_copy_func<T> copy_func(dgap_buf);
2197 for_each_dgap<T, d_copy_func<T> >(gap_buf, copy_func);
2198 return copy_func.dgap_buf_;
2199}
2200
2201/*!
2202 \brief Convert D-GAP buffer into GAP buffer
2203
2204 GAP representation is GAP[N] = DGAP[N] + DGAP[N-1]
2205
2206 \param dgap_buf - Delta-GAP buffer
2207 \param gap_header - GAP header word
2208 \param gap_buf - GAP buffer
2209
2210 \internal
2211 @ingroup gapfunc
2212*/
2213template<typename T>
2214void dgap_2_gap(const T* BMRESTRICT dgap_buf,
2215 T* BMRESTRICT gap_buf, T gap_header=0) BMNOEXCEPT
2216{
2217 const T* pcurr = dgap_buf;
2218 unsigned len;
2219 if (!gap_header) // GAP header is already part of the stream
2220 {
2221 len = *pcurr >> 3;
2222 *gap_buf++ = *pcurr++; // copy GAP header
2223 }
2224 else // GAP header passed as a parameter
2225 {
2226 len = gap_header >> 3;
2227 *gap_buf++ = gap_header; // assign GAP header
2228 }
2229 --len; // last element is actually not encoded
2230 const T* pend = pcurr + len;
2231
2232 *gap_buf = *pcurr++; // copy first element
2233 if (*gap_buf == 0)
2234 *gap_buf = 65535; // fix +1 overflow
2235 else
2236 *gap_buf = T(*gap_buf - 1);
2237
2238 for (++gap_buf; pcurr < pend; ++pcurr)
2239 {
2240 T prev = *(gap_buf-1); // don't remove temp(undef expression!)
2241 *gap_buf++ = T(*pcurr + prev);
2242 }
2243 *gap_buf = 65535; // add missing last element
2244}
2245
2246
2247/*!
2248 \brief Lexicographical comparison of GAP buffers.
2249 \param buf1 - First GAP buffer pointer.
2250 \param buf2 - Second GAP buffer pointer.
2251 \return <0 - less, =0 - equal, >0 - greater.
2252
2253 @ingroup gapfunc
2254*/
2255template<typename T>
2256int gapcmp(const T* buf1, const T* buf2) BMNOEXCEPT
2257{
2258 const T* pcurr1 = buf1;
2259 const T* pend1 = pcurr1 + (*pcurr1 >> 3);
2260 unsigned bitval1 = *buf1 & 1;
2261 ++pcurr1;
2262
2263 const T* pcurr2 = buf2;
2264 unsigned bitval2 = *buf2 & 1;
2265 ++pcurr2;
2266
2267 while (pcurr1 <= pend1)
2268 {
2269 if (*pcurr1 == *pcurr2)
2270 {
2271 if (bitval1 != bitval2)
2272 {
2273 return (bitval1) ? 1 : -1;
2274 }
2275 }
2276 else
2277 {
2278 if (bitval1 == bitval2)
2279 {
2280 if (bitval1)
2281 {
2282 return (*pcurr1 < *pcurr2) ? -1 : 1;
2283 }
2284 else
2285 {
2286 return (*pcurr1 < *pcurr2) ? 1 : -1;
2287 }
2288 }
2289 else
2290 {
2291 return (bitval1) ? 1 : -1;
2292 }
2293 }
2294
2295 ++pcurr1; ++pcurr2;
2296
2297 bitval1 ^= 1;
2298 bitval2 ^= 1;
2299 }
2300
2301 return 0;
2302}
2303
2304/*!
2305 \brief Find first bit which is different between two GAP-blocks
2306 \param buf1 - block 1
2307 \param buf2 - block 2
2308 \param pos - out - position of difference (undefined if blocks are equal)
2309 \return true if difference was found
2310
2311 @ingroup gapfunc
2312*/
2313template<typename T>
2315 const T* BMRESTRICT buf2,
2316 unsigned* BMRESTRICT pos) BMNOEXCEPT
2317{
2318 BM_ASSERT(buf1 && buf2 && pos);
2319
2320 const T* pcurr1 = buf1;
2321 const T* pend1 = pcurr1 + (*pcurr1 >> 3);
2322 const T* pcurr2 = buf2;
2323 for (++pcurr1, ++pcurr2; pcurr1 <= pend1; ++pcurr1, ++pcurr2)
2324 {
2325 if (*pcurr1 != *pcurr2)
2326 {
2327 *pos = 1 + ((*pcurr1 < *pcurr2) ? *pcurr1 : *pcurr2);
2328 return true;
2329 }
2330 } // for
2331 return false;
2332}
2333
2334// -------------------------------------------------------------------------
2335//
2336
2337/*!
2338 \brief Abstract operation for GAP buffers.
2339 Receives functor F as a template argument
2340 \param dest - destination memory buffer.
2341 \param vect1 - operand 1 GAP encoded buffer.
2342 \param vect1_mask - XOR mask for starting bitflag for vector1
2343 can be 0 or 1 (1 inverts the vector)
2344 \param vect2 - operand 2 GAP encoded buffer.
2345 \param vect2_mask - same as vect1_mask
2346 \param f - operation functor.
2347 \param dlen - destination length after the operation
2348
2349 \note Internal function.
2350 @internal
2351
2352 @ingroup gapfunc
2353*/
2354template<typename T, class F>
2356 const T* BMRESTRICT vect1,
2357 unsigned vect1_mask,
2358 const T* BMRESTRICT vect2,
2359 unsigned vect2_mask,
2360 F& f,
2361 unsigned& dlen) BMNOEXCEPT2
2362{
2363 const T* cur1 = vect1;
2364 const T* cur2 = vect2;
2365
2366 T bitval1 = (T)((*cur1++ & 1) ^ vect1_mask);
2367 T bitval2 = (T)((*cur2++ & 1) ^ vect2_mask);
2368
2369 T bitval = (T) f(bitval1, bitval2);
2370 T bitval_prev = bitval;
2371
2372 T* res = dest;
2373 *res = bitval;
2374 ++res;
2375
2376 T c1 = *cur1; T c2 = *cur2;
2377 while (1)
2378 {
2379 bitval = (T) f(bitval1, bitval2);
2380
2381 // Check if GAP value changes and we need to
2382 // start the next one
2383 //
2384 res += (bitval != bitval_prev);
2385 bitval_prev = bitval;
2386 if (c1 < c2) // (*cur1 < *cur2)
2387 {
2388 *res = c1;
2389 ++cur1; c1 = *cur1;
2390 bitval1 ^= 1;
2391 }
2392 else // >=
2393 {
2394 *res = c2;
2395 if (c2 < c1) // (*cur2 < *cur1)
2396 {
2397 bitval2 ^= 1;
2398 }
2399 else // equal
2400 {
2401 if (c2 == (bm::gap_max_bits - 1))
2402 break;
2403
2404 ++cur1; c1 = *cur1;
2405 bitval1 ^= 1; bitval2 ^= 1;
2406 }
2407 ++cur2; c2 = *cur2;
2408 }
2409 } // while
2410
2411 dlen = (unsigned)(res - dest);
2412 *dest = (T)((*dest & 7) + (dlen << 3));
2413}
2414
2415
2416/*!
2417 \brief Abstract operation for GAP buffers (predicts legth)
2418 Receives functor F as a template argument
2419 \param vect1 - operand 1 GAP encoded buffer.
2420 \param vect2 - operand 2 GAP encoded buffer.
2421 \param f - operation functor.
2422 \param dlen - destination length after the operation
2423 \param limit - maximum target length limit,
2424 returns false if limit is reached
2425 \return true if operation would be successfull or
2426 false if limit reached
2427
2428 \note Internal function.
2429 @internal
2430
2431 @ingroup gapfunc
2432*/
2433template<typename T, class F>
2434bool gap_buff_dry_op(const T* BMRESTRICT vect1,
2435 const T* BMRESTRICT vect2,
2436 F& f,
2437 unsigned& dlen,
2438 unsigned limit) BMNOEXCEPT2
2439{
2440 const T* cur1 = vect1;
2441 const T* cur2 = vect2;
2442
2443 T bitval1 = (T)((*cur1++ & 1));
2444 T bitval2 = (T)((*cur2++ & 1));
2445
2446 T bitval = (T) f(bitval1, bitval2);
2447 T bitval_prev = bitval;
2448
2449 unsigned len = 1;
2450
2451 T c1 = *cur1; T c2 = *cur2;
2452 while (1)
2453 {
2454 bitval = (T) f(bitval1, bitval2);
2455
2456 // Check if GAP value changes and we need to
2457 // start the next one
2458 //
2459 len += (bitval != bitval_prev);
2460 if (len > limit)
2461 return false;
2462 bitval_prev = bitval;
2463 if (c1 < c2)
2464 {
2465 ++cur1; c1 = *cur1;
2466 bitval1 ^= 1;
2467 }
2468 else // >=
2469 {
2470 if (c2 < c1) // (*cur2 < *cur1)
2471 {
2472 bitval2 ^= 1;
2473 }
2474 else // equal
2475 {
2476 if (c2 == (bm::gap_max_bits - 1))
2477 break;
2478
2479 ++cur1; c1 = *cur1;
2480 bitval1 ^= 1; bitval2 ^= 1;
2481 }
2482 ++cur2; c2 = *cur2;
2483 }
2484
2485 } // while
2486
2487 dlen = len;
2488 return true;
2489}
2490
2491
2492/*!
2493 \brief Abstract distance test operation for GAP buffers.
2494 Receives functor F as a template argument
2495 \param vect1 - operand 1 GAP encoded buffer.
2496 \param vect1_mask - XOR mask for starting bitflag for vector1
2497 can be 0 or 1 (1 inverts the vector)
2498 \param vect2 - operand 2 GAP encoded buffer.
2499 \param vect2_mask - same as vect1_mask
2500 \param f - operation functor.
2501 \note Internal function.
2502 \return non zero value if operation result returns any 1 bit
2503
2504 @ingroup gapfunc
2505*/
2506template<typename T, class F>
2507unsigned gap_buff_any_op(const T* BMRESTRICT vect1,
2508 unsigned vect1_mask,
2509 const T* BMRESTRICT vect2,
2510 unsigned vect2_mask,
2511 F f) BMNOEXCEPT2
2512{
2513 const T* cur1 = vect1;
2514 const T* cur2 = vect2;
2515
2516 unsigned bitval1 = (*cur1++ & 1) ^ vect1_mask;
2517 unsigned bitval2 = (*cur2++ & 1) ^ vect2_mask;
2518
2519 unsigned bitval = f(bitval1, bitval2);
2520 if (bitval)
2521 return bitval;
2522 unsigned bitval_prev = bitval;
2523
2524 while (1)
2525 {
2526 bitval = f(bitval1, bitval2);
2527 if (bitval)
2528 return bitval;
2529
2530 if (bitval != bitval_prev)
2531 bitval_prev = bitval;
2532
2533 if (*cur1 < *cur2)
2534 {
2535 ++cur1;
2536 bitval1 ^= 1;
2537 }
2538 else // >=
2539 {
2540 if (*cur2 < *cur1)
2541 {
2542 bitval2 ^= 1;
2543 }
2544 else // equal
2545 {
2546 if (*cur2 == (bm::gap_max_bits - 1))
2547 {
2548 break;
2549 }
2550
2551 ++cur1;
2552 bitval1 ^= 1;
2553 bitval2 ^= 1;
2554 }
2555 ++cur2;
2556 }
2557
2558 } // while
2559
2560 return 0;
2561}
2562
2563
2564
2565/*!
2566 \brief Abstract distance(similarity) operation for GAP buffers.
2567 Receives functor F as a template argument
2568 \param vect1 - operand 1 GAP encoded buffer.
2569 \param vect2 - operand 2 GAP encoded buffer.
2570 \param f - operation functor.
2571 \note Internal function.
2572
2573 @ingroup gapfunc
2574*/
2575template<typename T, class F>
2576unsigned gap_buff_count_op(const T* vect1, const T* vect2, F f) BMNOEXCEPT2
2577{
2578 unsigned count;// = 0;
2579 const T* cur1 = vect1;
2580 const T* cur2 = vect2;
2581
2582 unsigned bitval1 = (*cur1++ & 1);
2583 unsigned bitval2 = (*cur2++ & 1);
2584 unsigned bitval = count = f(bitval1, bitval2);
2585 unsigned bitval_prev = bitval;
2586
2587 //if (bitval) ++count;
2588
2589 T res, res_prev;
2590 res = res_prev = 0;
2591
2592 while (1)
2593 {
2594 bitval = f(bitval1, bitval2);
2595
2596 // Check if GAP value changes and we need to
2597 // start the next one.
2598 if (bitval != bitval_prev)
2599 {
2600 bitval_prev = bitval;
2601 res_prev = res;
2602 }
2603
2604 if (*cur1 < *cur2)
2605 {
2606 res = *cur1;
2607 if (bitval)
2608 {
2609 count += res - res_prev;
2610 res_prev = res;
2611 }
2612 ++cur1; bitval1 ^= 1;
2613 }
2614 else // >=
2615 {
2616 res = *cur2;
2617 if (bitval)
2618 {
2619 count += res - res_prev;
2620 res_prev = res;
2621 }
2622 if (*cur2 < *cur1)
2623 {
2624 bitval2 ^= 1;
2625 }
2626 else // equal
2627 {
2628 if (*cur2 == (bm::gap_max_bits - 1))
2629 break;
2630
2631 ++cur1;
2632 bitval1 ^= 1; bitval2 ^= 1;
2633 }
2634 ++cur2;
2635 }
2636
2637 } // while
2638
2639 return count;
2640}
2641
2642
2643#ifdef __GNUG__
2644#pragma GCC diagnostic push
2645#pragma GCC diagnostic ignored "-Wconversion"
2646#endif
2647
2648/*!
2649 \brief Sets or clears bit in the GAP buffer.
2650
2651 \param val - new bit value
2652 \param buf - GAP buffer.
2653 \param pos - Index of bit to set.
2654 \param is_set - (OUT) flag if bit was actually set.
2655
2656 \return New GAP buffer length.
2657
2658 @ingroup gapfunc
2659*/
2660template<typename T>
2661unsigned gap_set_value(unsigned val,
2662 T* BMRESTRICT buf,
2663 unsigned pos,
2664 unsigned* BMRESTRICT is_set) BMNOEXCEPT
2665{
2667
2668 unsigned curr = bm::gap_bfind(buf, pos, is_set);
2669 T end = (T)(*buf >> 3);
2670 if (*is_set == val)
2671 {
2672 *is_set = 0;
2673 return end;
2674 }
2675 *is_set = 1;
2676
2677 T* pcurr = buf + curr;
2678 T* pprev = pcurr - 1;
2679 T* pend = buf + end;
2680
2681 // Special case, first bit GAP operation. There is no platform beside it.
2682 // initial flag must be inverted.
2683 if (!pos)
2684 {
2685 *buf ^= 1;
2686 if (buf[1]) // We need to insert a 1 bit GAP here
2687 {
2688 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2689 buf[1] = 0;
2690 ++end;
2691 }
2692 else // Only 1 bit in the GAP. We need to delete the first GAP.
2693 {
2694 pprev = buf + 1; pcurr = pprev + 1;
2695 goto copy_gaps;
2696 }
2697 }
2698 else
2699 if (curr > 1 && ((unsigned)(*pprev))+1 == pos) // Left border bit
2700 {
2701 ++(*pprev);
2702 if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2703 {
2704 --end;
2705 if (pcurr != pend) // GAP merge: 2 GAPS to be deleted
2706 {
2707 ++pcurr;
2708 copy_gaps:
2709 --end;
2710 do { *pprev++ = *pcurr++; } while (pcurr < pend);
2711 }
2712 }
2713 }
2714 else
2715 if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2716 {
2717 --(*pcurr);
2718 end += (pcurr == pend);
2719 }
2720 else // Worst case: split current GAP
2721 {
2722 if (*pcurr != bm::gap_max_bits-1) // last gap does not need memmove
2723 ::memmove(pcurr+2, pcurr, (end - curr + 1)*(sizeof(T)));
2724 end += 2;
2725 pcurr[0] = (T)(pos-1);
2726 pcurr[1] = (T)pos;
2727 }
2728
2729 // Set correct length word and last border word
2730 *buf = (T)((*buf & 7) + (end << 3));
2731 buf[end] = bm::gap_max_bits-1;
2732 return end;
2733}
2734
2735/*!
2736 \brief Sets or clears bit in the GAP buffer.
2737
2738 \param val - new bit value
2739 \param buf - GAP buffer.
2740 \param pos - Index of bit to set.
2741
2742 \return New GAP buffer length.
2743
2744 @ingroup gapfunc
2745*/
2746template<typename T>
2747unsigned gap_set_value(unsigned val,
2748 T* BMRESTRICT buf,
2749 unsigned pos) BMNOEXCEPT
2750{
2752 unsigned is_set;
2753 unsigned curr = bm::gap_bfind(buf, pos, &is_set);
2754 T end = (T)(*buf >> 3);
2755 if (is_set == val)
2756 return end;
2757
2758 T* pcurr = buf + curr;
2759 T* pprev = pcurr - 1;
2760 T* pend = buf + end;
2761
2762 // Special case, first bit GAP operation. There is no platform beside it.
2763 // initial flag must be inverted.
2764 if (!pos)
2765 {
2766 *buf ^= 1;
2767 if (buf[1]) // We need to insert a 1 bit GAP here
2768 {
2769 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2770 buf[1] = 0;
2771 ++end;
2772 }
2773 else // Only 1 bit in the GAP. We need to delete the first GAP.
2774 {
2775 pprev = buf + 1; pcurr = pprev + 1;
2776 goto copy_gaps;
2777 }
2778 }
2779 else
2780 if (curr > 1 && ((unsigned)(*pprev))+1 == pos) // Left border bit
2781 {
2782 ++(*pprev);
2783 if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2784 {
2785 --end;
2786 if (pcurr != pend) // GAP merge: 2 GAPS to be deleted
2787 {
2788 ++pcurr;
2789 copy_gaps:
2790 --end;
2791 do { *pprev++ = *pcurr++; } while (pcurr < pend);
2792 }
2793 }
2794 }
2795 else
2796 if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2797 {
2798 --(*pcurr);
2799 end += (pcurr == pend);
2800 }
2801 else // Worst case: split current GAP
2802 {
2803 if (*pcurr != bm::gap_max_bits-1) // last gap does not need memmove
2804 ::memmove(pcurr+2, pcurr, (end - curr + 1)*(sizeof(T)));
2805 end += 2;
2806 pcurr[0] = (T)(pos-1);
2807 pcurr[1] = (T)pos;
2808 }
2809
2810 // Set correct length word and last border word
2811 *buf = (T)((*buf & 7) + (end << 3));
2812 buf[end] = bm::gap_max_bits-1;
2813 return end;
2814}
2815
2816/*!
2817 \brief Add new value to the end of GAP buffer.
2818
2819 \param buf - GAP buffer.
2820 \param pos - Index of bit to set.
2821
2822 \return New GAP buffer length.
2823
2824 @ingroup gapfunc
2825*/
2826template<typename T>
2827unsigned gap_add_value(T* buf, unsigned pos) BMNOEXCEPT
2828{
2830
2831 T end = (T)(*buf >> 3);
2832 T curr = end;
2833 T* pcurr = buf + end;
2834 T* pend = pcurr;
2835 T* pprev = pcurr - 1;
2836
2837 // Special case, first bit GAP operation. There is no platform beside it.
2838 // initial flag must be inverted.
2839 if (!pos)
2840 {
2841 *buf ^= 1;
2842 if ( buf[1] ) // We need to insert a 1 bit platform here.
2843 {
2844 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2845 buf[1] = 0;
2846 ++end;
2847 }
2848 else // Only 1 bit in the GAP. We need to delete the first GAP.
2849 {
2850 pprev = buf + 1; pcurr = pprev + 1;
2851 --end;
2852 do { *pprev++ = *pcurr++; } while (pcurr < pend);
2853 }
2854 }
2855 else if (((unsigned)(*pprev))+1 == pos && (curr > 1) ) // Left border bit
2856 {
2857 ++(*pprev);
2858 if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2859 {
2860 --end;
2861 BM_ASSERT(pcurr == pend);
2862 }
2863 }
2864 else if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2865 {
2866 --(*pcurr);
2867 end += (pcurr == pend);
2868 }
2869 else // Worst case we need to split current block.
2870 {
2871 pcurr[0] = (T)(pos-1);
2872 pcurr[1] = (T)pos;
2873 end = (T)(end+2);
2874 }
2875
2876 // Set correct length word.
2877 *buf = (T)((*buf & 7) + (end << 3));
2878 buf[end] = bm::gap_max_bits - 1;
2879 return end;
2880}
2881
2882#ifdef __GNUG__
2883#pragma GCC diagnostic pop
2884#endif
2885
2886
2887/*!
2888 @brief Right shift GAP block by 1 bit
2889 @param buf - block pointer
2890 @param co_flag - carry over from the previous block
2891 @param new_len - output length of the GAP block after the operation
2892
2893 @return carry over bit (1 or 0)
2894 @ingroup gapfunc
2895*/
2896template<typename T>
2898 unsigned co_flag, unsigned* BMRESTRICT new_len) BMNOEXCEPT
2899{
2900 BM_ASSERT(new_len);
2901 bool co;
2902 // 1: increment all GAP values by 1
2903 {
2904 unsigned bitval = *buf & 1;
2905 if (buf[1] == bm::gap_max_bits-1) // full GAP block
2906 {
2907 co = bitval;
2908 }
2909 else
2910 {
2911 unsigned len = (*buf >> 3);
2912 unsigned i = 1;
2913 for (; i < len; ++i)
2914 {
2915 buf[i]++;
2916 bitval ^= 1;
2917 } // for i
2918 BM_ASSERT(buf[i] == bm::gap_max_bits-1);
2919 if (buf[i-1] == bm::gap_max_bits-1) // last element shifts out
2920 {
2921 // Set correct length word
2922 --len;
2923 *buf = (T)((*buf & 7) + (len << 3));
2924 }
2925 co = bitval;
2926 }
2927 }
2928 // set bit position 0 with carry-in flag
2929 {
2930 unsigned is_set;
2931 *new_len = bm::gap_set_value(co_flag, buf, 0, &is_set);
2932 }
2933 return co;
2934}
2935
2936/*!
2937 @brief Left shift GAP block by 1 bit
2938 @param buf - block pointer
2939 @param co_flag - carry over from the previous block
2940 @param new_len - new length of the block
2941
2942 @return carry over bit (1 or 0)
2943 @ingroup gapfunc
2944*/
2945template<typename T>
2947 unsigned co_flag, unsigned* BMRESTRICT new_len) BMNOEXCEPT
2948{
2949 BM_ASSERT(new_len);
2950 unsigned is_set;
2951
2952 // 1: decrement all GAP values by 1
2953 //
2954 unsigned bitval = *buf & 1;
2955 bool co0 = bitval;
2956
2957 if (!buf[1]) // cannot decrement (corner case)
2958 {
2959 bitval ^= 1;
2960 *new_len = bm::gap_set_value(bitval, buf, 0, &is_set);
2961
2962 BM_ASSERT(is_set);
2963 BM_ASSERT(buf[1]);
2964 BM_ASSERT(bitval == unsigned(*buf & 1u));
2965
2966 if (*new_len == 1)
2967 {
2968 *new_len = bm::gap_set_value(co_flag, buf,
2969 bm::gap_max_bits-1, &is_set);
2970 return co0;
2971 }
2972 }
2973 if (buf[1] != bm::gap_max_bits-1) // full GAP block
2974 {
2975 BM_ASSERT(buf[1]);
2976 unsigned len = (*buf >> 3);
2977 unsigned i = 1;
2978 for (; i < len; ++i)
2979 {
2980 buf[i]--;
2981 bitval ^= 1;
2982 } // for i
2983 BM_ASSERT(buf[i] == bm::gap_max_bits-1);
2984 }
2985 // 2: set last bit position with carry-in flag
2986 //
2987 *new_len = bm::gap_set_value(co_flag, buf, bm::gap_max_bits-1, &is_set);
2988 return co0;
2989}
2990
2991
2992/*!
2993 \brief Convert array to GAP buffer.
2994
2995 \param buf - GAP buffer.
2996 \param arr - array of values to set
2997 \param len - length of the array
2998
2999 \return New GAP buffer length.
3000
3001 @ingroup gapfunc
3002*/
3003
3004template<typename T>
3005unsigned gap_set_array(T* buf, const T* arr, unsigned len) BMNOEXCEPT
3006{
3007 *buf = (T)((*buf & 6u) + (1u << 3)); // gap header setup
3008
3009 T* pcurr = buf + 1;
3010
3011 unsigned i = 0;
3012 T curr = arr[i];
3013 if (curr != 0) // need to add the first gap: (0 to arr[0]-1)
3014 {
3015 *pcurr = (T)(curr - 1);
3016 ++pcurr;
3017 }
3018 else
3019 {
3020 ++(*buf); // GAP starts with 1
3021 }
3022 T prev = curr;
3023 T acc = prev;
3024
3025 for (i = 1; i < len; ++i)
3026 {
3027 curr = arr[i];
3028 if (curr == prev + 1)
3029 {
3030 ++acc;
3031 prev = curr;
3032 }
3033 else
3034 {
3035 *pcurr++ = acc;
3036 acc = curr;
3037 *pcurr++ = (T)(curr-1);
3038 }
3039 prev = curr;
3040 }
3041 *pcurr = acc;
3042 if (acc != bm::gap_max_bits - 1)
3043 {
3044 ++pcurr;
3045 *pcurr = bm::gap_max_bits - 1;
3046 }
3047
3048 unsigned gap_len = unsigned(pcurr - buf);
3049 BM_ASSERT(gap_len == ((gap_len << 3) >> 3));
3050
3051 *buf = (T)((*buf & 7) + (gap_len << 3));
3052 return gap_len+1;
3053}
3054
3055
3056//------------------------------------------------------------------------
3057
3058/**
3059 \brief Compute number of GAPs in bit-array
3060 \param arr - array of BITs
3061 \param len - array length
3062
3063 @ingroup gapfunc
3064*/
3065template<typename T>
3066unsigned bit_array_compute_gaps(const T* arr, unsigned len) BMNOEXCEPT
3067{
3068 unsigned gap_count = 1;
3069 T prev = arr[0];
3070 if (prev > 0)
3071 ++gap_count;
3072 for (unsigned i = 1; i < len; ++i)
3073 {
3074 T curr = arr[i];
3075 if (curr != prev + 1)
3076 {
3077 gap_count += 2;
3078 }
3079 prev = curr;
3080 }
3081 return gap_count;
3082}
3083
3084
3085//------------------------------------------------------------------------
3086
3087/**
3088 \brief Searches for the next 1 bit in the GAP block
3089 \param buf - GAP buffer
3090 \param nbit - bit index to start checking from.
3091 \param prev - returns previously checked value
3092
3093 \return 0 if not found
3094
3095 @ingroup gapfunc
3096*/
3097template<typename T>
3098unsigned gap_block_find(const T* BMRESTRICT buf,
3099 unsigned nbit,
3101{
3103
3104 unsigned bitval;
3105 unsigned gap_idx = bm::gap_bfind(buf, nbit, &bitval);
3106
3107 if (bitval) // positive block.
3108 {
3109 *prev = nbit;
3110 return 1u;
3111 }
3112 unsigned val = buf[gap_idx] + 1;
3113 *prev = val;
3114 return (val != bm::gap_max_bits); // no bug here.
3115}
3116
3117//------------------------------------------------------------------------
3118
3119
3120/*!
3121 \brief Set 1 bit in a block
3122 @ingroup bitfunc
3123*/
3125void set_bit(unsigned* dest, unsigned bitpos) BMNOEXCEPT
3126{
3127 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3128 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3129 nbit &= bm::set_word_mask;
3130 dest[nword] |= unsigned(1u << nbit);
3131}
3132
3133/*!
3134 \brief Set 1 bit in a block
3135 @ingroup bitfunc
3136*/
3138void clear_bit(unsigned* dest, unsigned bitpos) BMNOEXCEPT
3139{
3140 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3141 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3142 nbit &= bm::set_word_mask;
3143 dest[nword] &= ~(unsigned(1u << nbit));
3144}
3145
3146/*!
3147 \brief Test 1 bit in a block
3148
3149 @ingroup bitfunc
3150*/
3152unsigned test_bit(const unsigned* block, unsigned bitpos) BMNOEXCEPT
3153{
3154 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3155 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3156 nbit &= bm::set_word_mask;
3157 return (block[nword] >> nbit) & 1u;
3158}
3159
3160
3161/*!
3162 \brief Sets bits to 1 in the bitblock.
3163 \param dest - Bitset buffer.
3164 \param bitpos - Offset of the start bit.
3165 \param bitcount - number of bits to set.
3166
3167 @ingroup bitfunc
3168*/
3169inline
3170void or_bit_block(unsigned* dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
3171{
3172 const unsigned maskFF = ~0u;
3173
3174 dest += unsigned(bitpos >> bm::set_word_shift); // nword
3175 bitpos &= bm::set_word_mask;
3176
3177 if (bitcount == 1u) // special case (only 1 bit to set)
3178 {
3179 *dest |= (1u << bitpos);
3180 return;
3181 }
3182
3183 if (bitpos) // starting pos is not aligned
3184 {
3185 unsigned mask_r = maskFF << bitpos;
3186 unsigned right_margin = bitpos + bitcount;
3187 if (right_margin < 32)
3188 {
3189 *dest |= (maskFF >> (32 - right_margin)) & mask_r;
3190 return;
3191 }
3192 *dest++ |= mask_r;
3193 bitcount -= 32 - bitpos;
3194 }
3195 for ( ;bitcount >= 64; bitcount-=64, dest+=2)
3196 dest[0] = dest[1] = maskFF;
3197 if (bitcount >= 32)
3198 {
3199 *dest++ = maskFF; bitcount -= 32;
3200 }
3201 if (bitcount)
3202 {
3203 *dest |= maskFF >> (32 - bitcount);
3204 }
3205}
3206
3207
3208/*!
3209 \brief SUB (AND NOT) bit interval to 1 in the bitblock.
3210 \param dest - Bitset buffer.
3211 \param bitpos - Offset of the start bit.
3212 \param bitcount - number of bits to set.
3213
3214 @ingroup bitfunc
3215*/
3216inline
3217void sub_bit_block(unsigned* dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
3218{
3219 const unsigned maskFF = ~0u;
3220
3221 dest += unsigned(bitpos >> bm::set_word_shift); // nword
3222 bitpos &= bm::set_word_mask;
3223
3224 if (bitcount == 1u) // special case (only 1 bit to set)
3225 {
3226 *dest &= ~(1u << bitpos);
3227 return;
3228 }
3229
3230 if (bitpos) // starting pos is not aligned
3231 {
3232 unsigned mask_r = maskFF << bitpos;
3233 unsigned right_margin = bitpos + bitcount;
3234 if (right_margin < 32)
3235 {
3236 *dest &= ~((maskFF >> (32 - right_margin)) & mask_r);
3237 return;
3238 }
3239 *dest++ &= ~mask_r;
3240 bitcount -= 32 - bitpos;
3241 }
3242 for ( ;bitcount >= 64; bitcount-=64, dest+=2)
3243 dest[0] = dest[1] = 0u;
3244 if (bitcount >= 32)
3245 {
3246 *dest++ = 0u; bitcount -= 32;
3247 }
3248 if (bitcount)
3249 {
3250 *dest &= ~(maskFF >> (32 - bitcount));
3251 }
3252}
3253
3254
3255
3256/*!
3257 \brief XOR bit interval to 1 in the bitblock.
3258 \param dest - Bitset buffer.
3259 \param bitpos - Offset of the start bit.
3260 \param bitcount - number of bits to set.
3261
3262 @ingroup bitfunc
3263*/
3264inline void xor_bit_block(unsigned* dest,
3265 unsigned bitpos,
3266 unsigned bitcount) BMNOEXCEPT
3267{
3268 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3269 unsigned nword = unsigned(nbit >> bm::set_word_shift);
3270 nbit &= bm::set_word_mask;
3271
3272 bm::word_t* word = dest + nword;
3273
3274 if (bitcount == 1) // special case (only 1 bit to set)
3275 {
3276 *word ^= unsigned(1 << nbit);
3277 return;
3278 }
3279
3280 if (nbit) // starting position is not aligned
3281 {
3282 unsigned right_margin = nbit + bitcount;
3283
3284 // here we checking if we setting bits only in the current
3285 // word. Example: 00111000000000000000000000000000 (32 bits word)
3286
3287 if (right_margin < 32)
3288 {
3289 unsigned mask =
3291 bm::block_set_table<true>::_left[right_margin-1];
3292 *word ^= mask;
3293 return;
3294 }
3295 *word ^= block_set_table<true>::_right[nbit];
3296 bitcount -= 32 - nbit;
3297 ++word;
3298 }
3299 for ( ;bitcount >= 64; bitcount-=64, word+=2)
3300 {
3301 word[0] ^= ~0u; word[1] ^= ~0u;
3302 }
3303 if (bitcount >= 32)
3304 {
3305 *word++ ^= ~0u; bitcount -= 32;
3306 }
3307 if (bitcount)
3308 *word ^= block_set_table<true>::_left[bitcount-1];
3309}
3310
3311
3312/*!
3313 \brief SUB (AND NOT) GAP block to bitblock.
3314 \param dest - bitblock buffer pointer.
3315 \param pcurr - GAP buffer pointer.
3316
3317 @ingroup gapfunc
3318*/
3319template<typename T>
3320void gap_sub_to_bitset(unsigned* BMRESTRICT dest,
3321 const T* BMRESTRICT pcurr) BMNOEXCEPT
3322{
3323 BM_ASSERT(dest && pcurr);
3324
3325 const T* pend = pcurr + (*pcurr >> 3);
3326 if (*pcurr & 1) // Starts with 1
3327 {
3328 bm::sub_bit_block(dest, 0, 1 + pcurr[1]);
3329 ++pcurr;
3330 }
3331 for (pcurr += 2; pcurr <= pend; pcurr += 2)
3332 {
3333 BM_ASSERT(*pcurr > pcurr[-1]);
3334 bm::sub_bit_block(dest, 1 + pcurr[-1], *pcurr - pcurr[-1]);
3335 }
3336}
3337
3338
3339/*!
3340 \brief SUB (AND NOT) GAP block to bitblock with digest assist
3341
3342 \param dest - bitblock buffer pointer.
3343 \param pcurr - GAP buffer pointer.
3344 \param digest0 - digest of 0 strides inside bit block
3345
3346 @ingroup gapfunc
3347*/
3348template<typename T>
3349void gap_sub_to_bitset(unsigned* BMRESTRICT dest,
3350 const T* BMRESTRICT pcurr, bm::id64_t digest0) BMNOEXCEPT
3351{
3352 BM_ASSERT(dest && pcurr);
3353
3354 const T* pend = pcurr + (*pcurr >> 3);
3355 if (*pcurr & 1) // Starts with 1
3356 {
3357 bool all_zero = bm::check_zero_digest(digest0, 0, pcurr[1]+1);
3358 if (!all_zero)
3359 bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3360 pcurr += 3;
3361 }
3362 else
3363 pcurr += 2;
3364
3365 // wind forward to digest start
3366 {
3367 unsigned tz = bm::count_trailing_zeros_u64(digest0);
3368 unsigned start_pos = tz << set_block_digest_pos_shift;
3369 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0"
3370 {
3371 if (*pcurr >= start_pos)
3372 break;
3373 }
3374 }
3375
3376 unsigned lz = bm::count_leading_zeros_u64(digest0);
3377 unsigned stop_pos = (64u - lz) << set_block_digest_pos_shift;
3378
3379 unsigned bc, pos;
3380 T prev;
3381 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "1" again
3382 {
3383 BM_ASSERT(*pcurr > *(pcurr-1));
3384 prev = pcurr[-1];
3385 bc = *pcurr - prev;
3386 pos = 1u + prev;
3387
3388 bool all_zero = bm::check_zero_digest(digest0, prev, *pcurr);
3389 if (!all_zero)
3390 bm::sub_bit_block(dest, pos, bc);
3391
3392 if (pos > stop_pos)
3393 break; // early break is possible based on digest tail
3394
3395 } // for
3396}
3397
3398
3399
3400/*!
3401 \brief XOR GAP block to bitblock.
3402 \param dest - bitblock buffer pointer.
3403 \param pcurr - GAP buffer pointer.
3404
3405 @ingroup gapfunc
3406*/
3407template<typename T>
3408void gap_xor_to_bitset(unsigned* BMRESTRICT dest,
3409 const T* BMRESTRICT pcurr) BMNOEXCEPT
3410{
3411 BM_ASSERT(dest && pcurr);
3412
3413 const T* pend = pcurr + (*pcurr >> 3);
3414 if (*pcurr & 1) // Starts with 1
3415 {
3416 bm::xor_bit_block(dest, 0, 1 + pcurr[1]);
3417 ++pcurr;
3418 }
3419 for (pcurr += 2; pcurr <= pend; pcurr += 2)
3420 {
3421 BM_ASSERT(*pcurr > pcurr[-1]);
3422 bm::xor_bit_block(dest, 1 + pcurr[-1], *pcurr - pcurr[-1]);
3423 }
3424}
3425
3426
3427/*!
3428 \brief Adds(OR) GAP block to bitblock.
3429 \param dest - bitblock buffer pointer.
3430 \param pcurr - GAP buffer pointer.
3431 \param len - gap length
3432
3433 @ingroup gapfunc
3434*/
3435template<typename T>
3436void gap_add_to_bitset(unsigned* BMRESTRICT dest,
3437 const T* BMRESTRICT pcurr, unsigned len) BMNOEXCEPT
3438{
3439 BM_ASSERT(dest && pcurr);
3440
3441 const T* pend = pcurr + len;
3442 if (*pcurr & 1) // Starts with 1
3443 {
3444 bm::or_bit_block(dest, 0, 1 + pcurr[1]);
3445 pcurr += 3;
3446 }
3447 else
3448 pcurr += 2;
3449
3450 unsigned bc, pos;
3451 for (; pcurr <= pend; )
3452 {
3453 BM_ASSERT(*pcurr > pcurr[-1]);
3454 pos = 1u + pcurr[-1];
3455 bc = *pcurr - pcurr[-1];
3456 pcurr += 2;
3457 bm::or_bit_block(dest, pos, bc);
3458 }
3459}
3460
3461
3462/*!
3463 \brief Adds(OR) GAP block to bitblock.
3464 \param dest - bitblock buffer pointer.
3465 \param pcurr - GAP buffer pointer.
3466
3467 @ingroup gapfunc
3468*/
3469template<typename T>
3470void gap_add_to_bitset(unsigned* BMRESTRICT dest,
3471 const T* BMRESTRICT pcurr) BMNOEXCEPT
3472{
3473 unsigned len = (*pcurr >> 3);
3474 gap_add_to_bitset(dest, pcurr, len);
3475}
3476
3477
3478/*!
3479 \brief ANDs GAP block to bitblock.
3480 \param dest - bitblock buffer pointer.
3481 \param pcurr - GAP buffer pointer.
3482
3483 @ingroup gapfunc
3484*/
3485template<typename T>
3486void gap_and_to_bitset(unsigned* BMRESTRICT dest,
3487 const T* BMRESTRICT pcurr) BMNOEXCEPT
3488{
3489 BM_ASSERT(dest && pcurr);
3490
3491 const T* pend = pcurr + (*pcurr >> 3);
3492 if (!(*pcurr & 1) ) // Starts with 0
3493 {
3494 bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3495 pcurr += 3;
3496 }
3497 else
3498 pcurr += 2;
3499
3500 unsigned bc, pos;
3501 for (; pcurr <= pend; ) // now we are in GAP "0" again
3502 {
3503 BM_ASSERT(*pcurr > *(pcurr-1));
3504 pos = 1u + pcurr[-1];
3505 bc = *pcurr - pcurr[-1];
3506 pcurr += 2;
3507 bm::sub_bit_block(dest, pos, bc);
3508 }
3509}
3510
3511
3512/*!
3513 \brief ANDs GAP block to bitblock with digest assist
3514 \param dest - bitblock buffer pointer.
3515 \param pcurr - GAP buffer pointer.
3516 \param digest0 - digest of 0 strides for the destination
3517
3518 @ingroup gapfunc
3519*/
3520template<typename T>
3521void gap_and_to_bitset(unsigned* BMRESTRICT dest,
3522 const T* BMRESTRICT pcurr, bm::id64_t digest0) BMNOEXCEPT
3523{
3524 BM_ASSERT(dest && pcurr);
3525 if (!digest0)
3526 return;
3527
3528 const T* pend = pcurr + (*pcurr >> 3);
3529 if (!(*pcurr & 1) ) // Starts with 0
3530 {
3531 bool all_zero = bm::check_zero_digest(digest0, 0, pcurr[1]+1);
3532 if (!all_zero)
3533 bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3534 pcurr += 3;
3535 }
3536 else
3537 pcurr += 2;
3538
3539 // wind forward to digest start
3540 {
3541 unsigned tz = bm::count_trailing_zeros_u64(digest0);
3542 unsigned start_pos = tz << set_block_digest_pos_shift;
3543 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0"
3544 {
3545 if (*pcurr >= start_pos)
3546 break;
3547 }
3548 }
3549
3550 unsigned lz = bm::count_leading_zeros_u64(digest0);
3551 unsigned stop_pos = (64u - lz) << set_block_digest_pos_shift;
3552
3553 unsigned bc, pos;
3554 T prev;
3555 for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0" again
3556 {
3557 BM_ASSERT(*pcurr > *(pcurr-1));
3558
3559 prev = pcurr[-1];
3560 bc = *pcurr - prev;
3561 pos = 1u + prev;
3562
3563 bool all_zero = bm::check_zero_digest(digest0, prev, *pcurr);
3564 if (!all_zero)
3565 bm::sub_bit_block(dest, pos, bc);
3566
3567 if (pos > stop_pos) // early break is possible based on digest tail
3568 break;
3569
3570 } // for
3571}
3572
3573
3574/*!
3575 \brief Compute bitcount of bit block AND masked by GAP block
3576 \param block - bitblock buffer pointer
3577 \param pcurr - GAP buffer pointer
3578 \return bitcount - cardinality of the AND product
3579
3580 @ingroup gapfunc
3581*/
3582template<typename T>
3584 const T* BMRESTRICT pcurr) BMNOEXCEPT
3585{
3586 BM_ASSERT(block);
3587 const T* pend = pcurr + (*pcurr >> 3);
3588 bm::id_t count = 0;
3589 if (*pcurr & 1) // Starts with 1
3590 {
3591 count += bm::bit_block_calc_count_range(block, 0, pcurr[1]);
3592 ++pcurr;
3593 }
3594 for (pcurr +=2 ;pcurr <= pend; pcurr += 2)
3595 {
3596 count += bm::bit_block_calc_count_range(block, pcurr[-1]+1, *pcurr);
3597 }
3598 return count;
3599}
3600
3601
3602/*!
3603 \brief Bitcount test of bit block AND masked by GAP block.
3604 \param block - bitblock buffer pointer
3605 \param pcurr - GAP buffer pointer
3606 \return non-zero value if AND produces any result
3607
3608 @ingroup gapfunc
3609*/
3610template<typename T>
3612 const T* BMRESTRICT pcurr) BMNOEXCEPT
3613{
3614 BM_ASSERT(block);
3615
3616 const T* pend = pcurr + (*pcurr >> 3);
3617 bm::id_t count = 0;
3618 if (*pcurr & 1) // Starts with 1
3619 {
3620 count = bm::bit_block_any_range(block, 0, pcurr[1]);
3621 ++pcurr;
3622 }
3623 for (pcurr +=2 ;!count && pcurr <= pend; pcurr += 2)
3624 {
3625 count = bm::bit_block_any_range(block, pcurr[-1]+1, *pcurr);
3626 }
3627 return count;
3628}
3629
3630
3631
3632/*!
3633 \brief Compute bitcount of bit block SUB masked by GAP block.
3634 \param block - bitblock buffer pointer.
3635 \param buf - GAP buffer pointer.
3636 \return bit-count result of AND NOT operation
3637
3638 @ingroup gapfunc
3639*/
3640template<typename T>
3642 const T* BMRESTRICT buf) BMNOEXCEPT
3643{
3644 BM_ASSERT(block);
3645
3646 const T* pcurr = buf;
3647 const T* pend = pcurr + (*pcurr >> 3);
3648 ++pcurr;
3649
3650 bm::id_t count = 0;
3651
3652 if (!(*buf & 1)) // Starts with 0
3653 {
3654 count += bit_block_calc_count_range(block, 0, *pcurr);
3655 ++pcurr;
3656 }
3657 ++pcurr; // now we are in GAP "0" again
3658
3659 for (;pcurr <= pend; pcurr+=2)
3660 {
3661 count += bm::bit_block_calc_count_range(block, *(pcurr-1)+1, *pcurr);
3662 }
3663 return count;
3664}
3665
3666
3667/*!
3668 \brief Compute bitcount test of bit block SUB masked by GAP block
3669 \param block - bitblock buffer pointer
3670 \param buf - GAP buffer pointer
3671 \return non-zero value if AND NOT produces any 1 bits
3672
3673 @ingroup gapfunc
3674*/
3675template<typename T>
3677 const T* BMRESTRICT buf) BMNOEXCEPT
3678{
3679 BM_ASSERT(block);
3680
3681 const T* pcurr = buf;
3682 const T* pend = pcurr + (*pcurr >> 3);
3683 ++pcurr;
3684
3685 bm::id_t count = 0;
3686
3687 if (!(*buf & 1)) // Starts with 0
3688 {
3689 count += bit_block_any_range(block, 0, *pcurr);
3690 if (count)
3691 return count;
3692 ++pcurr;
3693 }
3694 ++pcurr; // now we are in GAP "0" again
3695
3696 for (; !count && pcurr <= pend; pcurr+=2)
3697 {
3698 count += bm::bit_block_any_range(block, *(pcurr-1)+1, *pcurr);
3699 }
3700 return count;
3701}
3702
3703
3704
3705/*!
3706 \brief Compute bitcount of bit block XOR masked by GAP block
3707 \param block - bitblock buffer pointer
3708 \param buf - GAP buffer pointer
3709 \return bit count value of XOR operation
3710
3711 @ingroup gapfunc
3712*/
3713template<typename T>
3715 const T* BMRESTRICT buf) BMNOEXCEPT
3716{
3717 BM_ASSERT(block);
3718
3719 const T* pcurr = buf;
3720 const T* pend = pcurr + (*pcurr >> 3);
3721 ++pcurr;
3722
3723 unsigned bitval = *buf & 1;
3724
3725 bm::id_t count = bm::bit_block_calc_count_range(block, 0, *pcurr);
3726 if (bitval)
3727 {
3728 count = *pcurr + 1 - count;
3729 }
3730
3731 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
3732 {
3733 T prev = (T)(*(pcurr-1)+1);
3734 bm::id_t c = bit_block_calc_count_range(block, prev, *pcurr);
3735
3736 if (bitval) // 1 gap; means Result = Total_Bits - BitCount;
3737 c = (*pcurr - prev + 1) - c;
3738 count += c;
3739 }
3740 return count;
3741}
3742
3743/*!
3744 \brief Compute bitcount test of bit block XOR masked by GAP block.
3745 \param block - bitblock buffer pointer
3746 \param buf - GAP buffer pointer
3747 \return non-zero value if XOR returns anything
3748
3749 @ingroup gapfunc
3750*/
3751template<typename T>
3753 const T* BMRESTRICT buf) BMNOEXCEPT
3754{
3755 BM_ASSERT(block);
3756
3757 const T* pcurr = buf;
3758 const T* pend = pcurr + (*pcurr >> 3);
3759 ++pcurr;
3760
3761 unsigned bitval = *buf & 1;
3762
3763 bm::id_t count = bit_block_any_range(block, 0, *pcurr);
3764 if (bitval)
3765 count = *pcurr + 1 - count;
3766
3767 for (bitval^=1, ++pcurr; !count && pcurr <= pend; bitval^=1, ++pcurr)
3768 {
3769 T prev = (T)(*(pcurr-1)+1);
3770 bm::id_t c = bit_block_any_range(block, prev, *pcurr);
3771
3772 if (bitval) // 1 gap; means Result = Total_Bits - BitCount;
3773 c = (*pcurr - prev + 1) - c;
3774 count += c;
3775 }
3776 return count;
3777}
3778
3779
3780
3781/*!
3782 \brief Compute bitcount of bit block OR masked by GAP block.
3783 \param block - bitblock buffer pointer.
3784 \param buf - GAP buffer pointer.
3785 \return bit count of OR
3786
3787 @ingroup gapfunc
3788*/
3789template<typename T>
3791 const T* BMRESTRICT buf) BMNOEXCEPT
3792{
3793 BM_ASSERT(block);
3794 const T* pcurr = buf;
3795 const T* pend = pcurr + (*pcurr >> 3);
3796 ++pcurr;
3797
3798 unsigned bitval = *buf & 1;
3799
3800 bm::id_t count = bitval ? *pcurr + 1
3801 : bm::bit_block_calc_count_range(block, 0, *pcurr);
3802 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
3803 {
3804 T prev = (T)(*(pcurr-1)+1);
3805 bm::id_t c =
3806 bitval ? (*pcurr - prev + 1)
3807 : bm::bit_block_calc_count_range(block, prev, *pcurr);
3808 count += c;
3809 }
3810 return count;
3811}
3812
3813/*!
3814 \brief Compute bitcount test of bit block OR masked by GAP block
3815 \param block - bitblock buffer pointer
3816 \param buf - GAP buffer pointer
3817 \return non zero value if union (OR) returns anything
3818
3819 @ingroup gapfunc
3820*/
3821template<typename T>
3823 const T* BMRESTRICT buf) BMNOEXCEPT
3824{
3825 bool b = !bm::gap_is_all_zero(buf) ||
3826 !bm::bit_is_all_zero(block);
3827 return b;
3828}
3829
3830
3831
3832/*!
3833 \brief Bitblock memset operation.
3834
3835 \param dst - destination block.
3836 \param value - value to set.
3837
3838 @ingroup bitfunc
3839*/
3840inline
3842{
3843#ifdef BMVECTOPT
3844 VECT_SET_BLOCK(dst, value);
3845#else
3846 ::memset(dst, int(value), bm::set_block_size * sizeof(bm::word_t));
3847#endif
3848}
3849
3850
3851/*!
3852 \brief GAP block to bitblock conversion.
3853 \param dest - bitblock buffer pointer.
3854 \param buf - GAP buffer pointer.
3855
3856 @ingroup gapfunc
3857*/
3858template<typename T>
3860 const T* BMRESTRICT buf) BMNOEXCEPT
3861{
3862 bm::bit_block_set(dest, 0);
3863 bm::gap_add_to_bitset(dest, buf);
3864}
3865
3866
3867
3868/*!
3869 \brief Smart GAP block to bitblock conversion.
3870
3871 Checks if GAP block is ALL-ZERO or ALL-ON. In those cases returns
3872 pointer on special static bitblocks.
3873
3874 \param dest - bitblock buffer pointer.
3875 \param buf - GAP buffer pointer.
3876 \param set_max - max possible bitset length
3877
3878 @ingroup gapfunc
3879*/
3880template<typename T>
3882 const T* BMRESTRICT buf,
3883 id_t set_max) BMNOEXCEPT
3884{
3885 if (buf[1] == set_max - 1)
3886 return (buf[0] & 1) ? FULL_BLOCK_REAL_ADDR : 0;
3887 bm::gap_convert_to_bitset(dest, buf);
3888 return dest;
3889}
3890
3891
3892/*!
3893 \brief Calculates sum of all words in GAP block. (For debugging purposes)
3894 \note For debugging and testing ONLY.
3895 \param buf - GAP buffer pointer.
3896 \return Sum of all words.
3897
3898 @ingroup gapfunc
3899 @internal
3900*/
3901template<typename T>
3902unsigned gap_control_sum(const T* buf) BMNOEXCEPT
3903{
3904 unsigned end = *buf >> 3;
3905
3906 const T* pcurr = buf;
3907 const T* pend = pcurr + (*pcurr >> 3);
3908 ++pcurr;
3909
3910 if (*buf & 1) // Starts with 1
3911 {
3912 ++pcurr;
3913 }
3914 ++pcurr; // now we are in GAP "1" again
3915 while (pcurr <= pend)
3916 {
3917 BM_ASSERT(*pcurr > *(pcurr-1));
3918 pcurr += 2;
3919 }
3920 return buf[end];
3921}
3922
3923
3924/*!
3925 \brief Sets all bits to 0 or 1 (GAP)
3926 \param buf - GAP buffer pointer.
3927 \param set_max - max possible bitset length
3928 \param value - value to set
3929
3930 @ingroup gapfunc
3931*/
3932template<class T>
3933void gap_set_all(T* buf, unsigned set_max, unsigned value) BMNOEXCEPT
3934{
3935 BM_ASSERT(value == 0 || value == 1);
3936 *buf = (T)((*buf & 6u) + (1u << 3) + value);
3937 *(++buf) = (T)(set_max - 1);
3938}
3939
3940
3941/*!
3942 \brief Init gap block so it has block in it (can be whole block)
3943 \param buf - GAP buffer pointer.
3944 \param from - one block start
3945 \param to - one block end
3946 \param value - (block value)1 or 0
3947
3948 @ingroup gapfunc
3949*/
3950template<class T>
3952 T from,
3953 T to,
3954 T value) BMNOEXCEPT
3955{
3956 BM_ASSERT(value == 0 || value == 1);
3957 const unsigned set_max = bm::bits_in_block;
3958
3959 unsigned gap_len;
3960 if (from == 0)
3961 {
3962 if (to == set_max - 1)
3963 {
3964 bm::gap_set_all(buf, set_max, value);
3965 }
3966 else
3967 {
3968 gap_len = 2;
3969 buf[1] = (T)to;
3970 buf[2] = (T)(set_max - 1);
3971 buf[0] = (T)((*buf & 6u) + (gap_len << 3) + value);
3972 }
3973 return;
3974 }
3975 // from != 0
3976
3977 value = !value;
3978 if (to == set_max - 1)
3979 {
3980 gap_len = 2;
3981 buf[1] = (T)(from - 1);
3982 buf[2] = (T)(set_max - 1);
3983 }
3984 else
3985 {
3986 gap_len = 3;
3987 buf[1] = (T) (from - 1);
3988 buf[2] = (T) to;
3989 buf[3] = (T)(set_max - 1);
3990 }
3991 buf[0] = (T)((*buf & 6u) + (gap_len << 3) + value);
3992}
3993
3994
3995/*!
3996 \brief Inverts all bits in the GAP buffer.
3997 \param buf - GAP buffer pointer.
3998
3999 @ingroup gapfunc
4000*/
4001template<typename T> void gap_invert(T* buf) BMNOEXCEPT
4002{
4003 *buf ^= 1;
4004}
4005
4006
4007#ifdef __GNUG__
4008#pragma GCC diagnostic push
4009#pragma GCC diagnostic ignored "-Wconversion"
4010#endif
4011
4012/*!
4013 \brief Sets GAP block capacity level.
4014 \param buf - GAP buffer pointer.
4015 \param level new GAP block capacity level.
4016
4017 @ingroup gapfunc
4018*/
4019template<typename T>
4020void set_gap_level(T* buf, int level) BMNOEXCEPT
4021{
4022 BM_ASSERT(level >= 0);
4023 BM_ASSERT(unsigned(level) < bm::gap_levels);
4024
4025 *buf = (T)(((level & 3) << 1) | (*buf & 1) | (*buf & ~7));
4026}
4027#ifdef __GNUG__
4028#pragma GCC diagnostic pop
4029#endif
4030
4031
4032
4033/*!
4034 \brief Calculates GAP block capacity level.
4035 \param len - GAP buffer length.
4036 \param glevel_len - GAP lengths table
4037 \return GAP block capacity level.
4038 -1 if block does not fit any level.
4039 @ingroup gapfunc
4040*/
4041template<typename T>
4042int gap_calc_level(unsigned len, const T* glevel_len) BMNOEXCEPT
4043{
4044 if (len <= unsigned(glevel_len[0]-4)) return 0;
4045 if (len <= unsigned(glevel_len[1]-4)) return 1;
4046 if (len <= unsigned(glevel_len[2]-4)) return 2;
4047 if (len <= unsigned(glevel_len[3]-4)) return 3;
4048
4050 return -1;
4051}
4052
4053/*! @brief Returns number of free elements in GAP block array.
4054 Difference between GAP block capacity on this level and actual GAP length.
4055
4056 @param buf - GAP buffer pointer
4057 @param glevel_len - GAP lengths table
4058
4059 @return Number of free GAP elements
4060 @ingroup gapfunc
4061*/
4062template<typename T>
4063inline unsigned gap_free_elements(const T* BMRESTRICT buf,
4064 const T* BMRESTRICT glevel_len) BMNOEXCEPT
4065{
4066 unsigned len = bm::gap_length(buf);
4067 unsigned capacity = bm::gap_capacity(buf, glevel_len);
4068 return capacity - len;
4069}
4070
4071/*!
4072 \brief Lexicographical comparison of BIT buffers.
4073 \param buf1 - First buffer pointer.
4074 \param buf2 - Second buffer pointer.
4075 \param len - Buffer length in elements (T).
4076 \return <0 - less, =0 - equal, >0 - greater.
4077
4078 @ingroup bitfunc
4079*/
4080template<typename T>
4081int bitcmp(const T* buf1, const T* buf2, unsigned len) BMNOEXCEPT
4082{
4083 BM_ASSERT(len);
4084 const T* pend1 = buf1 + len;
4085 do
4086 {
4087 T w1 = *buf1++;
4088 T w2 = *buf2++;
4089 T diff = w1 ^ w2;
4090 if (diff)
4091 return (w1 & diff & -diff) ? 1 : -1;
4092 } while (buf1 < pend1);
4093 return 0;
4094}
4095
4096/*!
4097 \brief Find first bit which is different between two bit-blocks
4098 \param blk1 - block 1
4099 \param blk2 - block 2
4100 \param pos - out - position of difference (undefined if blocks are equal)
4101 \return true if difference was found
4102
4103 @ingroup bitfunc
4104*/
4105inline
4107 const bm::word_t* BMRESTRICT blk2,
4108 unsigned* BMRESTRICT pos) BMNOEXCEPT
4109{
4110 BM_ASSERT(blk1 && blk2 && pos);
4111#ifdef VECT_BIT_FIND_DIFF
4112 bool f = VECT_BIT_FIND_DIFF(blk1, blk2, pos);
4113 return f;
4114#else
4115#ifdef BM64OPT
4116 BM_ASSERT(sizeof(bm::wordop_t) == 8);
4117
4118 const bm::wordop_t* b1 = (const bm::wordop_t*) blk1;
4119 const bm::wordop_t* b2 = (const bm::wordop_t*) blk2;
4120
4121 for (unsigned i = 0; i < bm::set_block_size/2; ++i)
4122 {
4123 bm::wordop_t w1 = b1[i]; bm::wordop_t w2 = b2[i];
4124 bm::wordop_t diff = w1 ^ w2;
4125 if (diff)
4126 {
4127 unsigned idx = bm::count_trailing_zeros_u64(diff);
4128 *pos = unsigned(idx + (i * 8u * unsigned(sizeof(bm::wordop_t))));
4129 return true;
4130 }
4131 } // for
4132#else
4133 for (unsigned i = 0; i < bm::set_block_size; ++i)
4134 {
4135 bm::word_t w1 = blk1[i]; bm::word_t w2 = blk2[i];
4136 bm::word_t diff = w1 ^ w2;
4137 if (diff)
4138 {
4139 unsigned idx = bm::bit_scan_forward32(diff); // trailing zeros
4140 *pos = unsigned(idx + (i * 8u * sizeof(bm::word_t)));
4141 return true;
4142 }
4143 } // for
4144#endif
4145#endif
4146 return false;
4147}
4148
4149
4150#ifndef BMAVX2OPT
4151
4152/*!
4153 \brief Converts bit block to GAP.
4154 \param dest - Destinatio GAP buffer.
4155 \param block - Source bitblock buffer.
4156 \param dest_len - length of the destination buffer.
4157 \return New length of GAP block or 0 if conversion failed
4158 (insufficicent space).
4159
4160 @ingroup gapfunc
4161*/
4162inline
4164 const unsigned* BMRESTRICT block,
4165 unsigned dest_len) BMNOEXCEPT
4166{
4167 const unsigned* BMRESTRICT block_end = block + bm::set_block_size;
4168 gap_word_t* BMRESTRICT pcurr = dest;
4169 gap_word_t* BMRESTRICT end = dest + dest_len; (void)end;
4170
4171 unsigned bitval = (*block) & 1u;
4172 *pcurr++ = bm::gap_word_t(bitval);
4173 *pcurr = 0;
4174 unsigned bit_idx = 0;
4175
4176 do
4177 {
4178 unsigned val = *block;
4179 while (!val || val == ~0u)
4180 {
4181 if (bitval != unsigned(bool(val)))
4182 {
4183 *pcurr++ = (gap_word_t)(bit_idx-1);
4184 bitval ^= 1u;
4185 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4186 BM_ASSERT(pcurr != end);
4187 }
4188 bit_idx += unsigned(sizeof(*block) * 8);
4189 if (++block >= block_end)
4190 goto complete;
4191 val = *block;
4192 } // while
4193
4194 // process "0100011" word
4195 //
4196 unsigned bits_consumed = 0;
4197 do
4198 {
4199 unsigned tz = 1u;
4200 if (bitval != (val & 1u))
4201 {
4202 *pcurr++ = (gap_word_t)(bit_idx-1);
4203 bitval ^= 1u;
4204 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4205 BM_ASSERT(pcurr != end);
4206 }
4207 else // match, find the next idx
4208 {
4209 tz = bm::bit_scan_forward32(bitval ? ~val : val);
4210 // possible alternative:
4211 // tz = bm::count_trailing_zeros(bitval ? ~val : val);
4212 }
4213
4214 bits_consumed += tz;
4215 bit_idx += tz;
4216 val >>= tz;
4217
4218 if (!val)
4219 {
4220 if (bits_consumed < 32u)
4221 {
4222 *pcurr++ = (gap_word_t)(bit_idx-1);
4223 bitval ^= 1u;
4224 bit_idx += 32u - bits_consumed;
4225 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4226 BM_ASSERT(pcurr != end);
4227 }
4228 break;
4229 }
4230 } while (1);
4231
4232 } while(++block < block_end);
4233
4234complete:
4235 *pcurr = (gap_word_t)(bit_idx-1);
4236 unsigned len = (unsigned)(pcurr - dest);
4237 *dest = (gap_word_t)((*dest & 7) + (len << 3));
4238 return len;
4239}
4240#endif
4241
4242/**
4243 Convert bit block to GAP representation
4244 @internal
4245 @ingroup bitfunc
4246*/
4247inline
4249 const unsigned* BMRESTRICT block,
4250 unsigned dest_len) BMNOEXCEPT
4251{
4252#if defined(VECT_BIT_TO_GAP)
4253 return VECT_BIT_TO_GAP(dest, block, dest_len);
4254#else
4255 return bm::bit_block_to_gap(dest, block, dest_len);
4256#endif
4257}
4258
4259
4260/*!
4261 \brief Iterate gap block as delta-bits with a functor
4262 @ingroup gapfunc
4263*/
4264template<class T, class F>
4265void for_each_gap_dbit(const T* buf, F& func)
4266{
4267 const T* pcurr = buf;
4268 const T* pend = pcurr + (*pcurr >> 3);
4269
4270 ++pcurr;
4271
4272 unsigned prev = 0;
4273 unsigned first_inc;
4274
4275 if (*buf & 1)
4276 {
4277 first_inc = 0;
4278 unsigned to = *pcurr;
4279 for (unsigned i = 0; i <= to; ++i)
4280 {
4281 func(1);
4282 }
4283 prev = to;
4284 ++pcurr;
4285 }
4286 else
4287 {
4288 first_inc = 1;
4289 }
4290 ++pcurr; // set GAP to 1
4291
4292 while (pcurr <= pend)
4293 {
4294 unsigned from = *(pcurr-1)+1;
4295 unsigned to = *pcurr;
4296 if (first_inc)
4297 {
4298 func(from - prev + first_inc);
4299 first_inc = 0;
4300 }
4301 else
4302 {
4303 func(from - prev);
4304 }
4305
4306 for (unsigned i = from+1; i <= to; ++i)
4307 {
4308 func(1);
4309 }
4310 prev = to;
4311 pcurr += 2; // jump to the next positive GAP
4312 }
4313}
4314
4315/*!
4316 \brief Convert gap block into array of ints corresponding to 1 bits
4317 @ingroup gapfunc
4318*/
4319template<typename D, typename T>
4321 const T* BMRESTRICT buf,
4322 unsigned dest_len,
4323 bool invert = false) BMNOEXCEPT
4324{
4325 const T* BMRESTRICT pcurr = buf;
4326 const T* pend = pcurr + (*pcurr >> 3);
4327
4328 D* BMRESTRICT dest_curr = dest;
4329 ++pcurr;
4330
4331 int bitval = (*buf) & 1;
4332 if (invert)
4333 bitval = !bitval; // invert the GAP buffer
4334
4335 if (bitval)
4336 {
4337 if (unsigned(*pcurr + 1) >= dest_len)
4338 return 0; // insufficient space
4339 dest_len -= *pcurr;
4340 T to = *pcurr;
4341 for (T i = 0; ;++i)
4342 {
4343 *dest_curr++ = i;
4344 if (i == to) break;
4345 }
4346 ++pcurr;
4347 }
4348 ++pcurr; // set GAP to 1
4349
4350 while (pcurr <= pend)
4351 {
4352 unsigned pending = *pcurr - *(pcurr-1);
4353 if (pending >= dest_len)
4354 return 0;
4355 dest_len -= pending;
4356 T from = (T)(*(pcurr-1)+1);
4357 T to = *pcurr;
4358 for (T i = from; ;++i)
4359 {
4360 *dest_curr++ = i;
4361 if (i == to) break;
4362 }
4363 pcurr += 2; // jump to the next positive GAP
4364 }
4365 return (D) (dest_curr - dest);
4366}
4367
4368
4369
4370/*!
4371 @brief Bitcount for bit block
4372
4373 Function calculates number of 1 bits in the given array of words.
4374 Make sure the addresses are aligned.
4375
4376 @ingroup bitfunc
4377*/
4378inline
4380{
4381 const bm::word_t* block_end = block + bm::set_block_size;
4382 bm::id_t count = 0;
4383
4384#ifdef BMVECTOPT
4385 count = VECT_BITCOUNT(block, block_end);
4386#else
4387#ifdef BM64OPT
4388 // 64-bit optimized algorithm. No sparse vect opt.
4389 // instead it uses 4-way parallel pipelined version
4390
4391 const bm::id64_t* b1 = (bm::id64_t*) block;
4392 const bm::id64_t* b2 = (bm::id64_t*) block_end;
4393 do
4394 {
4395 bm::id64_t x = b1[0];
4396 bm::id64_t y = b1[1];
4397 bm::id64_t u = b1[2];
4398 bm::id64_t v = b1[3];
4399
4400 if (x | y | u | v)
4401 {
4402 unsigned c = bitcount64_4way(x, y, u, v);
4403 BM_ASSERT(c);
4404 count += c;
4405 }
4406 b1 += 4;
4407 } while (b1 < b2);
4408#else
4409 // For 32 bit code the fastest method is
4410 // to use bitcount table for each byte in the block.
4411 // As optimization for sparse bitsets used bits accumulator
4412 // to collect ON bits using bitwise OR.
4413 bm::word_t acc = *block++;
4414 do
4415 {
4416 bm::word_t in = *block++;
4417 bm::word_t acc_prev = acc;
4418 acc |= in;
4419 if (acc_prev &= in) // accumulator miss: counting bits
4420 {
4421 BM_INCWORD_BITCOUNT(count, acc);
4422 acc = acc_prev;
4423 }
4424 } while (block < block_end);
4425
4426 BM_INCWORD_BITCOUNT(count, acc); // count-in remaining accumulator
4427
4428#endif
4429#endif
4430 return count;
4431}
4432
4433/*!
4434 @brief Bitcount for bit block
4435
4436 Function calculates number of 1 bits in the given array of words.
4437 uses digest to understand zero areas
4438
4439 @ingroup bitfunc
4440*/
4441inline
4443 bm::id64_t digest) BMNOEXCEPT
4444{
4445 bm::id_t count = 0;
4446 bm::id64_t d = digest;
4447 while (d)
4448 {
4449 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
4450
4451 unsigned wave = bm::word_bitcount64(t - 1);
4452 unsigned off = wave * bm::set_block_digest_wave_size;
4453
4455 (const bm::bit_block_t::bunion_t*)(&block[off]);
4456 unsigned j = 0;
4457 do
4458 {
4459 count += bm::word_bitcount64(src_u->w64[j+0]) +
4460 bm::word_bitcount64(src_u->w64[j+1]) +
4461 bm::word_bitcount64(src_u->w64[j+2]) +
4462 bm::word_bitcount64(src_u->w64[j+3]);
4463 j += 4;
4464 } while (j < bm::set_block_digest_wave_size/2);
4465
4466 d = bm::bmi_bslr_u64(d); // d &= d - 1;
4467 } // while
4468 return count;
4469}
4470
4471
4472
4473/*!
4474 @brief Bitcount for bit string
4475
4476 Added for back-compatibility purposes, not block aligned,
4477 not SIMD accelerated
4478
4479 @ingroup bitfunc
4480*/
4481inline
4483 const bm::word_t* block_end) BMNOEXCEPT
4484{
4485 bm::id_t count = 0;
4486 bm::word_t acc = *block++;
4487 do
4488 {
4489 bm::word_t in = *block++;
4490 bm::word_t acc_prev = acc;
4491 acc |= in;
4492 if (acc_prev &= in) // accumulator miss: counting bits
4493 {
4494 BM_INCWORD_BITCOUNT(count, acc);
4495 acc = acc_prev;
4496 }
4497 } while (block < block_end);
4498
4499 BM_INCWORD_BITCOUNT(count, acc); // count-in remaining accumulator
4500 return count;
4501}
4502
4503
4504
4505/*!
4506 Function calculates number of times when bit value changed
4507 (1-0 or 0-1).
4508
4509 For 001 result is 2
4510 010 - 3
4511 011 - 2
4512 111 - 1
4513
4514 @ingroup bitfunc
4515*/
4516inline
4518{
4519 unsigned count = 1;
4520 w ^= (w >> 1);
4521
4522 count += bm::word_bitcount(w);
4523 count -= (w >> ((sizeof(w) * 8) - 1));
4524 return count;
4525}
4526
4527
4528/*!
4529 Function calculates number of times when bit value changed
4530 @internal
4531*/
4532inline
4533unsigned bit_block_change32(const bm::word_t* block, unsigned size) BMNOEXCEPT
4534{
4535 unsigned gap_count = 1;
4536
4537 bm::word_t w, w0, w_prev, w_l;
4538 w = w0 = *block;
4539
4540 const int w_shift = int(sizeof(w) * 8 - 1);
4541 w ^= (w >> 1);
4542 BM_INCWORD_BITCOUNT(gap_count, w);
4543 gap_count -= (w_prev = (w0 >> w_shift)); // negative value correction
4544
4545 const bm::word_t* block_end = block + size; // bm::set_block_size;
4546 for (++block; block < block_end; ++block)
4547 {
4548 w = w0 = *block;
4549 ++gap_count;
4550 if (!w)
4551 {
4552 gap_count -= !w_prev;
4553 w_prev = 0;
4554 }
4555 else
4556 {
4557 w ^= (w >> 1);
4558 BM_INCWORD_BITCOUNT(gap_count, w);
4559
4560 w_l = w0 & 1;
4561 gap_count -= (w0 >> w_shift); // negative value correction
4562 gap_count -= !(w_prev ^ w_l); // word border correction
4563
4564 w_prev = (w0 >> w_shift);
4565 }
4566 } // for
4567 return gap_count;
4568}
4569
4570
4571/*!
4572 Function calculates basic bit-block statistics
4573 number of times when bit value changed (GAPS)
4574 and population count
4575 @param block - bit-block pointer
4576 @param gc - [output] gap_count
4577 @param bc - [output] bit count
4578 @internal
4579*/
4580inline
4582 unsigned* BMRESTRICT gc, unsigned* BMRESTRICT bc) BMNOEXCEPT
4583{
4584 BM_ASSERT(gc);
4585 BM_ASSERT(bc);
4586
4587 #ifdef VECT_BLOCK_CHANGE_BC
4588 VECT_BLOCK_CHANGE_BC(block, gc, bc);
4589 #else
4591 *bc = bm::bit_block_count(block);
4592 #endif
4593}
4594
4595
4596
4597/*!
4598 Function calculates number of times when bit value changed
4599 (1-0 or 0-1) in the bit block.
4600
4601 @param block - bit-block start pointer
4602 @return number of 1-0, 0-1 transitions
4603
4604 @ingroup bitfunc
4605*/
4606inline
4608{
4609#if defined(VECT_BLOCK_CHANGE)
4611#else
4613#endif
4614}
4615
4616/*!
4617 Check if all bits are 1 in [left, right] range
4618 @ingroup bitfunc
4619*/
4620inline
4622 bm::word_t left,
4623 bm::word_t right) BMNOEXCEPT
4624{
4625 BM_ASSERT(left <= right);
4626 BM_ASSERT(right <= bm::gap_max_bits-1);
4627
4628 unsigned nword, nbit, bitcount, temp;
4629 nbit = left & bm::set_word_mask;
4630 const bm::word_t* word =
4631 block + (nword = unsigned(left >> bm::set_word_shift));
4632 if (left == right) // special case (only 1 bit to check)
4633 return (*word >> nbit) & 1u;
4634
4635 if (nbit) // starting position is not aligned
4636 {
4637 unsigned right_margin = nbit + right - left;
4638 if (right_margin < 32)
4639 {
4640 unsigned mask =
4642 block_set_table<true>::_left[right_margin];
4643 return mask == (*word & mask);
4644 }
4645 temp = *word & block_set_table<true>::_right[nbit];
4646 if (temp != block_set_table<true>::_right[nbit])
4647 return false;
4648 bitcount = (right - left + 1u) - (32 - nbit);
4649 ++word;
4650 }
4651 else
4652 {
4653 bitcount = right - left + 1u;
4654 }
4655
4656 // now when we are word aligned, we can scan the bit-stream
4657 const bm::id64_t maskFF64 = ~0ull;
4658 const bm::word_t maskFF = ~0u;
4659 // loop unrolled to evaluate 4 words at a time
4660 // SIMD showed no advantage, unless evaluate sub-wave intervals
4661 //
4662 for ( ;bitcount >= 128; bitcount-=128, word+=4)
4663 {
4664 bm::id64_t w64_0 = bm::id64_t(word[0]) + (bm::id64_t(word[1]) << 32);
4665 bm::id64_t w64_1 = bm::id64_t(word[2]) + (bm::id64_t(word[3]) << 32);
4666 if ((w64_0 ^ maskFF64) | (w64_1 ^ maskFF64))
4667 return false;
4668 } // for
4669
4670 for ( ;bitcount >= 32; bitcount-=32, ++word)
4671 {
4672 if (*word != maskFF)
4673 return false;
4674 } // for
4675 BM_ASSERT(bitcount < 32);
4676
4677 if (bitcount) // we have a tail to count
4678 {
4679 temp = *word & block_set_table<true>::_left[bitcount-1];
4680 if (temp != block_set_table<true>::_left[bitcount-1])
4681 return false;
4682 }
4683
4684 return true;
4685}
4686
4687
4688
4689
4690/*!
4691 Function calculates number of 1 bits in the given array of words in
4692 the range between left anf right bits (borders included)
4693 Make sure the addr is aligned.
4694
4695 @ingroup bitfunc
4696*/
4697inline
4699 bm::word_t left,
4700 bm::word_t right) BMNOEXCEPT
4701{
4702 BM_ASSERT(left <= right);
4703 BM_ASSERT(right <= bm::gap_max_bits-1);
4704
4705 unsigned nword, nbit, bitcount, count;
4706 nbit = left & bm::set_word_mask;
4707 const bm::word_t* word =
4708 block + (nword = unsigned(left >> bm::set_word_shift));
4709 if (left == right) // special case (only 1 bit to check)
4710 {
4711 return (*word >> nbit) & 1u;
4712 }
4713
4714 count = 0;
4715 bitcount = right - left + 1u;
4716 if (nbit) // starting position is not aligned
4717 {
4718 unsigned right_margin = nbit + right - left;
4719 if (right_margin < 32)
4720 {
4721 unsigned mask =
4723 block_set_table<true>::_left[right_margin];
4724 return bm::word_bitcount(*word & mask);
4725 }
4726 count = bm::word_bitcount(*word & block_set_table<true>::_right[nbit]);
4727 bitcount -= 32 - nbit;
4728 ++word;
4729 }
4730
4731 // now when we are word aligned, we can count bits the usual way
4732 //
4733 #if defined(BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
4734 for ( ;bitcount >= 64; bitcount-=64)
4735 {
4736 bm::id64_t w64 = *((bm::id64_t*)word); // x86 unaligned(!) read
4737 count += unsigned(_mm_popcnt_u64(w64));
4738 word += 2;
4739 }
4740 if (bitcount >= 32)
4741 {
4742 count += bm::word_bitcount(*word++);
4743 bitcount-=32;
4744 }
4745 #else
4746 for ( ;bitcount >= 32; bitcount-=32, ++word)
4747 count += bm::word_bitcount(*word);
4748 #endif
4749 BM_ASSERT(bitcount < 32);
4750
4751 if (bitcount) // we have a tail to count
4752 {
4753 count += bm::word_bitcount(
4754 *word & block_set_table<true>::_left[bitcount-1]);
4755 }
4756 return count;
4757}
4758
4759/*!
4760 Function calculates number of 1 bits in the given array of words in
4761 the range between 0 anf right bits (borders included)
4762 Make sure the addr is aligned.
4763
4764 @ingroup bitfunc
4765*/
4766inline
4768 bm::word_t right) BMNOEXCEPT
4769{
4770 BM_ASSERT(block);
4771 if (!right) // special case, first bit check
4772 return *block & 1u;
4773 bm::id_t count = 0;
4774
4775 unsigned bitcount = right + 1;
4776
4777 // AVX2 or 64-bit loop unroll
4778 #if defined(BMAVX2OPT) || defined(BMAVX512OPT)
4779 BM_AVX2_POPCNT_PROLOG
4780
4781 __m256i cnt = _mm256_setzero_si256();
4782 bm::id64_t* cnt64;
4783
4784 for ( ;bitcount >= 256; bitcount -= 256)
4785 {
4786 const __m256i* src = (__m256i*)block;
4787 __m256i xmm256 = _mm256_load_si256(src);
4788 BM_AVX2_BIT_COUNT(bc, xmm256);
4789 cnt = _mm256_add_epi64(cnt, bc);
4790
4791 block += 8;
4792 }
4793 cnt64 = (bm::id64_t*)&cnt;
4794 count += (unsigned)(cnt64[0] + cnt64[1] + cnt64[2] + cnt64[3]);
4795 #endif
4796
4797 for ( ;bitcount >= 64; bitcount -= 64)
4798 {
4799 bm::id64_t* p = (bm::id64_t*)block;
4800 count += bm::word_bitcount64(*p);
4801 block += 2;
4802 }
4803 if (bitcount >= 32)
4804 {
4805 count += bm::word_bitcount(*block++);
4806 bitcount-=32;
4807 }
4808
4809 if (bitcount) // we have a tail to count
4810 {
4811 count +=
4812 bm::word_bitcount(*block & block_set_table<true>::_left[bitcount-1]);
4813 }
4814 return count;
4815}
4816
4817
4818
4819/*!
4820 Cyclic rotation of bit-block left by 1 bit
4821 @ingroup bitfunc
4822*/
4823inline
4825{
4826 bm::word_t co_flag = (block[0] >> 31) & 1; // carry over bit
4827 for (unsigned i = 0; i < bm::set_block_size-1; ++i)
4828 {
4829 block[i] = (block[i] << 1) | (block[i + 1] >> 31);
4830 }
4831 block[set_block_size - 1] = (block[set_block_size - 1] << 1) | co_flag;
4832}
4833
4834/*!
4835 @brief Unrolled cyclic rotation of bit-block left by 1 bit
4836 @param block - bit-block pointer
4837 @ingroup bitfunc
4838*/
4839inline
4841{
4842 bm::word_t co_flag = (block[0] >> 31) & 1; // carry over bit
4843 const unsigned unroll_factor = 4;
4844 bm::word_t w0, w1, w2, w3;
4845
4846 unsigned i;
4847 for (i = 0; i < bm::set_block_size - unroll_factor; i += unroll_factor)
4848 {
4849 w0 = block[i + 1] >> 31;
4850 w1 = block[i + 2] >> 31;
4851 w2 = block[i + 3] >> 31;
4852 w3 = block[i + 4] >> 31;
4853
4854 block[0 + i] = (block[0 + i] << 1) | w0;
4855 block[1 + i] = (block[1 + i] << 1) | w1;
4856 block[2 + i] = (block[2 + i] << 1) | w2;
4857 block[3 + i] = (block[3 + i] << 1) | w3;
4858 }
4859 block[i] = (block[i] << 1) | (block[i + 1] >> 31);
4860 block[i + 1] = (block[i + 1] << 1) | (block[i + 2] >> 31);
4861 block[i + 2] = (block[i + 2] << 1) | (block[i + 3] >> 31);
4862 block[set_block_size - 1] = (block[set_block_size - 1] << 1) | co_flag;
4863}
4864
4865/*!
4866 @brief insert bit into position and shift the rest right with carryover
4867
4868 @param block - bit-block pointer
4869 @param bitpos - bit position to insert
4870 @param value - bit value (0|1) to insert
4871
4872 @return carry over value
4873 @ingroup bitfunc
4874*/
4875inline
4877 unsigned bitpos, bool value) BMNOEXCEPT
4878{
4879 BM_ASSERT(block);
4880 BM_ASSERT(bitpos < 65536);
4881
4882 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
4883 unsigned nword = unsigned(nbit >> bm::set_word_shift);
4884 nbit &= bm::set_word_mask;
4885
4886 bm::word_t co_flag = 0;
4887 if (nbit)
4888 {
4889 unsigned r_mask = bm::block_set_table<true>::_right[nbit];
4890 bm::word_t w = block[nword] & r_mask;
4891 bm::word_t wl = block[nword] & ~r_mask;
4892 co_flag = w >> 31;
4893 w <<= 1u;
4894 block[nword] = w | (unsigned(value) << nbit) | wl;
4895 ++nword;
4896 }
4897 else
4898 {
4899 co_flag = value;
4900 }
4901
4902 for (unsigned i = nword; i < bm::set_block_size; ++i)
4903 {
4904 bm::word_t w = block[i];
4905 bm::word_t co_flag1 = w >> 31;
4906 w = (w << 1u) | co_flag;
4907 block[i] = w;
4908 co_flag = co_flag1;
4909 } // for i
4910 return co_flag;
4911}
4912
4913
4914
4915/*!
4916 @brief Right bit-shift bitblock by 1 bit (reference)
4917 @param block - bit-block pointer
4918 @param empty_acc - [out] contains 0 if block is empty
4919 @param co_flag - carry over from the previous block
4920
4921 @return carry over bit (1 or 0)
4922 @ingroup bitfunc
4923*/
4924inline
4926 bm::word_t* BMRESTRICT empty_acc,
4927 bm::word_t co_flag) BMNOEXCEPT
4928{
4929 BM_ASSERT(block);
4930 BM_ASSERT(empty_acc);
4931
4932 bm::word_t acc = 0;
4933 for (unsigned i = 0; i < bm::set_block_size; ++i)
4934 {
4935 bm::word_t w = block[i];
4936 bm::word_t co_flag1 = w >> 31;
4937 acc |= w = (w << 1u) | co_flag;
4938 block[i] = w;
4939 co_flag = co_flag1;
4940 }
4941 *empty_acc = acc;
4942 return co_flag;
4943}
4944
4945/*!
4946 @brief Right bit-shift of bit-block by 1 bit (loop unrolled)
4947 @param block - bit-block pointer
4948 @param empty_acc - [out] contains 0 if block is empty
4949 @param co_flag - carry over from the previous block
4950
4951 @return carry over bit (1 or 0)
4952 @ingroup bitfunc
4953*/
4954inline
4956 bm::word_t* BMRESTRICT empty_acc,
4957 bm::word_t co_flag) BMNOEXCEPT
4958{
4959 BM_ASSERT(block);
4960 BM_ASSERT(empty_acc);
4961 #if defined(VECT_SHIFT_R1)
4962 return VECT_SHIFT_R1(block, empty_acc, co_flag);
4963 #else
4964 return bm::bit_block_shift_r1(block, empty_acc, co_flag);
4965 #endif
4966}
4967
4968
4969/*!
4970 @brief Left bit-shift bitblock by 1 bit (reference)
4971 @param block - bit-block pointer
4972 @param empty_acc - [out] contains 0 if block is empty
4973 @param co_flag - carry over from the prev/next block
4974
4975 @return carry over bit (1 or 0)
4976
4977 @ingroup bitfunc
4978*/
4979inline
4981 bm::word_t* empty_acc, bm::word_t co_flag) BMNOEXCEPT
4982{
4983 BM_ASSERT(block);
4984 BM_ASSERT(empty_acc);
4985
4986 bm::word_t acc = 0;
4987 for (int i = bm::set_block_size-1; i >= 0; --i)
4988 {
4989 bm::word_t w = block[i];
4990 bm::word_t co_flag1 = w & 1u;
4991 acc |= w = (w >> 1u) | (co_flag << 31u);
4992 block[i] = w;
4993 co_flag = co_flag1;
4994 }
4995
4996 *empty_acc = acc;
4997 return co_flag;
4998}
4999
5000/*!
5001 @brief Left bit-shift of bit-block by 1 bit (loop unrolled)
5002 @param block - bit-block pointer
5003 @param empty_acc - [out] contains 0 if block is empty
5004 @param co_flag - carry over from the prev/next block
5005
5006 @return carry over bit (1 or 0)
5007 @ingroup bitfunc
5008*/
5009inline
5011 bm::word_t* empty_acc,
5012 bm::word_t co_flag) BMNOEXCEPT
5013{
5014 BM_ASSERT(block);
5015 BM_ASSERT(empty_acc);
5016 #if defined(VECT_SHIFT_L1)
5017 return VECT_SHIFT_L1(block, empty_acc, co_flag);
5018 #else
5019 return bm::bit_block_shift_l1(block, empty_acc, co_flag);
5020 #endif
5021}
5022
5023/*!
5024 @brief erase bit from position and shift the rest right with carryover
5025
5026 @param block - bit-block pointer
5027 @param bitpos - bit position to insert
5028 @param carry_over - bit value to add to the end (0|1)
5029
5030 @ingroup bitfunc
5031*/
5032inline
5034 unsigned bitpos,
5035 bool carry_over) BMNOEXCEPT
5036{
5037 BM_ASSERT(block);
5038 BM_ASSERT(bitpos < 65536);
5039
5040 if (!bitpos)
5041 {
5042 bm::word_t acc;
5043 bm::bit_block_shift_l1_unr(block, &acc, carry_over);
5044 return;
5045 }
5046
5047 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
5048 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5049 nbit &= bm::set_word_mask;
5050
5051 bm::word_t co_flag = carry_over;
5052 for (unsigned i = bm::set_block_size-1; i > nword; --i)
5053 {
5054 bm::word_t w = block[i];
5055 bm::word_t co_flag1 = w & 1u;
5056 w = (w >> 1u) | (co_flag << 31u);
5057 block[i] = w;
5058 co_flag = co_flag1;
5059 }
5060
5061 if (nbit)
5062 {
5063 unsigned r_mask = bm::block_set_table<true>::_right[nbit];
5064 bm::word_t w = block[nword] & r_mask;
5065 bm::word_t wl = block[nword] & ~r_mask;
5066 w &= ~(1 << nbit); // clear the removed bit
5067 w >>= 1u;
5068 w |= wl | (co_flag << 31u);
5069 block[nword] = w;
5070 }
5071 else
5072 {
5073 block[nword] = (block[nword] >> 1u) | (co_flag << 31u);
5074 }
5075}
5076
5077
5078/*!
5079 @brief Right bit-shift of bit-block by 1 bit (reference) + AND
5080 @param block - bit-block pointer
5081 @param co_flag - carry over from the previous block
5082 @param mask_block - mask bit-block pointer
5083 @param digest - block digest
5084
5085 @return carry over bit (1 or 0)
5086 @ingroup bitfunc
5087*/
5088inline
5090 bm::word_t co_flag,
5091 const bm::word_t* BMRESTRICT mask_block,
5093{
5094 BM_ASSERT(block);
5095 BM_ASSERT(mask_block);
5096 BM_ASSERT(digest && *digest);
5097
5098
5099 bm::id64_t d = *digest;
5100
5101 unsigned di = 0;
5102 if (!co_flag)
5103 {
5104 bm::id64_t t = d & -d;
5105 di = bm::word_bitcount64(t - 1); // find start bit-index
5106 }
5107
5108 for (; di < 64; ++di)
5109 {
5110 const unsigned d_base = di * bm::set_block_digest_wave_size;
5111 bm::id64_t dmask = (1ull << di);
5112 if (d & dmask) // digest stride not empty
5113 {
5114 bm::word_t acc = 0;
5115 for (unsigned i = d_base; i < d_base + bm::set_block_digest_wave_size; ++i)
5116 {
5118
5119 bm::word_t w = block[i];
5120 bm::word_t co_flag1 = w >> 31;
5121 w = (w << 1u) | co_flag;
5122 acc |= block[i] = w & mask_block[i];
5123 co_flag = co_flag1;
5124 }
5125 if (!acc)
5126 d &= ~dmask; // update digest: clear stride bit
5127 }
5128 else // stride is empty
5129 {
5130 BM_ASSERT(block[d_base + bm::set_block_digest_wave_size -1]==0);
5131 BM_ASSERT(block[d_base]==0);
5132
5133 if (co_flag) // there is carry-over
5134 {
5135 BM_ASSERT(co_flag == 1);
5136 BM_ASSERT(block[d_base] == 0);
5137
5138 block[d_base] = co_flag & mask_block[d_base];
5139 if (block[d_base])
5140 d |= dmask; // update digest
5141 co_flag = 0;
5142 }
5143 }
5144 } // for di
5145
5146 *digest = d;
5147 return co_flag;
5148}
5149
5150/*!
5151 @brief Right bit-shift bitblock by 1 bit (reference) + AND
5152 @param block - bit-block pointer
5153 @param co_flag - carry over from the previous block
5154 @param mask_block - mask bit-block pointer
5155 @param digest - block digest
5156
5157 @return carry over bit (1 or 0)
5158 @ingroup bitfunc
5159*/
5160inline
5162 bm::word_t co_flag,
5163 const bm::word_t* BMRESTRICT mask_block,
5165{
5166 BM_ASSERT(block);
5167 BM_ASSERT(mask_block);
5168 BM_ASSERT(digest);
5169
5170 #if defined(VECT_SHIFT_R1_AND)
5171 return VECT_SHIFT_R1_AND(block, co_flag, mask_block, digest);
5172 #else
5173 return bm::bit_block_shift_r1_and(block, co_flag, mask_block, digest);
5174 #endif
5175}
5176
5177
5178/*!
5179 Function calculates if there is any number of 1 bits
5180 in the given array of words in the range between left anf right bits
5181 (borders included). Make sure the addresses are aligned.
5182
5183 @ingroup bitfunc
5184*/
5185inline
5187 bm::word_t left,
5188 bm::word_t right) BMNOEXCEPT
5189{
5190 BM_ASSERT(left <= right);
5191
5192 unsigned nbit = left; // unsigned(left & bm::set_block_mask);
5193 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5194 nbit &= bm::set_word_mask;
5195
5196 const bm::word_t* word = block + nword;
5197
5198 if (left == right) // special case (only 1 bit to check)
5199 {
5200 return (*word >> nbit) & 1;
5201 }
5202 unsigned acc;
5203 unsigned bitcount = right - left + 1;
5204
5205 if (nbit) // starting position is not aligned
5206 {
5207 unsigned right_margin = nbit + (right - left);
5208 if (right_margin < 32)
5209 {
5210 unsigned mask =
5212 block_set_table<true>::_left[right_margin];
5213 return *word & mask;
5214 }
5215 else
5216 {
5217 acc = *word & block_set_table<true>::_right[nbit];
5218 if (acc)
5219 return acc;
5220 bitcount -= 32 - nbit;
5221 }
5222 ++word;
5223 }
5224
5225 // loop unrolled to evaluate 4 words at a time
5226 // SIMD showed no advantage, unless evaluate sub-wave intervals
5227 //
5228 for ( ;bitcount >= 128; bitcount-=128, word+=4)
5229 {
5230 acc = word[0] | word[1] | word[2] | word[3];
5231 if (acc)
5232 return acc;
5233 } // for
5234
5235 acc = 0;
5236 for ( ;bitcount >= 32; bitcount -= 32)
5237 {
5238 acc |= *word++;
5239 } // for
5240
5241 if (bitcount) // we have a tail to count
5242 acc |= (*word) & block_set_table<true>::_left[bitcount-1];
5243
5244 return acc;
5245}
5246
5247// ----------------------------------------------------------------------
5248
5249/*! Function inverts block of bits
5250 @ingroup bitfunc
5251*/
5252template<typename T>
5254{
5256#ifdef BMVECTOPT
5257 VECT_INVERT_BLOCK(start);
5258#else
5259 T* end = (T*)((unsigned*)(start) + bm::set_block_size);
5260 do
5261 {
5262 start[0] = ~start[0];
5263 start[1] = ~start[1];
5264 start[2] = ~start[2];
5265 start[3] = ~start[3];
5266 start+=4;
5267 } while (start < end);
5268#endif
5269}
5270
5271// ----------------------------------------------------------------------
5272
5273/*! @brief Returns "true" if all bits in the block are 1
5274 @ingroup bitfunc
5275*/
5276inline
5278{
5279#if defined(BMSSE42OPT) || defined(BMAVX2OPT)
5280 return VECT_IS_ONE_BLOCK(start);
5281#else
5282 const bm::word_t* BMRESTRICT src_end = (bm::word_t*)start + bm::set_block_size;
5283 const bm::wordop_t* end = (const bm::wordop_t*)src_end;
5284 do
5285 {
5286 bm::wordop_t tmp =
5287 start[0] & start[1] & start[2] & start[3];
5288 if (tmp != bm::all_bits_mask)
5289 return false;
5290 start += 4;
5291 } while (start < end);
5292 return true;
5293#endif
5294}
5295
5296// ----------------------------------------------------------------------
5297
5298/*! @brief Returns "true" if all bits are 1 in the block [left, right]
5299 Function check for block varieties
5300 @internal
5301*/
5302inline
5304 unsigned left, unsigned right) BMNOEXCEPT
5305{
5306 BM_ASSERT(left <= right);
5307 BM_ASSERT(right < bm::gap_max_bits);
5308 if (block)
5309 {
5310 if (BM_IS_GAP(block))
5311 return bm::gap_is_all_one_range(BMGAP_PTR(block), left, right);
5312 if (block == FULL_BLOCK_FAKE_ADDR)
5313 return true;
5314 return bm::bit_block_is_all_one_range(block, left, right);
5315 }
5316 return false;
5317}
5318
5319/*! @brief Returns "true" if all bits are 1 in the block [left, right]
5320 and border bits are 0
5321 @internal
5322*/
5323inline
5325 unsigned left, unsigned right) BMNOEXCEPT
5326{
5327 BM_ASSERT(left <= right);
5328 BM_ASSERT(right < bm::gap_max_bits-1);
5329
5330 if (block)
5331 {
5332 bool is_left, is_right, all_one;
5333 if (BM_IS_GAP(block))
5334 {
5335 const bm::gap_word_t* gap = BMGAP_PTR(block);
5336 all_one = bm::gap_is_interval(gap, left, right);
5337 return all_one;
5338 }
5339 else // bit-block
5340 {
5341 if (block == FULL_BLOCK_FAKE_ADDR)
5342 return false;
5343 unsigned nword = ((left-1) >> bm::set_word_shift);
5344 is_left = block[nword] & (1u << ((left-1) & bm::set_word_mask));
5345 if (is_left == false)
5346 {
5347 nword = ((right + 1) >> bm::set_word_shift);
5348 is_right = block[nword] & (1u << ((right + 1) & bm::set_word_mask));
5349 if (is_right == false)
5350 {
5351 all_one = bm::bit_block_is_all_one_range(block, left, right);
5352 return all_one;
5353 }
5354 }
5355 }
5356 }
5357
5358 return false;
5359}
5360
5361// ----------------------------------------------------------------------
5362
5363/**
5364 \brief Searches for the last 1 bit in the 111 interval of a BIT block
5365 \param block - BIT buffer
5366 \param nbit - bit index to start checking from
5367 \param pos - [out] found value
5368
5369 \return false if not found
5370 @ingroup bitfunc
5371*/
5372inline
5374 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
5375{
5376 BM_ASSERT(block);
5377 BM_ASSERT(pos);
5378
5379 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5380 unsigned bit_pos = (nbit & bm::set_word_mask);
5381 bm::word_t w = block[nword];
5382 w &= (1u << bit_pos);
5383 if (!w)
5384 return false;
5385
5386 if (nbit == bm::gap_max_bits-1)
5387 {
5388 *pos = bm::gap_max_bits-1;
5389 return true;
5390 }
5391 *pos = nbit;
5392
5393 ++nbit;
5394 nword = unsigned(nbit >> bm::set_word_shift);
5395 bit_pos = (nbit & bm::set_word_mask);
5396
5397 w = (~block[nword]) >> bit_pos;
5398 w <<= bit_pos; // clear the trailing bits
5399 if (w)
5400 {
5401 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
5402 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))-1);
5403 return true;
5404 }
5405
5406 for (++nword; nword < bm::set_block_size; ++nword)
5407 {
5408 w = ~block[nword];
5409 if (w)
5410 {
5411 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
5412 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))-1);
5413 return true;
5414 }
5415 } // for nword
5416
5417 // 0 not found, all block is 1s...
5418 *pos = bm::gap_max_bits-1;
5419 return true;
5420}
5421
5422
5423/*! @brief Find end of the current 111 interval
5424 @return search result code 0 - not found, 1 found, 2 - found at the end
5425 @internal
5426*/
5427inline
5429 unsigned nbit_from,
5430 unsigned* BMRESTRICT found_nbit) BMNOEXCEPT
5431{
5432 BM_ASSERT(block && found_nbit);
5433 BM_ASSERT(nbit_from < bm::gap_max_bits);
5434
5435 bool b;
5436 if (BM_IS_GAP(block))
5437 {
5438 const bm::gap_word_t* gap = BMGAP_PTR(block);
5439 b = bm::gap_find_interval_end(gap, nbit_from, found_nbit);
5440 if (b && *found_nbit == bm::gap_max_bits-1)
5441 return 2; // end of block, keep searching
5442 }
5443 else // bit-block
5444 {
5445 if (IS_FULL_BLOCK(block))
5446 {
5447 *found_nbit = bm::gap_max_bits-1;
5448 return 2;
5449 }
5450 b = bm::bit_block_find_interval_end(block, nbit_from, found_nbit);
5451 if (b && *found_nbit == bm::gap_max_bits-1)
5452 return 2; // end of block, keep searching
5453 }
5454 return b;
5455}
5456
5457// ----------------------------------------------------------------------
5458
5459/**
5460 \brief Searches for the first 1 bit in the 111 interval of a BIT block
5461 \param block - BIT buffer
5462 \param nbit - bit index to start checking from
5463 \param pos - [out] found value
5464
5465 \return false if not found
5466 @ingroup bitfunc
5467*/
5468inline
5470 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
5471{
5472 BM_ASSERT(block);
5473 BM_ASSERT(pos);
5474
5475 unsigned nword = unsigned(nbit >> bm::set_word_shift);
5476 unsigned bit_pos = (nbit & bm::set_word_mask);
5477 bm::word_t w = block[nword];
5478 w &= (1u << bit_pos);
5479 if (!w)
5480 return false;
5481
5482 if (nbit == 0)
5483 {
5484 *pos = 0;
5485 return true;
5486 }
5487 *pos = nbit;
5488
5489 --nbit;
5490 nword = unsigned(nbit >> bm::set_word_shift);
5491 bit_pos = (nbit & bm::set_word_mask);
5492
5493 w = (~block[nword]) & block_set_table<true>::_left[bit_pos];
5494 if (w)
5495 {
5496 bit_pos = bm::bit_scan_reverse32(w);
5497 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))+1);
5498 return true;
5499 }
5500
5501 if (nword)
5502 {
5503 for (--nword; true; --nword)
5504 {
5505 w = ~block[nword];
5506 if (w)
5507 {
5508 bit_pos = bm::bit_scan_reverse32(w); // trailing zeros
5509 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))+1);
5510 return true;
5511 }
5512 if (!nword)
5513 break;
5514 } // for nword
5515 }
5516
5517 // 0 not found, all block is 1s...
5518 *pos = 0;
5519 return true;
5520}
5521
5522
5523/*! @brief Find start of the current 111 interval
5524 @return search result code 0 - not found, 1 found, 2 - found at the start
5525 @internal
5526*/
5527inline
5529 unsigned nbit_from,
5530 unsigned* BMRESTRICT found_nbit) BMNOEXCEPT
5531{
5532 BM_ASSERT(block && found_nbit);
5533 BM_ASSERT(nbit_from < bm::gap_max_bits);
5534 bool b;
5535 if (BM_IS_GAP(block))
5536 {
5537 const bm::gap_word_t* gap = BMGAP_PTR(block);
5538 b = bm::gap_find_interval_start(gap, nbit_from, found_nbit);
5539 if (b && *found_nbit == 0)
5540 return 2; // start of block, keep searching
5541 }
5542 else // bit-block
5543 {
5544 if (IS_FULL_BLOCK(block))
5545 {
5546 *found_nbit = 0;
5547 return 2;
5548 }
5549 b = bm::bit_block_find_interval_start(block, nbit_from, found_nbit);
5550 if (b && *found_nbit == 0)
5551 return 2; // start of block, keep searching
5552 }
5553 return b;
5554}
5555
5556// ----------------------------------------------------------------------
5557
5558/*! @brief Returns "true" if one bit is set in the block [left, right]
5559 Function check for block varieties
5560 @internal
5561*/
5562inline
5563bool block_any_range(const bm::word_t* const BMRESTRICT block,
5564 unsigned left, unsigned right) BMNOEXCEPT
5565{
5566 BM_ASSERT(left <= right);
5567 BM_ASSERT(right < bm::gap_max_bits);
5568 if (!block)
5569 return false;
5570 if (BM_IS_GAP(block))
5571 return bm::gap_any_range(BMGAP_PTR(block), left, right);
5572 if (IS_FULL_BLOCK(block))
5573 return true;
5574 return bm::bit_block_any_range(block, left, right);
5575}
5576
5577// ----------------------------------------------------------------------
5578
5579/*! @brief Returns "true" if one bit is set in the block
5580 Function check for block varieties
5581 @internal
5582*/
5583inline
5585{
5586 if (!block)
5587 return false;
5588 if (IS_FULL_BLOCK(block))
5589 return true;
5590 bool all_zero = (BM_IS_GAP(block)) ?
5592 : bm::bit_is_all_zero(block);
5593 return !all_zero;
5594}
5595
5596
5597
5598// ----------------------------------------------------------------------
5599
5600// GAP blocks manipulation functions:
5601
5602/*! \brief GAP and functor */
5603BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5604{
5605 return v1 & v2;
5606}
5607
5608
5609/*! \brief GAP xor functor */
5610BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5611{
5612 return v1 ^ v2;
5613}
5614
5615
5616/*! \brief GAP or functor */
5617BMFORCEINLINE unsigned or_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5618{
5619 return v1 | v2;
5620}
5621
5622/*! \brief GAP or functor */
5623BMFORCEINLINE unsigned sub_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5624{
5625 return v1 & ~v2;
5626}
5627
5628
5629/*!
5630 \brief GAP AND operation.
5631
5632 Function performs AND logical operation on gap vectors.
5633 If possible function put the result into vect1 and returns this
5634 pointer. Otherwise result is put into tmp_buf, which should be
5635 twice of the vector size.
5636
5637 \param vect1 - operand 1
5638 \param vect2 - operand 2
5639 \param tmp_buf - pointer on temporary buffer
5640 \param dsize - out size of the destination
5641 \return Result pointer (tmp_buf OR vect1)
5642
5643 @ingroup gapfunc
5644*/
5647 const gap_word_t* BMRESTRICT vect2,
5648 gap_word_t* BMRESTRICT tmp_buf,
5649 unsigned& dsize) BMNOEXCEPT
5650{
5651 bm::gap_buff_op(tmp_buf, vect1, 0, vect2, 0, bm::and_op, dsize);
5652 return tmp_buf;
5653}
5654
5655/*!
5656 \brief GAP AND operation test.
5657
5658 Function performs AND logical operation on gap vectors.
5659 If possible function put the result into vect1 and returns this
5660 pointer. Otherwise result is put into tmp_buf, which should be
5661 twice of the vector size.
5662
5663 \param vect1 - operand 1
5664 \param vect2 - operand 2
5665 \return non zero value if operation returns any 1 bit
5666
5667 @ingroup gapfunc
5668*/
5671 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5672{
5673 return gap_buff_any_op(vect1, 0, vect2, 0, and_op);
5674}
5675
5676
5677/*!
5678 \brief GAP bitcount AND operation test.
5679
5680 \param vect1 - operand 1
5681 \param vect2 - operand 2
5682 \return bitcount of vect1 AND vect2
5683
5684 @ingroup gapfunc
5685*/
5686inline
5688 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5689{
5690 return gap_buff_count_op(vect1, vect2, and_op);
5691}
5692
5693
5694
5695/*!
5696 \brief GAP XOR operation.
5697
5698 Function performs XOR logical operation on gap vectors.
5699 If possible function put the result into vect1 and returns this
5700 pointer. Otherwise result is put into tmp_buf, which should be
5701 twice of the vector size.
5702
5703 \param vect1 - operand 1
5704 \param vect2 - operand 2
5705 \param tmp_buf - pointer on temporary buffer
5706 \param dsize - out destination size
5707 \return Result pointer (tmp_buf)
5708
5709 @ingroup gapfunc
5710*/
5711inline
5713 const gap_word_t* BMRESTRICT vect2,
5714 gap_word_t* BMRESTRICT tmp_buf,
5715 unsigned& dsize) BMNOEXCEPT
5716{
5717 gap_buff_op(tmp_buf, vect1, 0, vect2, 0, bm::xor_op, dsize);
5718 return tmp_buf;
5719}
5720
5721/*! Light weight gap_operation_xor for len prediction
5722 @ingroup gapfunc
5723*/
5724inline
5726 const gap_word_t* BMRESTRICT vect2,
5727 unsigned& dsize,
5728 unsigned limit) BMNOEXCEPT
5729{
5730 return gap_buff_dry_op(vect1, vect2, bm::xor_op, dsize, limit);
5731}
5732
5733
5734/*!
5735 \brief GAP XOR operation test.
5736
5737 Function performs AND logical operation on gap vectors.
5738 If possible function put the result into vect1 and returns this
5739 pointer. Otherwise result is put into tmp_buf, which should be
5740 twice of the vector size.
5741
5742 \param vect1 - operand 1
5743 \param vect2 - operand 2
5744 \return non zero value if operation returns any 1 bit
5745
5746 @ingroup gapfunc
5747*/
5750 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5751{
5752 return gap_buff_any_op(vect1, 0, vect2, 0, bm::xor_op);
5753}
5754
5755/*!
5756 \brief GAP bitcount XOR operation test.
5757
5758 \param vect1 - operand 1
5759 \param vect2 - operand 2
5760 \return bitcount of vect1 XOR vect2
5761
5762 @ingroup gapfunc
5763*/
5766 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5767{
5768 return gap_buff_count_op(vect1, vect2, bm::xor_op);
5769}
5770
5771
5772/*!
5773 \brief GAP OR operation.
5774
5775 Function performs OR logical oparation on gap vectors.
5776 If possible function put the result into vect1 and returns this
5777 pointer. Otherwise result is put into tmp_buf, which should be
5778 twice of the vector size.
5779
5780 \param vect1 - operand 1
5781 \param vect2 - operand 2
5782 \param tmp_buf - pointer on temporary buffer
5783 \param dsize - out destination size
5784
5785 \return Result pointer (tmp_buf)
5786
5787 @ingroup gapfunc
5788*/
5789inline
5791 const gap_word_t* BMRESTRICT vect2,
5792 gap_word_t* BMRESTRICT tmp_buf,
5793 unsigned& dsize) BMNOEXCEPT
5794{
5795 gap_buff_op(tmp_buf, vect1, 1, vect2, 1, bm::and_op, dsize);
5796 gap_invert(tmp_buf);
5797 return tmp_buf;
5798}
5799
5800/*!
5801 \brief GAP bitcount OR operation test.
5802
5803 \param vect1 - operand 1
5804 \param vect2 - operand 2
5805 \return bitcount of vect1 OR vect2
5806
5807 @ingroup gapfunc
5808*/
5810unsigned gap_count_or(const gap_word_t* BMRESTRICT vect1,
5811 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5812{
5813 return gap_buff_count_op(vect1, vect2, bm::or_op);
5814}
5815
5816
5817
5818/*!
5819 \brief GAP SUB (AND NOT) operation.
5820
5821 Function performs SUB logical operation on gap vectors.
5822 If possible function put the result into vect1 and returns this
5823 pointer. Otherwise result is put into tmp_buf, which should be
5824 twice of the vector size.
5825
5826 \param vect1 - operand 1
5827 \param vect2 - operand 2
5828 \param tmp_buf - pointer on temporary buffer
5829 \param dsize - out destination size
5830
5831 \return Result pointer (tmp_buf)
5832
5833 @ingroup gapfunc
5834*/
5836 const gap_word_t* BMRESTRICT vect2,
5837 gap_word_t* BMRESTRICT tmp_buf,
5838 unsigned& dsize) BMNOEXCEPT
5839{
5840 gap_buff_op(tmp_buf, vect1, 0, vect2, 1, and_op, dsize);
5841 return tmp_buf;
5842}
5843
5844
5845/*!
5846 \brief GAP SUB operation test.
5847
5848 Function performs AND logical operation on gap vectors.
5849 If possible function put the result into vect1 and returns this
5850 pointer. Otherwise result is put into tmp_buf, which should be
5851 twice of the vector size.
5852
5853 \param vect1 - operand 1
5854 \param vect2 - operand 2
5855 \return non zero value if operation returns any 1 bit
5856
5857 @ingroup gapfunc
5858*/
5861 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5862{
5863 return gap_buff_any_op(vect1, 0, vect2, 1, bm::and_op);
5864}
5865
5866
5867/*!
5868\brief GAP bitcount SUB (AND NOT) operation test.
5869
5870\param vect1 - operand 1
5871\param vect2 - operand 2
5872\return bitcount of vect1 SUB (AND NOT) vect2
5873
5874@ingroup gapfunc
5875*/
5878 const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5879{
5880 return gap_buff_count_op(vect1, vect2, bm::sub_op);
5881}
5882
5883
5884// ----------------------------------------------------------------------
5885
5886// BIT blocks manipulation functions:
5887
5888
5889/*!
5890 \brief Bitblock copy operation.
5891
5892 \param dst - destination block.
5893 \param src - source block.
5894
5895 @ingroup bitfunc
5896*/
5897inline
5899 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5900{
5901#ifdef BMVECTOPT
5902 VECT_COPY_BLOCK(dst, src);
5903#else
5904 ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
5905#endif
5906}
5907
5908/*!
5909 \brief Bitblock copy/stream operation.
5910
5911 \param dst - destination block.
5912 \param src - source block.
5913
5914 @ingroup bitfunc
5915*/
5916inline
5918 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5919{
5920#ifdef VECT_STREAM_BLOCK
5921 VECT_STREAM_BLOCK(dst, src);
5922#else
5923 ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
5924#endif
5925}
5926
5927
5928/*!
5929 \brief Plain bitblock AND operation.
5930 Function does not analyse availability of source and destination blocks.
5931
5932 \param dst - destination block.
5933 \param src - source block.
5934
5935 \return 0 if AND operation did not produce anything (no 1s in the output)
5936
5937 @ingroup bitfunc
5938*/
5939inline
5941 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5942{
5943 BM_ASSERT(dst);
5944 BM_ASSERT(src);
5945 BM_ASSERT(dst != src);
5946
5947#ifdef BMVECTOPT
5948 bm::id64_t acc = VECT_AND_BLOCK(dst, src);
5949#else
5950 unsigned arr_sz = bm::set_block_size / 2;
5951
5954
5955 bm::id64_t acc = 0;
5956 for (unsigned i = 0; i < arr_sz; i+=4)
5957 {
5958 acc |= dst_u->w64[i] &= src_u->w64[i];
5959 acc |= dst_u->w64[i+1] &= src_u->w64[i+1];
5960 acc |= dst_u->w64[i+2] &= src_u->w64[i+2];
5961 acc |= dst_u->w64[i+3] &= src_u->w64[i+3];
5962 }
5963#endif
5964 return acc;
5965}
5966
5967/*!
5968 \brief digest based bit-block AND
5969
5970 \param dst - destination block.
5971 \param src - source block.
5972 \param digest - known digest of dst block
5973
5974 \return new digest
5975
5976 @ingroup bitfunc
5977*/
5978inline
5980 const bm::word_t* BMRESTRICT src,
5981 bm::id64_t digest) BMNOEXCEPT
5982{
5983 BM_ASSERT(dst);
5984 BM_ASSERT(src);
5985 BM_ASSERT(dst != src);
5986
5987 const bm::id64_t mask(1ull);
5988 bm::id64_t d = digest;
5989 while (d)
5990 {
5991 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
5992
5993 unsigned wave = bm::word_bitcount64(t - 1);
5994 unsigned off = wave * bm::set_block_digest_wave_size;
5995
5996 #if defined(VECT_AND_DIGEST)
5997 bool all_zero = VECT_AND_DIGEST(&dst[off], &src[off]);
5998 if (all_zero)
5999 digest &= ~(mask << wave);
6000 #else
6001 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)(&src[off]);
6003
6004 bm::id64_t acc = 0;
6005 unsigned j = 0;
6006 do
6007 {
6008 acc |= dst_u->w64[j+0] &= src_u->w64[j+0];
6009 acc |= dst_u->w64[j+1] &= src_u->w64[j+1];
6010 acc |= dst_u->w64[j+2] &= src_u->w64[j+2];
6011 acc |= dst_u->w64[j+3] &= src_u->w64[j+3];
6012 j+=4;
6013 } while (j < bm::set_block_digest_wave_size/2);
6014
6015 if (!acc) // all zero
6016 digest &= ~(mask << wave);
6017 #endif
6018
6019 d = bm::bmi_bslr_u64(d); // d &= d - 1;
6020 } // while
6021
6022 return digest;
6023}
6024
6025
6026/*!
6027 \brief digest based bit-block AND 5-way
6028
6029 \return new digest
6030
6031 @ingroup bitfunc
6032*/
6033inline
6035 const bm::word_t* BMRESTRICT src0,
6036 const bm::word_t* BMRESTRICT src1,
6037 const bm::word_t* BMRESTRICT src2,
6038 const bm::word_t* BMRESTRICT src3,
6039 bm::id64_t digest) BMNOEXCEPT
6040{
6041 BM_ASSERT(dst);
6042 BM_ASSERT(src0 && src1 && src2 && src3);
6043
6044 const bm::id64_t mask(1ull);
6045 bm::id64_t d = digest;
6046 while (d)
6047 {
6048 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
6049
6050 unsigned wave = bm::word_bitcount64(t - 1);
6051 unsigned off = wave * bm::set_block_digest_wave_size;
6052
6053#if defined(VECT_AND_DIGEST_5WAY)
6054 bool all_zero = VECT_AND_DIGEST_5WAY(&dst[off], &src0[off], &src1[off], &src2[off], &src3[off]);
6055 if (all_zero)
6056 digest &= ~(mask << wave);
6057#else
6058 const bm::bit_block_t::bunion_t* BMRESTRICT src_u0 = (const bm::bit_block_t::bunion_t*)(&src0[off]);
6059 const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 = (const bm::bit_block_t::bunion_t*)(&src1[off]);
6060 const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 = (const bm::bit_block_t::bunion_t*)(&src2[off]);
6061 const bm::bit_block_t::bunion_t* BMRESTRICT src_u3 = (const bm::bit_block_t::bunion_t*)(&src3[off]);
6063
6064 bm::id64_t acc = 0;
6065 unsigned j = 0;
6066 do
6067 {
6068 acc |= dst_u->w64[j + 0] &= src_u0->w64[j + 0] & src_u1->w64[j + 0] & src_u2->w64[j + 0] & src_u3->w64[j + 0];
6069 acc |= dst_u->w64[j + 1] &= src_u0->w64[j + 1] & src_u1->w64[j + 1] & src_u2->w64[j + 1] & src_u3->w64[j + 1];
6070 acc |= dst_u->w64[j + 2] &= src_u0->w64[j + 2] & src_u1->w64[j + 2] & src_u2->w64[j + 2] & src_u3->w64[j + 2];
6071 acc |= dst_u->w64[j + 3] &= src_u0->w64[j + 3] & src_u1->w64[j + 3] & src_u2->w64[j + 3] & src_u3->w64[j + 3];
6072 j += 4;
6073 } while (j < bm::set_block_digest_wave_size / 2);
6074
6075 if (!acc) // all zero
6076 digest &= ~(mask << wave);
6077#endif
6078
6079 d = bm::bmi_bslr_u64(d); // d &= d - 1;
6080 } // while
6081
6082 return digest;
6083}
6084
6085
6086/*!
6087 \brief digest based bit-block AND
6088
6089 dst = src1 AND src2
6090
6091 \param dst - destination block.
6092 \param src1 - source block.
6093 \param src2 - source block.
6094 \param digest - known initial digest
6095
6096 \return new digest
6097
6098 @ingroup bitfunc
6099*/
6100inline
6102 const bm::word_t* BMRESTRICT src1,
6103 const bm::word_t* BMRESTRICT src2,
6104 bm::id64_t digest) BMNOEXCEPT
6105{
6106 BM_ASSERT(dst);
6107 BM_ASSERT(src1 && src2);
6108 BM_ASSERT(dst != src1 && dst != src2);
6109
6110 const bm::id64_t mask(1ull);
6111 bm::id64_t d = digest;
6112 while (d)
6113 {
6114 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
6115
6116 unsigned wave = bm::word_bitcount64(t - 1);
6117 unsigned off = wave * bm::set_block_digest_wave_size;
6118
6119 #if defined(VECT_AND_DIGEST_2WAY)
6120 bool all_zero = VECT_AND_DIGEST_2WAY(&dst[off], &src1[off], &src2[off]);
6121 if (all_zero)
6122 digest &= ~(mask << wave);
6123 #else
6125 (const bm::bit_block_t::bunion_t*)(&src1[off]);
6127 (const bm::bit_block_t::bunion_t*)(&src2[off]);
6129 (bm::bit_block_t::bunion_t*)(&dst[off]);
6130
6131 bm::id64_t acc = 0;
6132 unsigned j = 0;
6133 do
6134 {
6135 acc |= dst_u->w64[j+0] = src_u1->w64[j+0] & src_u2->w64[j+0];
6136 acc |= dst_u->w64[j+1] = src_u1->w64[j+1] & src_u2->w64[j+1];
6137 acc |= dst_u->w64[j+2] = src_u1->w64[j+2] & src_u2->w64[j+2];
6138 acc |= dst_u->w64[j+3] = src_u1->w64[j+3] & src_u2->w64[j+3];
6139 j+=4;
6140 } while (j < bm::set_block_digest_wave_size/2);
6141
6142 if (!acc) // all zero
6143 digest &= ~(mask << wave);
6144 #endif
6145
6146 d = bm::bmi_bslr_u64(d); // d &= d - 1;
6147 } // while
6148
6149 return digest;
6150}
6151
6152
6153
6154/*!
6155 \brief Function ANDs two bitblocks and computes the bitcount.
6156 Function does not analyse availability of source blocks.
6157
6158 \param src1 - first bit block
6159 \param src2 - second bit block
6160
6161 @ingroup bitfunc
6162*/
6163inline
6165 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6166{
6167 unsigned count;
6168 const bm::word_t* src1_end = src1 + bm::set_block_size;
6169#ifdef BMVECTOPT
6170 count = VECT_BITCOUNT_AND(src1, src1_end, src2);
6171#else
6172 count = 0;
6173# ifdef BM64OPT
6174 const bm::id64_t* b1 = (bm::id64_t*) src1;
6175 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6176 const bm::id64_t* b2 = (bm::id64_t*) src2;
6177 do
6178 {
6179 count += bitcount64_4way(b1[0] & b2[0],
6180 b1[1] & b2[1],
6181 b1[2] & b2[2],
6182 b1[3] & b2[3]);
6183 b1 += 4;
6184 b2 += 4;
6185 } while (b1 < b1_end);
6186# else
6187 do
6188 {
6189 BM_INCWORD_BITCOUNT(count, src1[0] & src2[0]);
6190 BM_INCWORD_BITCOUNT(count, src1[1] & src2[1]);
6191 BM_INCWORD_BITCOUNT(count, src1[2] & src2[2]);
6192 BM_INCWORD_BITCOUNT(count, src1[3] & src2[3]);
6193
6194 src1+=4;
6195 src2+=4;
6196 } while (src1 < src1_end);
6197# endif
6198#endif
6199 return count;
6200}
6201
6202
6203/*!
6204 \brief Function ANDs two bitblocks and tests for any bit.
6205 Function does not analyse availability of source blocks.
6206
6207 \param src1 - first bit block
6208 \param src2 - second bit block
6209
6210 @ingroup bitfunc
6211*/
6212inline
6213unsigned bit_block_and_any(const bm::word_t* src1,
6214 const bm::word_t* src2) BMNOEXCEPT
6215{
6216 unsigned count = 0;
6217 const bm::word_t* src1_end = src1 + bm::set_block_size;
6218 do
6219 {
6220 count = (src1[0] & src2[0]) |
6221 (src1[1] & src2[1]) |
6222 (src1[2] & src2[2]) |
6223 (src1[3] & src2[3]);
6224
6225 src1+=4; src2+=4;
6226 } while ((src1 < src1_end) && !count);
6227 return count;
6228}
6229
6230
6231
6232
6233/*!
6234 \brief Function XORs two bitblocks and computes the bitcount.
6235 Function does not analyse availability of source blocks.
6236
6237 \param src1 - first bit block
6238 \param src2 - second bit block
6239
6240 @ingroup bitfunc
6241*/
6242inline
6244 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6245{
6246 unsigned count;
6247 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6248#ifdef BMVECTOPT
6249 count = VECT_BITCOUNT_XOR(src1, src1_end, src2);
6250#else
6251 count = 0;
6252# ifdef BM64OPT
6253 const bm::id64_t* b1 = (bm::id64_t*) src1;
6254 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6255 const bm::id64_t* b2 = (bm::id64_t*) src2;
6256 do
6257 {
6258 count += bitcount64_4way(b1[0] ^ b2[0],
6259 b1[1] ^ b2[1],
6260 b1[2] ^ b2[2],
6261 b1[3] ^ b2[3]);
6262 b1 += 4;
6263 b2 += 4;
6264 } while (b1 < b1_end);
6265# else
6266 do
6267 {
6268 BM_INCWORD_BITCOUNT(count, src1[0] ^ src2[0]);
6269 BM_INCWORD_BITCOUNT(count, src1[1] ^ src2[1]);
6270 BM_INCWORD_BITCOUNT(count, src1[2] ^ src2[2]);
6271 BM_INCWORD_BITCOUNT(count, src1[3] ^ src2[3]);
6272
6273 src1+=4;
6274 src2+=4;
6275 } while (src1 < src1_end);
6276# endif
6277#endif
6278 return count;
6279}
6280
6281
6282/*!
6283 \brief Function XORs two bitblocks and and tests for any bit.
6284 Function does not analyse availability of source blocks.
6285
6286 \param src1 - first bit block.
6287 \param src2 - second bit block.
6288
6289 @ingroup bitfunc
6290*/
6291inline
6293 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6294{
6295 unsigned count = 0;
6296 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6297 do
6298 {
6299 count = (src1[0] ^ src2[0]) |
6300 (src1[1] ^ src2[1]) |
6301 (src1[2] ^ src2[2]) |
6302 (src1[3] ^ src2[3]);
6303
6304 src1+=4; src2+=4;
6305 } while (!count && (src1 < src1_end));
6306 return count;
6307}
6308
6309/*!
6310 \brief Function SUBs two bitblocks and computes the bitcount.
6311 Function does not analyse availability of source blocks.
6312
6313 \param src1 - first bit block.
6314 \param src2 - second bit block.
6315
6316 @ingroup bitfunc
6317*/
6318inline
6320 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6321{
6322 unsigned count;
6323 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6324#ifdef BMVECTOPT
6325 count = VECT_BITCOUNT_SUB(src1, src1_end, src2);
6326#else
6327 count = 0;
6328# ifdef BM64OPT
6329 const bm::id64_t* b1 = (bm::id64_t*) src1;
6330 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6331 const bm::id64_t* b2 = (bm::id64_t*) src2;
6332 do
6333 {
6334 count += bitcount64_4way(b1[0] & ~b2[0],
6335 b1[1] & ~b2[1],
6336 b1[2] & ~b2[2],
6337 b1[3] & ~b2[3]);
6338 b1 += 4;
6339 b2 += 4;
6340 } while (b1 < b1_end);
6341# else
6342 do
6343 {
6344 BM_INCWORD_BITCOUNT(count, src1[0] & ~src2[0]);
6345 BM_INCWORD_BITCOUNT(count, src1[1] & ~src2[1]);
6346 BM_INCWORD_BITCOUNT(count, src1[2] & ~src2[2]);
6347 BM_INCWORD_BITCOUNT(count, src1[3] & ~src2[3]);
6348
6349 src1+=4;
6350 src2+=4;
6351 } while (src1 < src1_end);
6352# endif
6353#endif
6354 return count;
6355}
6356
6357/*!
6358 \brief Function SUBs two bitblocks and and tests for any bit.
6359 Function does not analyse availability of source blocks.
6360
6361 \param src1 - first bit block.
6362 \param src2 - second bit block.
6363
6364 @ingroup bitfunc
6365*/
6366inline
6368 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6369{
6370 unsigned count = 0;
6371 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6372
6373 do
6374 {
6375 count = (src1[0] & ~src2[0]) |
6376 (src1[1] & ~src2[1]) |
6377 (src1[2] & ~src2[2]) |
6378 (src1[3] & ~src2[3]);
6379
6380 src1+=4; src2+=4;
6381 } while ((src1 < src1_end) && (count == 0));
6382 return count;
6383}
6384
6385
6386/*!
6387 \brief Function ORs two bitblocks and computes the bitcount.
6388 Function does not analyse availability of source blocks.
6389
6390 \param src1 - first bit block
6391 \param src2 - second bit block.
6392
6393 @ingroup bitfunc
6394*/
6395inline
6396unsigned bit_block_or_count(const bm::word_t* src1,
6397 const bm::word_t* src2) BMNOEXCEPT
6398{
6399 unsigned count;
6400 const bm::word_t* src1_end = src1 + bm::set_block_size;
6401#ifdef BMVECTOPT
6402 count = VECT_BITCOUNT_OR(src1, src1_end, src2);
6403#else
6404 count = 0;
6405# ifdef BM64OPT
6406 const bm::id64_t* b1 = (bm::id64_t*) src1;
6407 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6408 const bm::id64_t* b2 = (bm::id64_t*) src2;
6409 do
6410 {
6411 count += bitcount64_4way(b1[0] | b2[0],
6412 b1[1] | b2[1],
6413 b1[2] | b2[2],
6414 b1[3] | b2[3]);
6415 b1 += 4;
6416 b2 += 4;
6417 } while (b1 < b1_end);
6418# else
6419 do
6420 {
6421 BM_INCWORD_BITCOUNT(count, src1[0] | src2[0]);
6422 BM_INCWORD_BITCOUNT(count, src1[1] | src2[1]);
6423 BM_INCWORD_BITCOUNT(count, src1[2] | src2[2]);
6424 BM_INCWORD_BITCOUNT(count, src1[3] | src2[3]);
6425
6426 src1+=4;
6427 src2+=4;
6428 } while (src1 < src1_end);
6429# endif
6430#endif
6431 return count;
6432}
6433
6434/*!
6435 \brief Function ORs two bitblocks and and tests for any bit.
6436 Function does not analyse availability of source blocks.
6437
6438 \param src1 - first bit block.
6439 \param src2 - second bit block.
6440
6441 @ingroup bitfunc
6442*/
6443inline
6445 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6446{
6447 unsigned count = 0;
6448 const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6449 do
6450 {
6451 count = (src1[0] | src2[0]) |
6452 (src1[1] | src2[1]) |
6453 (src1[2] | src2[2]) |
6454 (src1[3] | src2[3]);
6455
6456 src1+=4; src2+=4;
6457 } while (!count && (src1 < src1_end));
6458 return count;
6459}
6460
6461
6462
6463
6464/*!
6465 \brief bitblock AND operation.
6466
6467 \param dst - destination block.
6468 \param src - source block.
6469
6470 \returns pointer on destination block.
6471 If returned value equal to src means that block mutation requested.
6472 NULL is valid return value.
6473
6474 @ingroup bitfunc
6475*/
6477 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6478{
6479 BM_ASSERT(dst || src);
6480
6481 bm::word_t* ret = dst;
6482
6483 if (IS_VALID_ADDR(dst)) // The destination block already exists
6484 {
6485 if (!IS_VALID_ADDR(src))
6486 {
6487 if (IS_EMPTY_BLOCK(src))
6488 {
6489 //If the source block is zero
6490 //just clean the destination block
6491 return 0;
6492 }
6493 }
6494 else
6495 {
6496 // Regular operation AND on the whole block.
6497 //
6498 auto any = bm::bit_block_and(dst, src);
6499 if (!any)
6500 ret = 0;
6501 }
6502 }
6503 else // The destination block does not exist yet
6504 {
6505 if(!IS_VALID_ADDR(src))
6506 {
6507 if(IS_EMPTY_BLOCK(src))
6508 {
6509 // The source block is empty.
6510 // One argument empty - all result is empty.
6511 return 0;
6512 }
6513 // Here we have nothing to do.
6514 // Src block is all ON, dst block remains as it is
6515 }
6516 else // destination block does not exists, src - valid block
6517 {
6518 if (IS_FULL_BLOCK(dst))
6519 return const_cast<bm::word_t*>(src);
6520 // Nothng to do.
6521 // Dst block is all ZERO no combination required.
6522 }
6523 }
6524
6525 return ret;
6526}
6527
6528
6529/*!
6530 \brief Performs bitblock AND operation and calculates bitcount of the result.
6531
6532 \param src1 - first bit block.
6533 \param src2 - second bit block.
6534
6535 \returns bitcount value
6536
6537 @ingroup bitfunc
6538*/
6539inline
6541 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6542{
6543 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
6544 return 0;
6545 if (src1 == FULL_BLOCK_FAKE_ADDR)
6546 src1 = FULL_BLOCK_REAL_ADDR;
6547 if (src2 == FULL_BLOCK_FAKE_ADDR)
6548 src2 = FULL_BLOCK_REAL_ADDR;
6549
6550 return bit_block_and_count(src1, src2);
6551}
6552
6553/*!
6554 \brief Performs bitblock AND operation test.
6555
6556 \param src1 - first bit block.
6557 \param src2 - second bit block.
6558
6559 \returns non zero if there is any value
6560
6561 @ingroup bitfunc
6562*/
6563inline
6565 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6566{
6567 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
6568 return 0;
6569 if (src1 == FULL_BLOCK_FAKE_ADDR)
6570 src1 = FULL_BLOCK_REAL_ADDR;
6571 if (src2 == FULL_BLOCK_FAKE_ADDR)
6572 src2 = FULL_BLOCK_REAL_ADDR;
6573 return bit_block_and_any(src1, src2);
6574}
6575
6576
6577
6578/*!
6579 \brief Performs bitblock SUB operation and calculates bitcount of the result.
6580
6581 \param src1 - first bit block.
6582 \param src2 - second bit block
6583
6584 \returns bitcount value
6585
6586 @ingroup bitfunc
6587*/
6588inline
6590 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6591{
6592 if (src1 == src2)
6593 return 0;
6594 if (IS_EMPTY_BLOCK(src1))
6595 return 0;
6596 if (IS_EMPTY_BLOCK(src2)) // nothing to diff
6597 {
6598 if (IS_FULL_BLOCK(src1))
6599 return bm::gap_max_bits;
6600 return bm::bit_block_count(src1);
6601 }
6602 if (IS_FULL_BLOCK(src2))
6603 return 0;
6604
6605 if (src1 == FULL_BLOCK_FAKE_ADDR)
6606 src1 = FULL_BLOCK_REAL_ADDR;
6607 if (src2 == FULL_BLOCK_FAKE_ADDR)
6608 src2 = FULL_BLOCK_REAL_ADDR;
6609
6610 return bm::bit_block_sub_count(src1, src2);
6611}
6612
6613
6614/*!
6615 \brief Performs inverted bitblock SUB operation and calculates
6616 bitcount of the result.
6617
6618 \param src1 - first bit block.
6619 \param src2 - second bit block
6620
6621 \returns bitcount value
6622
6623 @ingroup bitfunc
6624*/
6625inline
6627 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6628{
6629 return bit_operation_sub_count(src2, src1);
6630}
6631
6632
6633/*!
6634 \brief Performs bitblock test of SUB operation.
6635
6636 \param src1 - first bit block.
6637 \param src2 - second bit block
6638
6639 \returns non zero value if there are any bits
6640
6641 @ingroup bitfunc
6642*/
6643inline
6645 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6646{
6647 if (IS_EMPTY_BLOCK(src1))
6648 return 0;
6649 if (src1 == src2)
6650 return 0;
6651
6652 if (IS_EMPTY_BLOCK(src2)) // nothing to diff
6653 return !bit_is_all_zero(src1);
6654
6655 if (IS_FULL_BLOCK(src2))
6656 return 0;
6657
6658 if (src1 == FULL_BLOCK_FAKE_ADDR)
6659 src1 = FULL_BLOCK_REAL_ADDR;
6660 if (src2 == FULL_BLOCK_FAKE_ADDR)
6661 src2 = FULL_BLOCK_REAL_ADDR;
6662
6663 return bm::bit_block_sub_any(src1, src2);
6664}
6665
6666
6667
6668/*!
6669 \brief Performs bitblock OR operation and calculates bitcount of the result.
6670
6671 \param src1 - first bit block.
6672 \param src2 - second bit block.
6673
6674 \returns bitcount value
6675
6676 @ingroup bitfunc
6677*/
6678inline
6680 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6681{
6682 if (IS_FULL_BLOCK(src1) || IS_FULL_BLOCK(src2))
6683 return bm::gap_max_bits;
6684
6685 if (IS_EMPTY_BLOCK(src1))
6686 {
6687 if (!IS_EMPTY_BLOCK(src2))
6688 return bm::bit_block_count(src2);
6689 else
6690 return 0; // both blocks are empty
6691 }
6692 else
6693 {
6694 if (IS_EMPTY_BLOCK(src2))
6695 return bm::bit_block_count(src1);
6696 }
6697 if (src1 == FULL_BLOCK_FAKE_ADDR)
6698 src1 = FULL_BLOCK_REAL_ADDR;
6699 if (src2 == FULL_BLOCK_FAKE_ADDR)
6700 src2 = FULL_BLOCK_REAL_ADDR;
6701
6702 return bm::bit_block_or_count(src1, src2);
6703}
6704
6705/*!
6706 \brief Performs bitblock OR operation test.
6707
6708 \param src1 - first bit block.
6709 \param src2 - second bit block.
6710
6711 \returns non zero value if there are any bits
6712
6713 @ingroup bitfunc
6714*/
6715inline
6717 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6718{
6719 if (IS_EMPTY_BLOCK(src1))
6720 {
6721 if (!IS_EMPTY_BLOCK(src2))
6722 return !bit_is_all_zero(src2);
6723 else
6724 return 0; // both blocks are empty
6725 }
6726 else
6727 {
6728 if (IS_EMPTY_BLOCK(src2))
6729 return !bit_is_all_zero(src1);
6730 }
6731 if (src1 == FULL_BLOCK_FAKE_ADDR)
6732 src1 = FULL_BLOCK_REAL_ADDR;
6733 if (src2 == FULL_BLOCK_FAKE_ADDR)
6734 src2 = FULL_BLOCK_REAL_ADDR;
6735
6736 return bit_block_or_any(src1, src2);
6737}
6738
6739/*!
6740 \brief Plain bitblock OR operation.
6741 Function does not analyse availability of source and destination blocks.
6742
6743 \param dst - destination block.
6744 \param src - source block.
6745
6746 @ingroup bitfunc
6747*/
6748inline
6750 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6751{
6752#ifdef BMVECTOPT
6753 return VECT_OR_BLOCK(dst, src);
6754#else
6755 const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*)src;
6756 const bm::wordop_t* BMRESTRICT wrd_end = (wordop_t*)(src + bm::set_block_size);
6757 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6758
6759 bm::wordop_t acc = 0;
6760 const bm::wordop_t not_acc = acc = ~acc;
6761
6762 do
6763 {
6764 acc &= (dst_ptr[0] |= wrd_ptr[0]);
6765 acc &= (dst_ptr[1] |= wrd_ptr[1]);
6766 acc &= (dst_ptr[2] |= wrd_ptr[2]);
6767 acc &= (dst_ptr[3] |= wrd_ptr[3]);
6768
6769 dst_ptr+=4;wrd_ptr+=4;
6770
6771 } while (wrd_ptr < wrd_end);
6772 return acc == not_acc;
6773#endif
6774}
6775
6776/*!
6777 \brief 2 way (target := source1 | source2) bitblock OR operation.
6778 \param dst - dest block [out]
6779 \param src1 - source 1
6780 \param src2 - source 2
6781
6782 @return 1 if produced block of ALL ones
6783 @ingroup bitfunc
6784*/
6785inline
6787 const bm::word_t* BMRESTRICT src1,
6788 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6789{
6790#ifdef BMVECTOPT
6791 return VECT_OR_BLOCK_2WAY(dst, src1, src2);
6792#else
6793 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6794 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6795 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6796 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6797
6798 bm::wordop_t acc = 0;
6799 const bm::wordop_t not_acc = acc = ~acc;
6800 do
6801 {
6802 acc &= (dst_ptr[0] = wrd_ptr1[0] | wrd_ptr2[0]);
6803 acc &= (dst_ptr[1] = wrd_ptr1[1] | wrd_ptr2[1]);
6804 acc &= (dst_ptr[2] = wrd_ptr1[2] | wrd_ptr2[2]);
6805 acc &= (dst_ptr[3] = wrd_ptr1[3] | wrd_ptr2[3]);
6806
6807 dst_ptr+=4; wrd_ptr1+=4; wrd_ptr2+=4;
6808
6809 } while (wrd_ptr1 < wrd_end1);
6810 return acc == not_acc;
6811#endif
6812}
6813
6814
6815/*!
6816 \brief 2 way (target := source1 ^ source2) bitblock XOR operation.
6817 \param dst - dest block [out]
6818 \param src1 - source 1
6819 \param src2 - source 2
6820
6821 @return OR accumulator
6822 @ingroup bitfunc
6823*/
6824inline
6826 const bm::word_t* BMRESTRICT src1,
6827 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6828{
6829#ifdef BMVECTOPT
6830 return VECT_XOR_BLOCK_2WAY(dst, src1, src2);
6831#else
6832 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6833 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6834 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6835 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6836
6837 bm::wordop_t acc = 0;
6838 do
6839 {
6840 acc |= (dst_ptr[0] = wrd_ptr1[0] ^ wrd_ptr2[0]);
6841 acc |= (dst_ptr[1] = wrd_ptr1[1] ^ wrd_ptr2[1]);
6842 acc |= (dst_ptr[2] = wrd_ptr1[2] ^ wrd_ptr2[2]);
6843 acc |= (dst_ptr[3] = wrd_ptr1[3] ^ wrd_ptr2[3]);
6844
6845 dst_ptr+=4; wrd_ptr1+=4; wrd_ptr2+=4;
6846
6847 } while (wrd_ptr1 < wrd_end1);
6848 return acc;
6849#endif
6850}
6851
6852
6853/*!
6854 \brief 3 way (target | source1 | source2) bitblock OR operation.
6855 Function does not analyse availability of source and destination blocks.
6856
6857 \param dst - sst-dest block [in,out]
6858 \param src1 - source 1
6859 \param src2 - source 2
6860
6861 @return 1 if produced block of ALL ones
6862
6863 @ingroup bitfunc
6864*/
6865inline
6867 const bm::word_t* BMRESTRICT src1,
6868 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6869{
6870#ifdef BMVECTOPT
6871 return VECT_OR_BLOCK_3WAY(dst, src1, src2);
6872#else
6873 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6874 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6875 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6876 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6877
6878 bm::wordop_t acc = 0;
6879 const bm::wordop_t not_acc = acc = ~acc;
6880 do
6881 {
6882 acc &= (dst_ptr[0] |= wrd_ptr1[0] | wrd_ptr2[0]);
6883 acc &= (dst_ptr[1] |= wrd_ptr1[1] | wrd_ptr2[1]);
6884 acc &= (dst_ptr[2] |= wrd_ptr1[2] | wrd_ptr2[2]);
6885 acc &= (dst_ptr[3] |= wrd_ptr1[3] | wrd_ptr2[3]);
6886
6887 dst_ptr+=4; wrd_ptr1+=4;wrd_ptr2+=4;
6888
6889 } while (wrd_ptr1 < wrd_end1);
6890 return acc == not_acc;
6891#endif
6892}
6893
6894
6895/*!
6896 \brief 5 way (target, source1, source2) bitblock OR operation.
6897 Function does not analyse availability of source and destination blocks.
6898
6899 \param dst - destination block.
6900 \param src1 - source1, etc
6901 \param src2 - source1, etc
6902 \param src3 - source1, etc
6903 \param src4 - source1, etc
6904
6905 @return 1 if produced block of ALL ones
6906
6907 @ingroup bitfunc
6908*/
6909inline
6911 const bm::word_t* BMRESTRICT src1,
6912 const bm::word_t* BMRESTRICT src2,
6913 const bm::word_t* BMRESTRICT src3,
6914 const bm::word_t* BMRESTRICT src4) BMNOEXCEPT
6915{
6916#ifdef BMVECTOPT
6917 return VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4);
6918#else
6919 const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6920 const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6921 const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6922 const bm::wordop_t* BMRESTRICT wrd_ptr3 = (wordop_t*)src3;
6923 const bm::wordop_t* BMRESTRICT wrd_ptr4 = (wordop_t*)src4;
6924 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6925
6926 bm::wordop_t acc = 0;
6927 const bm::wordop_t not_acc = acc = ~acc;
6928 do
6929 {
6930 acc &= (dst_ptr[0] |= wrd_ptr1[0] | wrd_ptr2[0] | wrd_ptr3[0] | wrd_ptr4[0]);
6931 acc &= (dst_ptr[1] |= wrd_ptr1[1] | wrd_ptr2[1] | wrd_ptr3[1] | wrd_ptr4[1]);
6932 acc &= (dst_ptr[2] |= wrd_ptr1[2] | wrd_ptr2[2] | wrd_ptr3[2] | wrd_ptr4[2]);
6933 acc &= (dst_ptr[3] |= wrd_ptr1[3] | wrd_ptr2[3] | wrd_ptr3[3] | wrd_ptr4[3]);
6934
6935 dst_ptr+=4;
6936 wrd_ptr1+=4;wrd_ptr2+=4;wrd_ptr3+=4;wrd_ptr4+=4;
6937
6938 } while (wrd_ptr1 < wrd_end1);
6939 return acc == not_acc;
6940#endif
6941}
6942
6943
6944
6945
6946/*!
6947 \brief Block OR operation. Makes analysis if block is 0 or FULL.
6948
6949 \param dst - destination block.
6950 \param src - source block.
6951
6952 \returns pointer on destination block.
6953 If returned value equal to src means that block mutation requested.
6954 NULL is valid return value.
6955
6956 @ingroup bitfunc
6957*/
6958inline
6960 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6961{
6962 BM_ASSERT(dst || src);
6963
6964 bm::word_t* ret = dst;
6965
6966 if (IS_VALID_ADDR(dst)) // The destination block already exists
6967 {
6968 if (!IS_VALID_ADDR(src))
6969 {
6970 if (IS_FULL_BLOCK(src))
6971 {
6972 // if the source block is all set
6973 // just set the destination block
6974 ::memset(dst, 0xFF, bm::set_block_size * sizeof(bm::word_t));
6975 }
6976 }
6977 else
6978 {
6979 // Regular operation OR on the whole block
6980 bm::bit_block_or(dst, src);
6981 }
6982 }
6983 else // The destination block does not exist yet
6984 {
6985 if (!IS_VALID_ADDR(src))
6986 {
6987 if (IS_FULL_BLOCK(src))
6988 {
6989 // The source block is all set, because dst does not exist
6990 // we can simply replace it.
6991 return const_cast<bm::word_t*>(FULL_BLOCK_FAKE_ADDR);
6992 }
6993 }
6994 else
6995 {
6996 if (dst == 0)
6997 {
6998 // The only case when we have to allocate the new block:
6999 // Src is all zero and Dst does not exist
7000 return const_cast<bm::word_t*>(src);
7001 }
7002 }
7003 }
7004 return ret;
7005}
7006
7007/*!
7008 \brief Plain bitblock SUB (AND NOT) operation.
7009 Function does not analyse availability of source and destination blocks.
7010
7011 \param dst - destination block.
7012 \param src - source block.
7013
7014 \return 0 if SUB operation did not produce anything (no 1s in the output)
7015
7016 @ingroup bitfunc
7017*/
7018inline
7020 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7021{
7022#ifdef BMVECTOPT
7023 bm::id64_t acc = VECT_SUB_BLOCK(dst, src);
7024 return acc;
7025#else
7026 unsigned arr_sz = bm::set_block_size / 2;
7027
7030
7031 bm::id64_t acc = 0;
7032 for (unsigned i = 0; i < arr_sz; i+=4)
7033 {
7034 acc |= dst_u->w64[i] &= ~src_u->w64[i];
7035 acc |= dst_u->w64[i+1] &= ~src_u->w64[i+1];
7036 acc |= dst_u->w64[i+2] &= ~src_u->w64[i+2];
7037 acc |= dst_u->w64[i+3] &= ~src_u->w64[i+3];
7038 }
7039 return acc;
7040#endif
7041}
7042
7043/*!
7044 \brief digest based bitblock SUB (AND NOT) operation
7045
7046 \param dst - destination block.
7047 \param src - source block.
7048 \param digest - known digest of dst block
7049
7050 \return new digest
7051
7052 @ingroup bitfunc
7053*/
7054inline
7056 const bm::word_t* BMRESTRICT src,
7057 bm::id64_t digest) BMNOEXCEPT
7058{
7059 BM_ASSERT(dst);
7060 BM_ASSERT(src);
7061 BM_ASSERT(dst != src);
7062
7063 const bm::id64_t mask(1ull);
7064
7065 bm::id64_t d = digest;
7066 while (d)
7067 {
7068 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
7069
7070 unsigned wave = bm::word_bitcount64(t - 1);
7071 unsigned off = wave * bm::set_block_digest_wave_size;
7072
7073 #if defined(VECT_SUB_DIGEST)
7074 bool all_zero = VECT_SUB_DIGEST(&dst[off], &src[off]);
7075 if (all_zero)
7076 digest &= ~(mask << wave);
7077 #else
7078 const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)(&src[off]);
7080
7081 bm::id64_t acc = 0;
7082 unsigned j = 0;
7083 do
7084 {
7085 acc |= dst_u->w64[j+0] &= ~src_u->w64[j+0];
7086 acc |= dst_u->w64[j+1] &= ~src_u->w64[j+1];
7087 acc |= dst_u->w64[j+2] &= ~src_u->w64[j+2];
7088 acc |= dst_u->w64[j+3] &= ~src_u->w64[j+3];
7089 j+=4;
7090 } while (j < bm::set_block_digest_wave_size/2);
7091
7092 if (!acc) // all zero
7093 digest &= ~(mask << wave);
7094 #endif
7095
7096 d = bm::bmi_bslr_u64(d); // d &= d - 1;
7097 } // while
7098
7099 return digest;
7100}
7101
7102/*!
7103 \brief digest based bitblock SUB (AND NOT) operation (3 operand)
7104
7105 \param dst - destination block.
7106 \param src1 - source block 1
7107 \param src2 - source block 2
7108 \param digest - known digest of dst block
7109
7110 \return new digest
7111
7112 @ingroup bitfunc
7113*/
7114inline
7116 const bm::word_t* BMRESTRICT src1,
7117 const bm::word_t* BMRESTRICT src2,
7118 bm::id64_t digest) BMNOEXCEPT
7119{
7120 BM_ASSERT(dst);
7121 BM_ASSERT(src1 && src2);
7122 BM_ASSERT(dst != src1 && dst != src2);
7123
7124 const bm::id64_t mask(1ull);
7125
7126 bm::id64_t d = digest;
7127 while (d)
7128 {
7129 bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
7130
7131 unsigned wave = bm::word_bitcount64(t - 1);
7132 unsigned off = wave * bm::set_block_digest_wave_size;
7133
7134 #if defined(VECT_SUB_DIGEST_2WAY)
7135 bool all_zero = VECT_SUB_DIGEST_2WAY(&dst[off], &src1[off], &src2[off]);
7136 if (all_zero)
7137 digest &= ~(mask << wave);
7138 #else
7140 (const bm::bit_block_t::bunion_t*)(&src1[off]);
7142 (const bm::bit_block_t::bunion_t*)(&src2[off]);
7144 (bm::bit_block_t::bunion_t*)(&dst[off]);
7145
7146 bm::id64_t acc = 0;
7147 unsigned j = 0;
7148 do
7149 {
7150 acc |= dst_u->w64[j+0] = src_u1->w64[j+0] & ~src_u2->w64[j+0];
7151 acc |= dst_u->w64[j+1] = src_u1->w64[j+1] & ~src_u2->w64[j+1];
7152 acc |= dst_u->w64[j+2] = src_u1->w64[j+2] & ~src_u2->w64[j+2];
7153 acc |= dst_u->w64[j+3] = src_u1->w64[j+3] & ~src_u2->w64[j+3];
7154 j+=4;
7155 } while (j < bm::set_block_digest_wave_size/2);
7156
7157 if (!acc) // all zero
7158 digest &= ~(mask << wave);
7159 #endif
7160
7161 d = bm::bmi_bslr_u64(d); // d &= d - 1;
7162 } // while
7163
7164 return digest;
7165}
7166
7167
7168
7169/*!
7170 \brief bitblock SUB operation.
7171
7172 \param dst - destination block.
7173 \param src - source block.
7174
7175 \returns pointer on destination block.
7176 If returned value equal to src means that block mutation requested.
7177 NULL is valid return value.
7178
7179 @ingroup bitfunc
7180*/
7181inline
7183 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7184{
7185 BM_ASSERT(dst || src);
7186
7187 bm::word_t* ret = dst;
7188 if (IS_VALID_ADDR(dst)) // The destination block already exists
7189 {
7190 if (!IS_VALID_ADDR(src))
7191 {
7192 if (IS_FULL_BLOCK(src))
7193 {
7194 // If the source block is all set
7195 // just clean the destination block
7196 return 0;
7197 }
7198 }
7199 else
7200 {
7201 auto any = bm::bit_block_sub(dst, src);
7202 if (!any)
7203 ret = 0;
7204 }
7205 }
7206 else // The destination block does not exist yet
7207 {
7208 if (!IS_VALID_ADDR(src))
7209 {
7210 if (IS_FULL_BLOCK(src))
7211 {
7212 // The source block is full
7213 return 0;
7214 }
7215 }
7216 else
7217 {
7218 if (IS_FULL_BLOCK(dst))
7219 {
7220 // The only case when we have to allocate the new block:
7221 // dst is all set and src exists
7222 return const_cast<bm::word_t*>(src);
7223 }
7224 }
7225 }
7226 return ret;
7227}
7228
7229
7230/*!
7231 \brief Plain bitblock XOR operation.
7232 Function does not analyse availability of source and destination blocks.
7233
7234 \param dst - destination block.
7235 \param src - source block.
7236
7237 @ingroup bitfunc
7238*/
7239inline
7241 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7242{
7243 BM_ASSERT(dst);
7244 BM_ASSERT(src);
7245 BM_ASSERT(dst != src);
7246
7247#ifdef BMVECTOPT
7248 bm::id64_t acc = VECT_XOR_BLOCK(dst, src);
7249#else
7250 unsigned arr_sz = bm::set_block_size / 2;
7251
7254
7255 bm::id64_t acc = 0;
7256 for (unsigned i = 0; i < arr_sz; i+=4)
7257 {
7258 acc |= dst_u->w64[i] ^= src_u->w64[i];
7259 acc |= dst_u->w64[i+1] ^= src_u->w64[i+1];
7260 acc |= dst_u->w64[i+2] ^= src_u->w64[i+2];
7261 acc |= dst_u->w64[i+3] ^= src_u->w64[i+3];
7262 }
7263#endif
7264 return acc;
7265}
7266
7267/*!
7268 \brief bitblock AND NOT with constant ~0 mask operation.
7269
7270 \param dst - destination block.
7271 \param src - source block.
7272
7273 @ingroup bitfunc
7274*/
7275inline
7277 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7278{
7279 const bm::word_t* BMRESTRICT src_end = src + bm::set_block_size;
7280#ifdef BMVECTOPT
7281 VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, ~0u);
7282#else
7283 bm::wordop_t* dst_ptr = (wordop_t*)dst;
7284 const bm::wordop_t* wrd_ptr = (wordop_t*) src;
7285 const bm::wordop_t* wrd_end = (wordop_t*) src_end;
7286
7287 do
7288 {
7289 dst_ptr[0] = bm::all_bits_mask & ~wrd_ptr[0];
7290 dst_ptr[1] = bm::all_bits_mask & ~wrd_ptr[1];
7291 dst_ptr[2] = bm::all_bits_mask & ~wrd_ptr[2];
7292 dst_ptr[3] = bm::all_bits_mask & ~wrd_ptr[3];
7293 dst_ptr+=4; wrd_ptr+=4;
7294 } while (wrd_ptr < wrd_end);
7295#endif
7296}
7297
7298/*!
7299 \brief bitblock XOR operation.
7300
7301 \param dst - destination block.
7302 \param src - source block.
7303
7304 \returns pointer on destination block.
7305 If returned value equal to src means that block mutation requested.
7306 NULL is valid return value.
7307
7308 @ingroup bitfunc
7309*/
7310inline
7312 const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7313{
7314 BM_ASSERT(dst || src);
7315 if (src == dst) return 0; // XOR rule
7316
7317 bm::word_t* ret = dst;
7318
7319 if (IS_VALID_ADDR(dst)) // The destination block already exists
7320 {
7321 if (!src) return dst;
7322
7323 bit_block_xor(dst, src);
7324 }
7325 else // The destination block does not exist yet
7326 {
7327 if (!src) return dst; // 1 xor 0 = 1
7328
7329 // Here we have two cases:
7330 // if dest block is full or zero if zero we need to copy the source
7331 // otherwise XOR loop against 0xFF...
7332 //BM_ASSERT(dst == 0);
7333 return const_cast<bm::word_t*>(src); // src is the final result
7334 }
7335 return ret;
7336}
7337
7338/*!
7339 \brief Performs bitblock XOR operation and calculates bitcount of the result.
7340
7341 \param src1 - bit block start ptr
7342 \param src2 - second bit block
7343
7344 \returns bitcount value
7345
7346 @ingroup bitfunc
7347*/
7348inline
7350 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
7351{
7352 if (src1 == src2)
7353 return 0;
7354
7355 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
7356 {
7357 const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
7358 if (IS_FULL_BLOCK(block))
7359 return bm::gap_max_bits;
7360 return bm::bit_block_count(block);
7361 }
7362 if (src1 == FULL_BLOCK_FAKE_ADDR)
7363 src1 = FULL_BLOCK_REAL_ADDR;
7364 if (src2 == FULL_BLOCK_FAKE_ADDR)
7365 src2 = FULL_BLOCK_REAL_ADDR;
7366
7367 return bit_block_xor_count(src1, src2);
7368}
7369
7370/*!
7371 \brief Performs bitblock XOR operation test.
7372
7373 \param src1 - bit block start ptr
7374 \param src2 - second bit block ptr
7375
7376 \returns non zero value if there are bits
7377
7378 @ingroup bitfunc
7379*/
7380inline
7382 const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
7383{
7384 if (src1 == src2)
7385 return 0;
7386 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
7387 {
7388 const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
7389 return !bit_is_all_zero(block);
7390 }
7391 return bm::bit_block_xor_any(src1, src2);
7392}
7393
7394
7395
7396/**
7397 \brief Inspects block for full zero words
7398
7399 \param blk - bit block pointer
7400 \param data_size - data size
7401
7402 \return size of all non-zero words
7403
7404 @ingroup bitfunc
7405*/
7406template<class T>
7407unsigned bit_count_nonzero_size(const T* blk, unsigned data_size) BMNOEXCEPT
7408{
7409 BM_ASSERT(blk && data_size);
7410 unsigned count = 0;
7411 const T* blk_end = blk + data_size - 2;
7412 do
7413 {
7414 if (*blk == 0)
7415 {
7416 // scan fwd to find 0 island length
7417 const T* blk_j = blk + 1;
7418 for (; blk_j < blk_end; ++blk_j)
7419 {
7420 if (*blk_j != 0)
7421 break;
7422 }
7423 blk = blk_j-1;
7424 count += (unsigned)sizeof(gap_word_t);
7425 }
7426 else
7427 {
7428 // scan fwd to find non-0 island length
7429 const T* blk_j = blk + 1;
7430 for ( ; blk_j < blk_end; ++blk_j)
7431 {
7432 if (*blk_j == 0)
7433 {
7434 // look ahead to identify and ignore short 0-run
7435 if (blk_j[1] | blk_j[2])
7436 {
7437 // skip zero word
7438 ++blk_j;
7439 continue;
7440 }
7441 break;
7442 }
7443 }
7444 count += unsigned(sizeof(gap_word_t));
7445 // count all bit-words now
7446 count += unsigned(blk_j - blk) * unsigned(sizeof(T));
7447 blk = blk_j;
7448 }
7449 ++blk;
7450 }
7451 while(blk < blk_end);
7452 return count + unsigned(2 * sizeof(T));
7453}
7454
7455
7456/**
7457 \brief Searches for the next 1 bit in the BIT block
7458 \param block - BIT buffer
7459 \param nbit - bit index to start checking from
7460 \param pos - [out] found value
7461
7462 \return 0 if not found
7463
7464 @ingroup bitfunc
7465*/
7466inline
7468 unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
7469{
7470 BM_ASSERT(block);
7471 BM_ASSERT(pos);
7472
7473 unsigned nword = unsigned(nbit >> bm::set_word_shift);
7474 unsigned bit_pos = (nbit & bm::set_word_mask);
7475
7476 bm::word_t w = block[nword];
7477 w &= (1u << bit_pos);
7478 if (w)
7479 {
7480 *pos = nbit;
7481 return 1;
7482 }
7483 w = block[nword] >> bit_pos;
7484 w <<= bit_pos; // clear the trailing bits
7485 if (w)
7486 {
7487 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
7488 *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t))));
7489 return 1;
7490 }
7491
7492 for (unsigned i = nword+1; i < bm::set_block_size; ++i)
7493 {
7494 w = block[i];
7495 if (w)
7496 {
7497 bit_pos = bm::bit_scan_forward32(w); // trailing zeros
7498 *pos = unsigned(bit_pos + (i * 8u * unsigned(sizeof(bm::word_t))));
7499 return w;
7500 }
7501 } // for i
7502 return 0u;
7503}
7504
7505
7506
7507
7508/*!
7509 \brief BIT block find the last set bit (backward search)
7510
7511 \param block - bit block buffer pointer
7512 \param last - index of the last 1 bit (out)
7513 \return true if found
7514
7515 @ingroup bitfunc
7516*/
7517inline
7519 unsigned* BMRESTRICT last) BMNOEXCEPT
7520{
7521 BM_ASSERT(block);
7522 BM_ASSERT(last);
7523
7524 // TODO: SIMD version
7525
7526 for (unsigned i = bm::set_block_size-1; true; --i)
7527 {
7528 bm::word_t w = block[i];
7529 if (w)
7530 {
7531 unsigned idx = bm::bit_scan_reverse(w);
7532 *last = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7533 return w;
7534 }
7535 if (i == 0)
7536 break;
7537 } // for i
7538 return 0u;
7539}
7540
7541/*!
7542 \brief BIT block find the first set bit
7543
7544 \param block - bit block buffer pointer
7545 \param pos - index of the first 1 bit (out)
7546 \return 0 if not found
7547
7548 @ingroup bitfunc
7549 @internal
7550*/
7551inline
7553 unsigned* BMRESTRICT pos) BMNOEXCEPT
7554{
7555 BM_ASSERT(block);
7556 BM_ASSERT(pos);
7557
7558#ifdef VECT_BIT_FIND_FIRST
7559 return VECT_BIT_FIND_FIRST(block, pos);
7560#else
7561 for (unsigned i = 0; i < bm::set_block_size; ++i)
7562 {
7563 bm::word_t w = block[i];
7564 if (w)
7565 {
7566 unsigned idx = bm::bit_scan_forward32(w); // trailing zeros
7567 *pos = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7568 return w;
7569 }
7570 } // for i
7571 return false;
7572#endif
7573}
7574
7575/*!
7576 \brief BIT block find the first set bit
7577
7578 \param block - bit block buffer pointer
7579 \param first - index of the first 1 bit (out)
7580 \param digest - known digest of dst block
7581
7582 \return 0 if not found
7583
7584 @ingroup bitfunc
7585*/
7586inline
7588 unsigned* BMRESTRICT first,
7589 bm::id64_t digest) BMNOEXCEPT
7590{
7591 BM_ASSERT(block);
7592 BM_ASSERT(first);
7593 BM_ASSERT(digest);
7594
7595 bm::id64_t t = bm::bmi_blsi_u64(digest); // d & -d;
7596
7597 unsigned wave = bm::word_bitcount64(t - 1);
7598 unsigned off = wave * bm::set_block_digest_wave_size;
7599 for (unsigned i = off; i < bm::set_block_size; ++i)
7600 {
7601 bm::word_t w = block[i];
7602 if (w)
7603 {
7604 unsigned idx = bit_scan_forward32(w); // trailing zeros
7605 *first = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7606 return w;
7607 }
7608 } // for i
7609 return 0u;
7610}
7611
7612
7613/*!
7614 \brief BIT block find the first set bit if only 1 bit is set
7615
7616 \param block - bit block buffer pointer
7617 \param first - index of the first 1 bit (out)
7618 \param digest - known digest of dst block
7619
7620 \return 0 if not found
7621
7622 @ingroup bitfunc
7623*/
7624inline
7626 unsigned* BMRESTRICT first,
7627 bm::id64_t digest) BMNOEXCEPT
7628{
7629 BM_ASSERT(block);
7630 BM_ASSERT(first);
7631
7632 unsigned bc = bm::word_bitcount64(digest);
7633 if (bc != 1)
7634 return false;
7635
7636 bool found = false;
7637 bm::id64_t t = bm::bmi_blsi_u64(digest); // d & -d;
7638
7639 unsigned wave = bm::word_bitcount64(t - 1);
7640 unsigned off = wave * bm::set_block_digest_wave_size;
7641 unsigned i;
7642 for (i = off; i < off + bm::set_block_digest_wave_size; ++i)
7643 {
7644 bm::word_t w = block[i];
7645 if (w)
7646 {
7647 bc = bm::word_bitcount(w);
7648 if (bc != 1)
7649 return false;
7650
7651 unsigned idx = bit_scan_forward32(w); // trailing zeros
7652 *first = unsigned(idx + (i * 8u * sizeof(bm::word_t)));
7653 found = true;
7654 break;
7655 }
7656 } // for i
7657
7658 // check if all other bits are zero
7659 for (++i; i < off + bm::set_block_digest_wave_size; ++i)
7660 {
7661 if (block[i])
7662 return false;
7663 }
7664 return found;
7665}
7666
7667
7668/*!
7669 \brief BIT block find position for the rank
7670
7671 \param block - bit block buffer pointer
7672 \param rank - rank to find (must be > 0)
7673 \param nbit_from - start bit position in block
7674 \param nbit_pos - (out)found position
7675
7676 \return 0 if position with rank was found, or
7677 the remaining rank (rank - population count)
7678
7679 @ingroup bitfunc
7680*/
7681template<typename SIZE_TYPE>
7682SIZE_TYPE bit_find_rank(const bm::word_t* const block,
7683 SIZE_TYPE rank,
7684 unsigned nbit_from,
7685 unsigned& nbit_pos) BMNOEXCEPT
7686{
7687 BM_ASSERT(block);
7688 BM_ASSERT(rank);
7689
7690 unsigned nword = nbit_from >> bm::set_word_shift;
7692
7693 unsigned pos = nbit_from;
7694 bm::id_t nbit = (nbit_from & bm::set_word_mask);
7695
7696 if (nbit)
7697 {
7698 bm::id_t w = block[nword];
7699 w >>= nbit;
7700 unsigned bc = bm::word_bitcount(w);
7701 if (bc < rank) // skip this
7702 {
7703 rank -= bc; pos += unsigned(32u - nbit);
7704 ++nword;
7705 }
7706 else // target word
7707 {
7708 unsigned idx = bm::word_select64(w, unsigned(rank));
7709 nbit_pos = pos + idx;
7710 return 0;
7711 }
7712 }
7713
7714 if (bm::conditional<sizeof(void*) == 8>::test()) // 64-bit fast scan
7715 {
7716 for (; nword < bm::set_block_size-1; nword+=2)
7717 {
7718 bm::id64_t w = (bm::id64_t(block[nword+1]) << 32) | bm::id64_t(block[nword]);
7720 if (bc >= rank) // target
7721 {
7722 unsigned idx = bm::word_select64(w, unsigned(rank));
7723 nbit_pos = pos + idx;
7724 return 0;
7725 }
7726 rank -= bc;
7727 pos += 64u;
7728 }
7729 }
7730
7731 for (; nword < bm::set_block_size; ++nword)
7732 {
7733 bm::id_t w = block[nword];
7735 if (rank > bc)
7736 {
7737 rank -= bc; pos += 32u;
7738 continue;
7739 }
7740 unsigned idx = bm::word_select64(w, unsigned(rank));
7741 nbit_pos = pos + idx;
7742 return 0;
7743 } // for nword
7744 return rank;
7745}
7746
7747/**
7748 \brief Find rank in block (GAP or BIT)
7749
7750 \param block - bit block buffer pointer
7751 \param rank - rank to find (must be > 0)
7752 \param nbit_from - start bit position in block
7753 \param nbit_pos - found position
7754
7755 \return 0 if position with rank was found, or
7756 the remaining rank (rank - population count)
7757
7758 @internal
7759*/
7760template<typename SIZE_TYPE>
7761SIZE_TYPE block_find_rank(const bm::word_t* const block,
7762 SIZE_TYPE rank,
7763 unsigned nbit_from,
7764 unsigned& nbit_pos) BMNOEXCEPT
7765{
7766 if (BM_IS_GAP(block))
7767 {
7768 const bm::gap_word_t* const gap_block = BMGAP_PTR(block);
7769 rank = bm::gap_find_rank(gap_block, rank, nbit_from, nbit_pos);
7770 }
7771 else
7772 {
7773 rank = bm::bit_find_rank(block, rank, nbit_from, nbit_pos);
7774 }
7775 return rank;
7776}
7777
7778
7779
7780/*!
7781 @brief Choose best representation for a bit-block
7782 @ingroup bitfunc
7783*/
7784inline
7786 unsigned total_possible_bitcount,
7787 unsigned gap_count,
7788 unsigned block_size) BMNOEXCEPT
7789{
7790 unsigned arr_size = unsigned(sizeof(bm::gap_word_t) * bit_count + sizeof(bm::gap_word_t));
7791 unsigned gap_size = unsigned(sizeof(bm::gap_word_t) * gap_count + sizeof(bm::gap_word_t));
7792 unsigned inv_arr_size = unsigned(sizeof(bm::gap_word_t) * (total_possible_bitcount - bit_count) + sizeof(bm::gap_word_t));
7793
7794 if ((gap_size < block_size) && (gap_size < arr_size) && (gap_size < inv_arr_size))
7795 {
7796 return bm::set_gap;
7797 }
7798
7799 if (arr_size < inv_arr_size)
7800 {
7801 if ((arr_size < block_size) && (arr_size < gap_size))
7802 {
7803 return bm::set_array1;
7804 }
7805 }
7806 else
7807 {
7808 if ((inv_arr_size < block_size) && (inv_arr_size < gap_size))
7809 {
7810 return bm::set_array0;
7811 }
7812 }
7813 return bm::set_bitset;
7814}
7815
7816/*!
7817 @brief Convert bit block into an array of ints corresponding to 1 bits
7818 @return destination size as a result of block decoding
7819 @ingroup bitfunc
7820*/
7821template<typename T>
7823 const unsigned* BMRESTRICT src,
7824 bm::id_t bits,
7825 unsigned dest_len,
7826 unsigned mask = 0) BMNOEXCEPT
7827{
7828 T* BMRESTRICT pcurr = dest;
7829 for (unsigned bit_idx=0; bit_idx < bits;
7830 ++src,bit_idx += unsigned(sizeof(*src) * 8))
7831 {
7832 unsigned val = *src ^ mask; // invert value by XOR 0xFF..
7833 if (val == 0)
7834 continue;
7835 if (pcurr + unsigned(sizeof(val)*8) >= dest + dest_len) // insufficient space
7836 return 0;
7837 // popscan loop to decode bits in a word
7838 while (val)
7839 {
7840 bm::id_t t = val & -val;
7841 *pcurr++ = (T)(bm::word_bitcount(t - 1) + bit_idx);
7842 val &= val - 1;
7843 }
7844 } // for
7845 return (T)(pcurr - dest);
7846}
7847
7848/**
7849 \brief Checks all conditions and returns true if block consists of only 0 bits
7850 \param blk - Blocks's pointer
7851 \param deep_scan - flag to do full bit block verification (scan)
7852 when deep scan is not requested result can be approximate
7853 \returns true if all bits are in the block are 0
7854
7855 @internal
7856*/
7857inline
7858bool check_block_zero(const bm::word_t* blk, bool deep_scan) BMNOEXCEPT
7859{
7860 if (!blk) return true;
7861 if (IS_FULL_BLOCK(blk)) return false;
7862
7863 bool ret;
7864 if (BM_IS_GAP(blk))
7865 ret = gap_is_all_zero(BMGAP_PTR(blk));
7866 else
7867 ret = deep_scan ? bm::bit_is_all_zero(blk) : 0;
7868 return ret;
7869}
7870
7871
7872/**
7873 \brief Checks if block has only 1 bits
7874 \param blk - Block's pointer
7875 \param deep_scan - flag to do full bit block verification (scan)
7876 when deep scan is not requested result can be approximate
7877 \return true if block consists of 1 bits.
7878
7879 @internal
7880*/
7881inline
7882bool check_block_one(const bm::word_t* blk, bool deep_scan) BMNOEXCEPT
7883{
7884 if (blk == 0) return false;
7885
7886 if (BM_IS_GAP(blk))
7887 return bm::gap_is_all_one(BMGAP_PTR(blk));
7888
7889 if (IS_FULL_BLOCK(blk))
7890 return true;
7891
7892 if (!deep_scan)
7893 return false; // block exists - presume it has 0 bits
7894
7895 return bm::is_bits_one((wordop_t*)blk);
7896}
7897
7898
7899
7900/*! @brief Calculates memory overhead for number of gap blocks sharing
7901 the same memory allocation table (level lengths table).
7902 @ingroup gapfunc
7903*/
7904template<typename T>
7905unsigned gap_overhead(const T* length,
7906 const T* length_end,
7907 const T* glevel_len) BMNOEXCEPT
7908{
7909 BM_ASSERT(length && length_end && glevel_len);
7910
7911 unsigned overhead = 0;
7912 for (;length < length_end; ++length)
7913 {
7914 unsigned len = *length;
7915 int level = gap_calc_level(len, glevel_len);
7916 BM_ASSERT(level >= 0 && level < (int)bm::gap_levels);
7917 unsigned capacity = glevel_len[level];
7918 BM_ASSERT(capacity >= len);
7919 overhead += capacity - len;
7920 }
7921 return overhead;
7922}
7923
7924
7925/*! @brief Finds optimal gap blocks lengths.
7926 @param length - first element of GAP lengths array
7927 @param length_end - end of the GAP lengths array
7928 @param glevel_len - destination GAP lengths array
7929 @ingroup gapfunc
7930*/
7931template<typename T>
7932bool improve_gap_levels(const T* length,
7933 const T* length_end,
7934 T* glevel_len) BMNOEXCEPT
7935{
7936 BM_ASSERT(length && length_end && glevel_len);
7937
7938 size_t lsize = size_t(length_end - length);
7939
7940 BM_ASSERT(lsize);
7941
7942 gap_word_t max_len = 0;
7943 unsigned i;
7944 for (i = 0; i < lsize; ++i)
7945 {
7946 if (length[i] > max_len)
7947 max_len = length[i];
7948 }
7949 if (max_len < 5 || lsize <= bm::gap_levels)
7950 {
7951 glevel_len[0] = T(max_len + 4);
7952 for (i = 1; i < bm::gap_levels; ++i)
7953 {
7954 glevel_len[i] = bm::gap_max_buff_len;
7955 }
7956 return true;
7957 }
7958
7959 glevel_len[bm::gap_levels-1] = T(max_len + 5);
7960
7961 unsigned min_overhead = gap_overhead(length, length_end, glevel_len);
7962 bool is_improved = false;
7963
7964 // main problem solving loop
7965 //
7966 for (i = bm::gap_levels-2; ; --i)
7967 {
7968 unsigned opt_len = 0;
7969 unsigned j;
7970 bool imp_flag = false;
7971 gap_word_t gap_saved_value = glevel_len[i];
7972 for (j = 0; j < lsize; ++j)
7973 {
7974 glevel_len[i] = T(length[j] + 4);
7975 unsigned ov = gap_overhead(length, length_end, glevel_len);
7976 if (ov <= min_overhead)
7977 {
7978 min_overhead = ov;
7979 opt_len = length[j]+4;
7980 imp_flag = true;
7981 }
7982 }
7983 if (imp_flag)
7984 {
7985 glevel_len[i] = (T)opt_len; // length[opt_idx]+4;
7986 is_improved = true;
7987 }
7988 else
7989 {
7990 glevel_len[i] = gap_saved_value;
7991 }
7992 if (i == 0)
7993 break;
7994 }
7995
7996 // Remove duplicates
7997 //
7998 T val = *glevel_len;
7999 T* gp = glevel_len;
8000 T* res = glevel_len;
8001 for (i = 0; i < bm::gap_levels; ++i)
8002 {
8003 if (val != *gp)
8004 {
8005 val = *gp;
8006 *++res = val;
8007 }
8008 ++gp;
8009 }
8010
8011 // Filling the "unused" part with max. possible value
8012 while (++res < (glevel_len + bm::gap_levels))
8013 {
8014 *res = bm::gap_max_buff_len;
8015 }
8016
8017 return is_improved;
8018
8019}
8020
8021/*!
8022 \brief Find first bit which is different between two blocks (GAP or bit)
8023 \param blk - block 1
8024 \param arg_blk - block 2
8025 \param pos - out - position of difference (undefined if blocks are equal)
8026 \return true if difference was found
8027 @internal
8028*/
8029inline
8031 const bm::word_t* BMRESTRICT arg_blk,
8032 unsigned* BMRESTRICT pos) BMNOEXCEPT
8033{
8034 // If one block is zero we check if the other one has at least
8035 // one bit ON
8036
8037 if (!blk || !arg_blk)
8038 {
8039 const bm::word_t* pblk; bool is_gap;
8040 if (blk)
8041 {
8042 pblk = blk;
8043 is_gap = BM_IS_GAP(blk);
8044 }
8045 else
8046 {
8047 pblk = arg_blk;
8048 is_gap = BM_IS_GAP(arg_blk);
8049 }
8050
8051 if (is_gap)
8052 {
8053 unsigned found = bm::gap_find_first(BMGAP_PTR(pblk), pos);
8054 if (found)
8055 return true;
8056 }
8057 else
8058 {
8059 unsigned found = bm::bit_find_first(pblk, pos);
8060 if (found)
8061 return true;
8062 }
8063 return false;
8064 }
8065
8066 bool arg_gap = BM_IS_GAP(arg_blk);
8067 bool gap = BM_IS_GAP(blk);
8068
8069 if (arg_gap != gap)
8070 {
8071 //BM_DECLARE_TEMP_BLOCK(temp_blk);
8072 bm::bit_block_t temp_blk;
8073 bm::word_t* blk1; bm::word_t* blk2;
8074
8075 if (gap)
8076 {
8078 BMGAP_PTR(blk));
8079 blk1 = (bm::word_t*)temp_blk;
8080 blk2 = (bm::word_t*)arg_blk;
8081 }
8082 else
8083 {
8085 BMGAP_PTR(arg_blk));
8086 blk1 = (bm::word_t*)blk;
8087 blk2 = (bm::word_t*)temp_blk;
8088 }
8089 bool found = bm::bit_find_first_diff(blk1, blk2, pos);
8090 if (found)
8091 return true;
8092 }
8093 else
8094 {
8095 if (gap)
8096 {
8097 bool found =
8099 BMGAP_PTR(arg_blk), pos);
8100 if (found)
8101 return true;
8102 }
8103 else
8104 {
8105 bool found = bm::bit_find_first_diff(blk, arg_blk, pos);
8106 if (found)
8107 return true;
8108 }
8109 }
8110 return false;
8111}
8112
8113
8114
8115
8116/**
8117 Bit-block get adapter, takes bitblock and represents it as a
8118 get_32() accessor function
8119 \internal
8120*/
8122{
8123public:
8124 bitblock_get_adapter(const bm::word_t* bit_block) : b_(bit_block) {}
8125
8127 bm::word_t get_32() BMNOEXCEPT { return *b_++; }
8128private:
8129 const bm::word_t* b_;
8130};
8131
8132
8133/**
8134 Bit-block store adapter, takes bitblock and saves results into it
8135 \internal
8136*/
8138{
8139public:
8140 bitblock_store_adapter(bm::word_t* bit_block) : b_(bit_block) {}
8142 void push_back(bm::word_t w) { *b_++ = w; }
8143private:
8144 bm::word_t* b_;
8145};
8146
8147/**
8148 Bit-block sum adapter, takes values and sums it
8149 /internal
8150*/
8152{
8153public:
8156 void push_back(bm::word_t w) BMNOEXCEPT { this->sum_+= w; }
8157 /// Get accumulated sum
8158 bm::word_t sum() const BMNOEXCEPT { return this->sum_; }
8159private:
8160 bm::word_t sum_;
8161};
8162
8163/**
8164 Adapter to get words from a range stream
8165 (see range serialized bit-block)
8166 \internal
8167*/
8168template<class DEC> class decoder_range_adapter
8169{
8170public:
8171 decoder_range_adapter(DEC& dec, unsigned from_idx, unsigned to_idx)
8172 : decoder_(dec),
8173 from_(from_idx),
8174 to_(to_idx),
8175 cnt_(0)
8176 {}
8177
8179 {
8180 if (cnt_ < from_ || cnt_ > to_)
8181 {
8182 ++cnt_; return 0;
8183 }
8184 ++cnt_;
8185 return decoder_.get_32();
8186 }
8187
8188private:
8189 DEC& decoder_;
8190 unsigned from_;
8191 unsigned to_;
8192 unsigned cnt_;
8193};
8194
8195
8196/*!
8197 Abstract recombination algorithm for two bit-blocks
8198 Bit blocks can come as dserialization decoders or bit-streams
8199*/
8200template<class It1, class It2, class BinaryOp, class Encoder>
8201void bit_recomb(It1& it1, It2& it2,
8202 BinaryOp& op,
8203 Encoder& enc,
8204 unsigned block_size = bm::set_block_size) BMNOEXCEPT
8205{
8206 for (unsigned i = 0; i < block_size; ++i)
8207 {
8208 bm::word_t w1 = it1.get_32();
8209 bm::word_t w2 = it2.get_32();
8210 bm::word_t w = op(w1, w2);
8211 enc.push_back( w );
8212 } // for
8213}
8214
8215/// Bit AND functor
8216template<typename W> struct bit_AND
8217{
8218 W operator()(W w1, W w2) BMNOEXCEPT { return w1 & w2; }
8219};
8220
8221/// Bit OR functor
8222template<typename W> struct bit_OR
8223{
8224 W operator()(W w1, W w2) BMNOEXCEPT { return w1 | w2; }
8225};
8226
8227/// Bit SUB functor
8228template<typename W> struct bit_SUB
8229{
8230 W operator()(W w1, W w2) BMNOEXCEPT { return w1 & ~w2; }
8231};
8232
8233/// Bit XOR functor
8234template<typename W> struct bit_XOR
8235{
8236 W operator()(W w1, W w2) BMNOEXCEPT { return w1 ^ w2; }
8237};
8238
8239/// Bit ASSIGN functor
8240template<typename W> struct bit_ASSIGN
8241{
8242 W operator()(W, W w2) BMNOEXCEPT { return w2; }
8243};
8244
8245/// Bit COUNT functor
8246template<typename W> struct bit_COUNT
8247{
8249 {
8250 w1 = 0;
8251 BM_INCWORD_BITCOUNT(w1, w2);
8252 return w1;
8253 }
8254};
8255
8256/// Bit COUNT AND functor
8257template<typename W> struct bit_COUNT_AND
8258{
8260 {
8261 W r = 0;
8262 BM_INCWORD_BITCOUNT(r, w1 & w2);
8263 return r;
8264 }
8265};
8266
8267/// Bit COUNT XOR functor
8268template<typename W> struct bit_COUNT_XOR
8269{
8271 {
8272 W r = 0;
8273 BM_INCWORD_BITCOUNT(r, w1 ^ w2);
8274 return r;
8275 }
8276};
8277
8278/// Bit COUNT OR functor
8279template<typename W> struct bit_COUNT_OR
8280{
8282 {
8283 W r = 0;
8284 BM_INCWORD_BITCOUNT(r, w1 | w2);
8285 return r;
8286 }
8287};
8288
8289
8290/// Bit COUNT SUB AB functor
8291template<typename W> struct bit_COUNT_SUB_AB
8292{
8294 {
8295 W r = 0;
8296 BM_INCWORD_BITCOUNT(r, w1 & (~w2));
8297 return r;
8298 }
8299};
8300
8301/// Bit SUB BA functor
8302template<typename W> struct bit_COUNT_SUB_BA
8303{
8305 {
8306 W r = 0;
8307 BM_INCWORD_BITCOUNT(r, w2 & (~w1));
8308 return r;
8309 }
8310};
8311
8312/// Bit COUNT A functor
8313template<typename W> struct bit_COUNT_A
8314{
8316 {
8317 W r = 0;
8318 BM_INCWORD_BITCOUNT(r, w1);
8319 return r;
8320 }
8321};
8322
8323/// Bit COUNT B functor
8324template<typename W> struct bit_COUNT_B
8325{
8327 {
8328 W r = 0;
8329 BM_INCWORD_BITCOUNT(r, w2);
8330 return r;
8331 }
8332};
8333
8334typedef
8336 const gap_word_t*);
8337
8338typedef
8339gap_word_t* (*gap_operation_func_type)(const gap_word_t* BMRESTRICT,
8340 const gap_word_t* BMRESTRICT,
8342 unsigned& );
8343
8344typedef
8346 const bm::word_t* BMRESTRICT);
8347
8348
8349template<bool T>
8377
8378template<bool T>
8385 0
8386};
8387
8388template<bool T>
8391 &gap_operation_and, // set_AND
8392 &gap_operation_or, // set_OR
8393 &gap_operation_sub, // set_SUB
8394 &gap_operation_xor, // set_XOR
8395 0
8396};
8397
8398
8399template<bool T>
8402 0, // set_AND
8403 0, // set_OR
8404 0, // set_SUB
8405 0, // set_XOR
8406 0, // set_ASSIGN
8407 0, // set_COUNT
8408 &bit_operation_and_count, // set_COUNT_AND
8409 &bit_operation_xor_count, // set_COUNT_XOR
8410 &bit_operation_or_count, // set_COUNT_OR
8411 &bit_operation_sub_count, // set_COUNT_SUB_AB
8412 &bit_operation_sub_count_inv, // set_COUNT_SUB_BA
8413 0, // set_COUNT_A
8414 0, // set_COUNT_B
8415};
8416
8417/**
8418 Size of bit decode wave in words
8419 @internal
8420 */
8421const unsigned short set_bitscan_wave_size = 4;
8422/*!
8423 \brief Unpacks word wave (Nx 32-bit words)
8424 \param w_ptr - pointer on wave start
8425 \param bits - pointer on the result array
8426 \return number of bits in the list
8427
8428 @ingroup bitfunc
8429 @internal
8430*/
8431inline
8432unsigned short
8434 unsigned char* BMRESTRICT bits) BMNOEXCEPT
8435{
8436 bm::word_t w0, w1;
8437 unsigned short cnt0;
8438
8439 w0 = w_ptr[0];
8440 w1 = w_ptr[1];
8441
8442#if defined(BMAVX512OPT) || defined(BMAVX2OPT) || defined(BMSSE42OPT)
8443 // combine into 64-bit word and scan (when HW popcnt64 is available)
8444 bm::id64_t w = (bm::id64_t(w1) << 32) | w0;
8445 cnt0 = (unsigned short) bm::bitscan_popcnt64(w, bits);
8446
8447 w0 = w_ptr[2];
8448 w1 = w_ptr[3];
8449 w = (bm::id64_t(w1) << 32) | w0;
8450 cnt0 += bm::bitscan_popcnt64(w, bits + cnt0, 64);
8451#else
8452 // decode wave as two 32-bit bitscan decodes
8453 cnt0 = bm::bitscan_popcnt(w0, bits);
8454 cnt0 += bm::bitscan_popcnt(w1, bits + cnt0, 32);
8455
8456 w0 = w_ptr[2];
8457 w1 = w_ptr[3];
8458 cnt0 += bm::bitscan_popcnt(w0, bits + cnt0, 64);
8459 cnt0 += bm::bitscan_popcnt(w1, bits + cnt0, 64+32);
8460#endif
8461 return cnt0;
8462}
8463
8464#if defined (BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
8465/**
8466 bit index to word gather-scatter algorithm (SIMD)
8467 @ingroup bitfunc
8468 @internal
8469*/
8470inline
8472 const bm::word_t* BMRESTRICT blk,
8473 const unsigned* BMRESTRICT idx,
8474 unsigned size, unsigned start,
8475 unsigned bit_idx) BMNOEXCEPT
8476{
8477typedef unsigned TRGW;
8478typedef unsigned IDX;
8479#if defined(BM64_SSE4)
8480 // optimized for unsigned
8481 if (bm::conditional<sizeof(TRGW)==4 && sizeof(IDX)==4>::test())
8482 {
8483 sse4_bit_block_gather_scatter(arr, blk, idx, size, start, bit_idx);
8484 return;
8485 }
8486#elif defined(BM64_AVX2) || defined(BM64_AVX512)
8487 if (bm::conditional<sizeof(TRGW)==4 && sizeof(IDX)==4>::test())
8488 {
8489 avx2_bit_block_gather_scatter(arr, blk, idx, size, start, bit_idx);
8490 return;
8491 }
8492#else
8493 BM_ASSERT(0);
8494#endif
8495}
8496#endif
8497
8498/**
8499 bit index to word gather-scatter algorithm
8500 @ingroup bitfunc
8501 @internal
8502*/
8503template<typename TRGW, typename IDX, typename SZ>
8505 const bm::word_t* BMRESTRICT blk,
8506 const IDX* BMRESTRICT idx,
8507 SZ size, SZ start, unsigned bit_idx) BMNOEXCEPT
8508{
8509 // TODO: SIMD for 64-bit index sizes and 64-bit target value size
8510 //
8511 TRGW mask1 = 1;
8512 const SZ len = (size - start);
8513 const SZ len_unr = len - (len % 2);
8514 SZ k = 0;
8515 for (; k < len_unr; k+=2)
8516 {
8517 const SZ base = start + k;
8518 const unsigned nbitA = unsigned(idx[base] & bm::set_block_mask);
8519 arr[base] |= (TRGW(bool(blk[nbitA >> bm::set_word_shift] &
8520 (mask1 << (nbitA & bm::set_word_mask)))) << bit_idx);
8521 const unsigned nbitB = unsigned(idx[base + 1] & bm::set_block_mask);
8522 arr[base+1] |= (TRGW(bool(blk[nbitB >> bm::set_word_shift] &
8523 (mask1 << (nbitB & bm::set_word_mask)))) << bit_idx);
8524 } // for k
8525 for (; k < len; ++k)
8526 {
8527 unsigned nbit = unsigned(idx[start + k] & bm::set_block_mask);
8528 arr[start + k] |= (TRGW(bool(blk[nbit >> bm::set_word_shift] &
8529 (mask1 << (nbit & bm::set_word_mask)))) << bit_idx);
8530 } // for k
8531}
8532
8533/**
8534 @brief block boundaries look ahead U32
8535
8536 @param idx - array to look into
8537 @param size - array size
8538 @param nb - block number to look ahead
8539 @param start - start offset in idx
8540
8541 @return block boundary offset end (no more match at the returned offset)
8542
8543 @internal
8544*/
8545inline
8548{
8549 BM_ASSERT(idx);
8550 BM_ASSERT(start < size);
8551
8552 // TODO: SIMD for 64-bit index vector
8553 for (;(start < size) &&
8554 (nb == (idx[start] >> bm::set_block_shift)); ++start)
8555 {}
8556 return start;
8557}
8558
8559/**
8560 @brief block boundaries look ahead U32
8561
8562 @param idx - array to look into
8563 @param size - array size
8564 @param nb - block number to look ahead
8565 @param start - start offset in idx
8566
8567 @return block boundary offset end (no more match at the returned offset)
8568
8569 @internal
8570*/
8571inline
8572unsigned idx_arr_block_lookup_u32(const unsigned* idx,
8573 unsigned size, unsigned nb, unsigned start) BMNOEXCEPT
8574{
8575 BM_ASSERT(idx);
8576 BM_ASSERT(start < size);
8577
8578#if defined(VECT_ARR_BLOCK_LOOKUP)
8579 return VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start);
8580#else
8581 for (;(start < size) &&
8582 (nb == unsigned(idx[start] >> bm::set_block_shift)); ++start)
8583 {}
8584 return start;
8585#endif
8586}
8587
8588// --------------------------------------------------------------
8589
8590
8591/**
8592 @brief set bits in a bit-block using global index
8593
8594 @param idx - array to look into
8595 @param block - block pointer to set bits
8596 @param start - index array start
8597 @param stop - index array stop in a range [start..stop)
8598
8599 @return block boundary offset end (no more match at the returned offset)
8600
8601 @internal
8602 @ingroup bitfunc
8603*/
8604inline
8606 const bm::id64_t* BMRESTRICT idx,
8607 bm::id64_t start, bm::id64_t stop) BMNOEXCEPT
8608{
8609 // TODO: SIMD for 64-bit mode
8610 for (bm::id64_t i = start; i < stop; ++i)
8611 {
8612 bm::id64_t n = idx[i];
8613 unsigned nbit = unsigned(n & bm::set_block_mask);
8614 unsigned nword = nbit >> bm::set_word_shift;
8615 nbit &= bm::set_word_mask;
8616 block[nword] |= (1u << nbit);
8617 } // for i
8618}
8619
8620
8621/**
8622 @brief set bits in a bit-block using global index
8623
8624 @param idx - array to look into
8625 @param block - block pointer to set bits
8626 @param start - index array start
8627 @param stop - index array stop in a range [start..stop)
8628
8629 @return block boundary offset end (no more match at the returned offset)
8630
8631 @internal
8632 @ingroup bitfunc
8633*/
8634inline
8636 const unsigned* BMRESTRICT idx,
8637 unsigned start, unsigned stop ) BMNOEXCEPT
8638{
8639#if defined(VECT_SET_BLOCK_BITS)
8640 VECT_SET_BLOCK_BITS(block, idx, start, stop);
8641#else
8642 for (unsigned i = start; i < stop; ++i)
8643 {
8644 unsigned n = idx[i];
8645 unsigned nbit = unsigned(n & bm::set_block_mask);
8646 unsigned nword = nbit >> bm::set_word_shift;
8647 nbit &= bm::set_word_mask;
8648 block[nword] |= (1u << nbit);
8649 } // for i
8650#endif
8651}
8652
8653
8654
8655// --------------------------------------------------------------
8656
8657/**
8658 @brief array range detector
8659 @internal
8660*/
8661inline
8663 unsigned& left, unsigned& right) BMNOEXCEPT
8664{
8665 BM_ASSERT(arr);
8666
8667 unsigned i, j;
8668 for (i = 0; i < bm::set_sub_array_size; ++i)
8669 {
8670 if (arr[i])
8671 {
8672 left = i;
8673 break;
8674 }
8675 }
8676 if (i == bm::set_sub_array_size)
8677 {
8678 left = right = 0;
8679 return false; // nothing here
8680 }
8681 for (j = bm::set_sub_array_size-1; j != i; --j)
8682 {
8683 if (arr[j])
8684 break;
8685 }
8686 right = j;
8687 return true;
8688}
8689
8690// --------------------------------------------------------------
8691
8692/**
8693 Linear lower bound search in unsigned array
8694 @internal
8695*/
8696inline
8697unsigned lower_bound_linear_u32(const unsigned* arr, unsigned target,
8698 unsigned from, unsigned to) BMNOEXCEPT
8699{
8700 BM_ASSERT(arr);
8701 BM_ASSERT(from <= to);
8702
8703#if defined(VECT_LOWER_BOUND_SCAN_U32)
8704 return VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to);
8705#else
8706 for (; from <= to; ++from)
8707 {
8708 if (arr[from] >= target)
8709 break;
8710 }
8711 return from;
8712#endif
8713}
8714
8715/**
8716 Linear lower bound search in unsigned LONG array
8717 @internal
8718*/
8719inline
8720unsigned lower_bound_linear_u64(const unsigned long long* arr,
8721 unsigned long long target,
8722 unsigned from, unsigned to) BMNOEXCEPT
8723{
8724 BM_ASSERT(arr);
8725 BM_ASSERT(from <= to);
8726
8727 // TODO: implement vectorized scan on u64 ints
8728 for (; from <= to; ++from)
8729 {
8730 if (arr[from] >= target)
8731 break;
8732 }
8733 return from;
8734}
8735
8736
8737
8738// --------------------------------------------------------------
8739
8740/**
8741 Hybrid, binary-linear lower bound search in unsigned array
8742 @internal
8743*/
8744inline
8745unsigned lower_bound_u32(const unsigned* arr, unsigned target,
8746 unsigned from, unsigned to) BMNOEXCEPT
8747{
8748 BM_ASSERT(arr);
8749 BM_ASSERT(from <= to);
8750 const unsigned linear_cutoff = 32;
8751
8752 unsigned l = from; unsigned r = to;
8753 unsigned dist = r - l;
8754 if (dist < linear_cutoff)
8755 {
8756 return bm::lower_bound_linear_u32(arr, target, l, r);
8757 }
8758
8759 while (l <= r)
8760 {
8761 unsigned mid = (r-l)/2+l;
8762 if (arr[mid] < target)
8763 l = mid+1;
8764 else
8765 r = mid-1;
8766 dist = r - l;
8767 if (dist < linear_cutoff)
8768 {
8769 return bm::lower_bound_linear_u32(arr, target, l, r);
8770 }
8771 }
8772 return l;
8773}
8774
8775/**
8776 Hybrid, binary-linear lower bound search in unsigned LONG array
8777 @internal
8778*/
8779inline
8780unsigned lower_bound_u64(const unsigned long long* arr,
8781 unsigned long long target,
8782 unsigned from, unsigned to) BMNOEXCEPT
8783{
8784 BM_ASSERT(arr);
8785 BM_ASSERT(from <= to);
8786 const unsigned linear_cutoff = 32;
8787
8788 unsigned l = from; unsigned r = to;
8789 unsigned dist = r - l;
8790 if (dist < linear_cutoff)
8791 {
8792 return bm::lower_bound_linear_u64(arr, target, l, r);
8793 }
8794
8795 while (l <= r)
8796 {
8797 unsigned mid = (r - l) / 2 + l;
8798 if (arr[mid] < target)
8799 l = mid + 1;
8800 else
8801 r = mid - 1;
8802 dist = r - l;
8803 if (dist < linear_cutoff)
8804 {
8805 return bm::lower_bound_linear_u64(arr, target, l, r);
8806 }
8807 }
8808 return l;
8809}
8810
8811
8812/**
8813 calculate bvector<> global bit-index from block-local coords
8814 @return bit index in linear bit-vector coordinates
8815 @internal
8816*/
8817#ifdef BM64ADDR
8818inline
8819bm::id64_t block_to_global_index(unsigned i, unsigned j,
8820 unsigned block_idx) BMNOEXCEPT
8821{
8823 base_idx += j * bm::gap_max_bits;
8824 return block_idx + base_idx;
8825}
8826#else
8827inline
8828bm::id_t block_to_global_index(unsigned i, unsigned j,
8829 unsigned block_idx) BMNOEXCEPT
8830{
8831 unsigned base_idx = i * bm::set_sub_array_size * bm::gap_max_bits;
8832 base_idx += j * bm::gap_max_bits;
8833 return block_idx + base_idx;
8834}
8835#endif
8836
8837// --------------------------------------------------------------
8838// Functions to work with int values stored in 64-bit pointers
8839// --------------------------------------------------------------
8840
8841/*!
8842 \brief helper union to interpret pointer as integers
8843 @internal
8844*/
8846{
8849 unsigned short i16[4];
8850};
8851
8852/*!
8853 Test presense of value in payload pointer
8854 @internal
8855*/
8856inline
8858{
8859 if (v == 0)
8860 {
8861 return (ptr.i16[1] == 0);
8862 }
8863 bm::id64_t r = (ptr.i16[1] == v) | (ptr.i16[2] == v) | (ptr.i16[3] == v);
8864 return r;
8865}
8866
8867// --------------------------------------------------------------------------
8868
8869} // namespace bm
8870
8871#endif
Definitions(internal)
#define IS_FULL_BLOCK(addr)
Definition bmdef.h:153
#define IS_VALID_ADDR(addr)
Definition bmdef.h:152
#define BMRESTRICT
Definition bmdef.h:193
#define BMNOEXCEPT
Definition bmdef.h:79
#define BMGAP_PTR(ptr)
Definition bmdef.h:179
#define BM_IS_GAP(ptr)
Definition bmdef.h:181
#define BMFORCEINLINE
Definition bmdef.h:203
#define BM_ASSERT
Definition bmdef.h:130
#define FULL_BLOCK_FAKE_ADDR
Definition bmdef.h:149
#define IS_EMPTY_BLOCK(addr)
Definition bmdef.h:154
#define BMNOEXCEPT2
Definition bmdef.h:82
#define FULL_BLOCK_REAL_ADDR
Definition bmdef.h:148
#define BM_VECT_ALIGN
Definition bmdef.h:359
#define VECT_SET_BLOCK(dst, value)
Definition bmsse4.h:1782
#define VECT_BITCOUNT_OR(first, last, mask)
Definition bmsse4.h:1725
#define VECT_BIT_FIND_DIFF(src1, src2, pos)
Definition bmsse4.h:1829
#define VECT_AND_DIGEST_5WAY(dst, src1, src2, src3, src4)
Definition bmsse4.h:1743
#define VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4)
Definition bmsse4.h:1758
#define VECT_COPY_BLOCK(dst, src)
Definition bmsse4.h:1776
#define VECT_BITCOUNT_AND(first, last, mask)
Definition bmsse4.h:1722
#define VECT_SHIFT_R1(b, acc, co)
Definition bmsse4.h:1803
#define VECT_XOR_BLOCK_2WAY(dst, src1, src2)
Definition bmsse4.h:1773
#define VECT_IS_ONE_BLOCK(dst)
Definition bmsse4.h:1788
#define VECT_STREAM_BLOCK(dst, src)
Definition bmsse4.h:1779
#define VECT_OR_BLOCK(dst, src)
Definition bmsse4.h:1749
#define VECT_SHIFT_R1_AND(b, co, m, digest)
Definition bmsse4.h:1806
#define VECT_SUB_BLOCK(dst, src)
Definition bmsse4.h:1761
#define VECT_IS_ZERO_BLOCK(dst)
Definition bmsse4.h:1785
#define VECT_SUB_DIGEST_2WAY(dst, src1, src2)
Definition bmsse4.h:1767
#define VECT_XOR_BLOCK(dst, src)
Definition bmsse4.h:1770
#define VECT_IS_DIGEST_ZERO(start)
Definition bmsse4.h:1791
#define VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, mask)
Definition bmsse4.h:1716
#define VECT_BLOCK_CHANGE_BC(block, gc, bc)
Definition bmsse4.h:1822
#define VECT_SHIFT_L1(b, acc, co)
Definition bmsse4.h:1800
#define VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to)
Definition bmsse4.h:1797
#define VECT_SET_BLOCK_BITS(block, idx, start, stop)
Definition bmsse4.h:1812
#define VECT_BITCOUNT_SUB(first, last, mask)
Definition bmsse4.h:1731
#define VECT_BITCOUNT_XOR(first, last, mask)
Definition bmsse4.h:1728
#define VECT_AND_DIGEST(dst, src)
Definition bmsse4.h:1740
#define VECT_BIT_FIND_FIRST(src, pos)
Definition bmsse4.h:1826
#define VECT_GAP_BFIND(buf, pos, is_set)
Definition bmsse4.h:1835
#define VECT_INVERT_BLOCK(first)
Definition bmsse4.h:1734
#define VECT_BLOCK_CHANGE(block, size)
Definition bmsse4.h:1815
#define VECT_AND_DIGEST_2WAY(dst, src1, src2)
Definition bmsse4.h:1746
#define VECT_BLOCK_SET_DIGEST(dst, val)
Definition bmsse4.h:1794
#define VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start)
Definition bmsse4.h:1809
#define VECT_BITCOUNT(first, last)
Definition bmsse4.h:1719
#define VECT_SUB_DIGEST(dst, src)
Definition bmsse4.h:1764
#define VECT_OR_BLOCK_3WAY(dst, src1, src2)
Definition bmsse4.h:1755
#define VECT_OR_BLOCK_2WAY(dst, src1, src2)
Definition bmsse4.h:1752
#define VECT_AND_BLOCK(dst, src)
Definition bmsse4.h:1737
Bit manipulation primitives (internal)
Bit-block get adapter, takes bitblock and represents it as a get_32() accessor function.
Definition bmfunc.h:8122
bitblock_get_adapter(const bm::word_t *bit_block)
Definition bmfunc.h:8124
BMFORCEINLINE bm::word_t get_32() BMNOEXCEPT
Definition bmfunc.h:8127
Bit-block store adapter, takes bitblock and saves results into it.
Definition bmfunc.h:8138
BMFORCEINLINE void push_back(bm::word_t w)
Definition bmfunc.h:8142
bitblock_store_adapter(bm::word_t *bit_block)
Definition bmfunc.h:8140
Bit-block sum adapter, takes values and sums it /internal.
Definition bmfunc.h:8152
BMFORCEINLINE void push_back(bm::word_t w) BMNOEXCEPT
Definition bmfunc.h:8156
bm::word_t sum() const BMNOEXCEPT
Get accumulated sum.
Definition bmfunc.h:8158
Adaptor to copy 1 bits to array.
Definition bmfunc.h:387
copy_to_array_functor(B *bits)
Definition bmfunc.h:389
void operator()(unsigned bit_idx0, unsigned bit_idx1, unsigned bit_idx2) BMNOEXCEPT
Definition bmfunc.h:403
void operator()(unsigned bit_idx0, unsigned bit_idx1, unsigned bit_idx2, unsigned bit_idx3) BMNOEXCEPT
Definition bmfunc.h:411
void operator()(unsigned bit_idx0, unsigned bit_idx1) BMNOEXCEPT
Definition bmfunc.h:396
void operator()(unsigned bit_idx) BMNOEXCEPT
Definition bmfunc.h:394
Adapter to get words from a range stream (see range serialized bit-block)
Definition bmfunc.h:8169
bm::word_t get_32() BMNOEXCEPT
Definition bmfunc.h:8178
decoder_range_adapter(DEC &dec, unsigned from_idx, unsigned to_idx)
Definition bmfunc.h:8171
unsigned sse2_gap_test(const unsigned short *BMRESTRICT buf, unsigned pos)
Hybrid binary search, starts as binary, then switches to scan.
Definition bmsse2.h:461
unsigned sse42_gap_test(const unsigned short *BMRESTRICT buf, unsigned pos)
Hybrid binary search, starts as binary, then switches to scan.
Definition bmsse4.h:1106
BMFORCEINLINE bool sse42_test_all_zero_wave(const void *ptr)
check if wave of pointers is all NULL
Definition bmsse4.h:595
bm::id_t bit_operation_and_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock AND operation test.
Definition bmfunc.h:6564
bm::id_t bit_block_calc_count_range(const bm::word_t *block, bm::word_t left, bm::word_t right) BMNOEXCEPT
Definition bmfunc.h:4698
bm::id64_t bit_block_and_5way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src0, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, const bm::word_t *BMRESTRICT src3, bm::id64_t digest) BMNOEXCEPT
digest based bit-block AND 5-way
Definition bmfunc.h:6034
unsigned bit_block_find(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the next 1 bit in the BIT block.
Definition bmfunc.h:7467
bool bit_block_or_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
2 way (target := source1 | source2) bitblock OR operation.
Definition bmfunc.h:6786
bm::id_t bit_operation_sub_count_inv(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs inverted bitblock SUB operation and calculates bitcount of the result.
Definition bmfunc.h:6626
BMFORCEINLINE bm::id_t word_bitcount(bm::id_t w) BMNOEXCEPT
Definition bmfunc.h:197
unsigned bit_block_and_any(const bm::word_t *src1, const bm::word_t *src2) BMNOEXCEPT
Function ANDs two bitblocks and tests for any bit. Function does not analyse availability of source b...
Definition bmfunc.h:6213
unsigned bit_block_sub_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function SUBs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition bmfunc.h:6319
unsigned bit_block_calc_change(const bm::word_t *block) BMNOEXCEPT
Definition bmfunc.h:4607
void bit_block_copy(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Bitblock copy operation.
Definition bmfunc.h:5898
bool bit_block_shift_r1_and(bm::word_t *BMRESTRICT block, bm::word_t co_flag, const bm::word_t *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest) BMNOEXCEPT
Right bit-shift of bit-block by 1 bit (reference) + AND.
Definition bmfunc.h:5089
T bit_convert_to_arr(T *BMRESTRICT dest, const unsigned *BMRESTRICT src, bm::id_t bits, unsigned dest_len, unsigned mask=0) BMNOEXCEPT
Convert bit block into an array of ints corresponding to 1 bits.
Definition bmfunc.h:7822
unsigned bit_count_nonzero_size(const T *blk, unsigned data_size) BMNOEXCEPT
Inspects block for full zero words.
Definition bmfunc.h:7407
bm::id_t bit_count_change(bm::word_t w) BMNOEXCEPT
Definition bmfunc.h:4517
unsigned bit_block_and_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function ANDs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition bmfunc.h:6164
bm::id_t bit_operation_or_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock OR operation test.
Definition bmfunc.h:6716
bm::id64_t bit_block_sub_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, bm::id64_t digest) BMNOEXCEPT
digest based bitblock SUB (AND NOT) operation (3 operand)
Definition bmfunc.h:7115
void block_init_digest0(bm::word_t *const block, bm::id64_t digest) BMNOEXCEPT
Init block with 000111000 pattren based on digest.
Definition bmfunc.h:704
bool check_zero_digest(bm::id64_t digest, unsigned bitpos_from, unsigned bitpos_to) BMNOEXCEPT
check if all digest bits for the range [from..to] are 0
Definition bmfunc.h:688
void or_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
Sets bits to 1 in the bitblock.
Definition bmfunc.h:3170
bm::id64_t bit_block_and(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock AND operation. Function does not analyse availability of source and destination blocks...
Definition bmfunc.h:5940
bm::id_t bit_block_count(const bm::word_t *block) BMNOEXCEPT
Bitcount for bit block.
Definition bmfunc.h:4379
bool bit_block_is_all_one_range(const bm::word_t *const BMRESTRICT block, bm::word_t left, bm::word_t right) BMNOEXCEPT
Definition bmfunc.h:4621
bool bit_block_or_5way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, const bm::word_t *BMRESTRICT src3, const bm::word_t *BMRESTRICT src4) BMNOEXCEPT
5 way (target, source1, source2) bitblock OR operation. Function does not analyse availability of sou...
Definition bmfunc.h:6910
BMFORCEINLINE void clear_bit(unsigned *dest, unsigned bitpos) BMNOEXCEPT
Set 1 bit in a block.
Definition bmfunc.h:3138
void xor_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
XOR bit interval to 1 in the bitblock.
Definition bmfunc.h:3264
BMFORCEINLINE bm::id64_t widx_to_digest_mask(unsigned w_idx) BMNOEXCEPT
Compute digest mask for word address in block.
Definition bmfunc.h:650
void bit_block_gather_scatter(unsigned *BMRESTRICT arr, const bm::word_t *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx) BMNOEXCEPT
bit index to word gather-scatter algorithm (SIMD)
Definition bmfunc.h:8471
bm::id_t bit_block_calc_count(const bm::word_t *block, const bm::word_t *block_end) BMNOEXCEPT
Bitcount for bit string.
Definition bmfunc.h:4482
int wordcmp0(T w1, T w2)
Lexicographical comparison of two words as bit strings (reference) Auxiliary implementation for testi...
Definition bmfunc.h:998
bm::word_t * bit_operation_xor(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock XOR operation.
Definition bmfunc.h:7311
bool bit_block_shift_r1_unr(bm::word_t *BMRESTRICT block, bm::word_t *BMRESTRICT empty_acc, bm::word_t co_flag) BMNOEXCEPT
Right bit-shift of bit-block by 1 bit (loop unrolled)
Definition bmfunc.h:4955
bm::set_representation best_representation(unsigned bit_count, unsigned total_possible_bitcount, unsigned gap_count, unsigned block_size) BMNOEXCEPT
Choose best representation for a bit-block.
Definition bmfunc.h:7785
bm::id_t bit_operation_or_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock OR operation and calculates bitcount of the result.
Definition bmfunc.h:6679
bool bit_block_shift_r1_and_unr(bm::word_t *BMRESTRICT block, bm::word_t co_flag, const bm::word_t *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest) BMNOEXCEPT
Right bit-shift bitblock by 1 bit (reference) + AND.
Definition bmfunc.h:5161
BMFORCEINLINE unsigned word_bitcount64(bm::id64_t x) BMNOEXCEPT
Definition bmfunc.h:230
SIZE_TYPE bit_find_rank(const bm::word_t *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
BIT block find position for the rank.
Definition bmfunc.h:7682
BMFORCEINLINE void set_bit(unsigned *dest, unsigned bitpos) BMNOEXCEPT
Set 1 bit in a block.
Definition bmfunc.h:3125
bm::id_t bit_block_calc_count_to(const bm::word_t *block, bm::word_t right) BMNOEXCEPT
Definition bmfunc.h:4767
void bit_block_erase(bm::word_t *block, unsigned bitpos, bool carry_over) BMNOEXCEPT
erase bit from position and shift the rest right with carryover
Definition bmfunc.h:5033
bool bit_block_find_interval_end(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the last 1 bit in the 111 interval of a BIT block.
Definition bmfunc.h:5373
bm::id_t bit_operation_xor_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock XOR operation test.
Definition bmfunc.h:7381
void bit_for_each_4(T w, F &func)
Templated algorithm to unpacks octet based word into list of ON bit indexes.
Definition bmfunc.h:295
unsigned bit_block_xor_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function XORs two bitblocks and and tests for any bit. Function does not analyse availability of sour...
Definition bmfunc.h:6292
bm::word_t * bit_operation_or(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Block OR operation. Makes analysis if block is 0 or FULL.
Definition bmfunc.h:6959
bm::id_t bit_operation_sub_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock test of SUB operation.
Definition bmfunc.h:6644
bool bit_block_shift_l1_unr(bm::word_t *block, bm::word_t *empty_acc, bm::word_t co_flag) BMNOEXCEPT
Left bit-shift of bit-block by 1 bit (loop unrolled)
Definition bmfunc.h:5010
bm::id64_t calc_block_digest0(const bm::word_t *const block) BMNOEXCEPT
Compute digest for 64 non-zero areas.
Definition bmfunc.h:734
bm::id64_t bit_block_and_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, bm::id64_t digest) BMNOEXCEPT
digest based bit-block AND
Definition bmfunc.h:6101
void sub_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
SUB (AND NOT) bit interval to 1 in the bitblock.
Definition bmfunc.h:3217
unsigned bit_scan_reverse(T value) BMNOEXCEPT
Definition bmutil.h:415
#define BM_INCWORD_BITCOUNT(cnt, w)
Definition bmdef.h:386
bm::word_t * bit_operation_sub(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock SUB operation.
Definition bmfunc.h:7182
void bit_for_each(T w, F &func)
Templated algorithm to unpacks word into list of ON bit indexes.
Definition bmfunc.h:365
unsigned short bitscan_wave(const bm::word_t *BMRESTRICT w_ptr, unsigned char *BMRESTRICT bits) BMNOEXCEPT
Unpacks word wave (Nx 32-bit words)
Definition bmfunc.h:8433
bool bit_find_first(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT pos) BMNOEXCEPT
BIT block find the first set bit.
Definition bmfunc.h:7552
void bit_invert(T *start) BMNOEXCEPT
Definition bmfunc.h:5253
unsigned bit_block_sub_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function SUBs two bitblocks and and tests for any bit. Function does not analyse availability of sour...
Definition bmfunc.h:6367
bool bit_find_first_diff(const bm::word_t *BMRESTRICT blk1, const bm::word_t *BMRESTRICT blk2, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two bit-blocks.
Definition bmfunc.h:4106
unsigned short bitscan_popcnt64(bm::id64_t w, B *bits) BMNOEXCEPT
Unpacks 64-bit word into list of ON bit indexes using popcnt method.
Definition bmfunc.h:518
bool bit_block_or_3way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
3 way (target | source1 | source2) bitblock OR operation. Function does not analyse availability of s...
Definition bmfunc.h:6866
int bitcmp(const T *buf1, const T *buf2, unsigned len) BMNOEXCEPT
Lexicographical comparison of BIT buffers.
Definition bmfunc.h:4081
unsigned bit_list_4(T w, B *bits) BMNOEXCEPT
Unpacks word into list of ON bit indexes (quad-bit based)
Definition bmfunc.h:456
unsigned bit_block_or_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function ORs two bitblocks and and tests for any bit. Function does not analyse availability of sourc...
Definition bmfunc.h:6444
bm::id64_t bit_block_xor(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock XOR operation. Function does not analyse availability of source and destination blocks...
Definition bmfunc.h:7240
bm::word_t * bit_operation_and(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock AND operation.
Definition bmfunc.h:6476
void bit_block_set(bm::word_t *BMRESTRICT dst, bm::word_t value) BMNOEXCEPT
Bitblock memset operation.
Definition bmfunc.h:3841
BMFORCEINLINE unsigned test_bit(const unsigned *block, unsigned bitpos) BMNOEXCEPT
Test 1 bit in a block.
Definition bmfunc.h:3152
void bit_block_rotate_left_1_unr(bm::word_t *block) BMNOEXCEPT
Unrolled cyclic rotation of bit-block left by 1 bit.
Definition bmfunc.h:4840
BMFORCEINLINE bm::id64_t digest_mask(unsigned from, unsigned to) BMNOEXCEPT
Compute digest mask for [from..to] positions.
Definition bmfunc.h:665
bool bit_block_or(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock OR operation. Function does not analyse availability of source and destination blocks.
Definition bmfunc.h:6749
bool bit_is_all_zero(const bm::word_t *BMRESTRICT start) BMNOEXCEPT
Returns "true" if all bits in the block are 0.
Definition bmfunc.h:1045
unsigned bit_find_last(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT last) BMNOEXCEPT
BIT block find the last set bit (backward search)
Definition bmfunc.h:7518
bool bit_find_first_if_1(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT first, bm::id64_t digest) BMNOEXCEPT
BIT block find the first set bit if only 1 bit is set.
Definition bmfunc.h:7625
bm::id64_t update_block_digest0(const bm::word_t *const block, bm::id64_t digest) BMNOEXCEPT
Compute digest for 64 non-zero areas based on existing digest (function revalidates zero areas)
Definition bmfunc.h:776
bool bit_block_shift_r1(bm::word_t *BMRESTRICT block, bm::word_t *BMRESTRICT empty_acc, bm::word_t co_flag) BMNOEXCEPT
Right bit-shift bitblock by 1 bit (reference)
Definition bmfunc.h:4925
unsigned short bitscan_popcnt(bm::id_t w, B *bits, unsigned short offs) BMNOEXCEPT
Unpacks word into list of ON bit indexes using popcnt method.
Definition bmfunc.h:475
bm::id_t bit_operation_xor_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock XOR operation and calculates bitcount of the result.
Definition bmfunc.h:7349
unsigned bit_list(T w, B *bits) BMNOEXCEPT
Unpacks word into list of ON bit indexes.
Definition bmfunc.h:438
void bit_block_rotate_left_1(bm::word_t *block) BMNOEXCEPT
Definition bmfunc.h:4824
unsigned bit_block_or_count(const bm::word_t *src1, const bm::word_t *src2) BMNOEXCEPT
Function ORs two bitblocks and computes the bitcount. Function does not analyse availability of sourc...
Definition bmfunc.h:6396
void bit_andnot_arr_ffmask(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock AND NOT with constant ~0 mask operation.
Definition bmfunc.h:7276
unsigned bit_block_xor_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function XORs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition bmfunc.h:6243
void bit_block_stream(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Bitblock copy/stream operation.
Definition bmfunc.h:5917
bm::word_t bit_block_insert(bm::word_t *BMRESTRICT block, unsigned bitpos, bool value) BMNOEXCEPT
insert bit into position and shift the rest right with carryover
Definition bmfunc.h:4876
bool bit_block_find_interval_start(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the first 1 bit in the 111 interval of a BIT block.
Definition bmfunc.h:5469
bm::id64_t bit_block_sub(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock SUB (AND NOT) operation. Function does not analyse availability of source and destinat...
Definition bmfunc.h:7019
int wordcmp(T a, T b)
Lexicographical comparison of two words as bit strings. Auxiliary implementation for testing and refe...
Definition bmfunc.h:1027
bm::id_t bit_operation_and_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock AND operation and calculates bitcount of the result.
Definition bmfunc.h:6540
bm::id_t bit_operation_sub_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock SUB operation and calculates bitcount of the result.
Definition bmfunc.h:6589
bm::id64_t bit_block_xor_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
2 way (target := source1 ^ source2) bitblock XOR operation.
Definition bmfunc.h:6825
bool is_bits_one(const bm::wordop_t *start) BMNOEXCEPT
Returns "true" if all bits in the block are 1.
Definition bmfunc.h:5277
bool bit_block_shift_l1(bm::word_t *block, bm::word_t *empty_acc, bm::word_t co_flag) BMNOEXCEPT
Left bit-shift bitblock by 1 bit (reference)
Definition bmfunc.h:4980
operation
Bit operations.
Definition bmconst.h:176
set_operation
Codes of set operations.
Definition bmconst.h:153
@ set_OR
Definition bmconst.h:155
@ set_SUB
Definition bmconst.h:156
@ set_COUNT
Definition bmconst.h:159
@ set_AND
Definition bmconst.h:154
@ set_XOR
Definition bmconst.h:157
@ set_END
Definition bmconst.h:168
gap_word_t * gap_operation_or(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP OR operation.
Definition bmfunc.h:5790
unsigned gap_test_unr(const T *BMRESTRICT buf, const unsigned pos) BMNOEXCEPT
Tests if bit = pos is true. Analog of bm::gap_test with SIMD unrolling.
Definition bmfunc.h:1298
BMFORCEINLINE unsigned gap_operation_any_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP XOR operation test.
Definition bmfunc.h:5749
unsigned gap_find_last(const T *BMRESTRICT buf, unsigned *BMRESTRICT last) BMNOEXCEPT
GAP block find the last set bit.
Definition bmfunc.h:1162
unsigned gap_bit_count_unr(const T *buf) BMNOEXCEPT
Calculates number of bits ON in GAP buffer. Loop unrolled version.
Definition bmfunc.h:1801
D gap_convert_to_arr(D *BMRESTRICT dest, const T *BMRESTRICT buf, unsigned dest_len, bool invert=false) BMNOEXCEPT
Convert gap block into array of ints corresponding to 1 bits.
Definition bmfunc.h:4320
void gap_and_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
ANDs GAP block to bitblock.
Definition bmfunc.h:3486
gap_word_t * gap_operation_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP XOR operation.
Definition bmfunc.h:5712
unsigned gap_set_value(unsigned val, T *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set) BMNOEXCEPT
Sets or clears bit in the GAP buffer.
Definition bmfunc.h:2661
T gap_level(const T *BMRESTRICT buf) BMNOEXCEPT
Returs GAP blocks capacity level.
Definition bmfunc.h:1144
bool gap_operation_dry_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, unsigned &dsize, unsigned limit) BMNOEXCEPT
Definition bmfunc.h:5725
bm::id_t gap_bitset_or_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block OR masked by GAP block.
Definition bmfunc.h:3822
void gap_split(const T *buf, T *arr0, T *arr1, T &arr0_cnt, T &arr1_cnt) BMNOEXCEPT
Definition bmfunc.h:1713
bool gap_find_interval_end(const T *const BMRESTRICT buf, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the last 1 bit in the 111 interval of a GAP block.
Definition bmfunc.h:1991
bm::id_t gap_bitset_xor_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block XOR masked by GAP block.
Definition bmfunc.h:3714
void gap_xor_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
XOR GAP block to bitblock.
Definition bmfunc.h:3408
bool gap_shift_r1(T *BMRESTRICT buf, unsigned co_flag, unsigned *BMRESTRICT new_len) BMNOEXCEPT
Right shift GAP block by 1 bit.
Definition bmfunc.h:2897
bm::id_t gap_bitset_and_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT pcurr) BMNOEXCEPT
Compute bitcount of bit block AND masked by GAP block.
Definition bmfunc.h:3583
int gapcmp(const T *buf1, const T *buf2) BMNOEXCEPT
Lexicographical comparison of GAP buffers.
Definition bmfunc.h:2256
unsigned gap_bit_count_to(const T *const buf, T right, bool is_corrected=false) BMNOEXCEPT
Counts 1 bits in GAP buffer in the closed [0, right] range.
Definition bmfunc.h:2101
BMFORCEINLINE unsigned gap_operation_any_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP SUB operation test.
Definition bmfunc.h:5860
unsigned gap_find_first(const T *BMRESTRICT buf, unsigned *BMRESTRICT first) BMNOEXCEPT
GAP block find the first set bit.
Definition bmfunc.h:1193
bm::id_t gap_bitset_sub_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block SUB masked by GAP block.
Definition bmfunc.h:3641
void for_each_gap_dbit(const T *buf, F &func)
Iterate gap block as delta-bits with a functor.
Definition bmfunc.h:4265
BMFORCEINLINE unsigned gap_operation_any_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP AND operation test.
Definition bmfunc.h:5670
BMFORCEINLINE unsigned gap_count_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount XOR operation test.
Definition bmfunc.h:5765
bool gap_find_interval_start(const T *const BMRESTRICT buf, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the first 1 bit in the 111 interval of a GAP block.
Definition bmfunc.h:2016
unsigned gap_free_elements(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returns number of free elements in GAP block array. Difference between GAP block capacity on this lev...
Definition bmfunc.h:4063
unsigned gap_test(const T *BMRESTRICT buf, unsigned pos) BMNOEXCEPT
Tests if bit = pos is true.
Definition bmfunc.h:1255
bm::id_t gap_bitset_sub_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block SUB masked by GAP block.
Definition bmfunc.h:3676
void gap_invert(T *buf) BMNOEXCEPT
Inverts all bits in the GAP buffer.
Definition bmfunc.h:4001
unsigned * gap_convert_to_bitset_smart(unsigned *BMRESTRICT dest, const T *BMRESTRICT buf, id_t set_max) BMNOEXCEPT
Smart GAP block to bitblock conversion.
Definition bmfunc.h:3881
unsigned gap_set_array(T *buf, const T *arr, unsigned len) BMNOEXCEPT
Convert array to GAP buffer.
Definition bmfunc.h:3005
void set_gap_level(T *buf, int level) BMNOEXCEPT
Sets GAP block capacity level.
Definition bmfunc.h:4020
void gap_sub_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
SUB (AND NOT) GAP block to bitblock.
Definition bmfunc.h:3320
unsigned gap_block_find(const T *BMRESTRICT buf, unsigned nbit, bm::id_t *BMRESTRICT prev) BMNOEXCEPT
Searches for the next 1 bit in the GAP block.
Definition bmfunc.h:3098
void gap_init_range_block(T *buf, T from, T to, T value) BMNOEXCEPT
Init gap block so it has block in it (can be whole block)
Definition bmfunc.h:3951
bm::id_t gap_bitset_xor_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block XOR masked by GAP block.
Definition bmfunc.h:3752
bm::id_t gap_bitset_or_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block OR masked by GAP block.
Definition bmfunc.h:3790
unsigned gap_count_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount AND operation test.
Definition bmfunc.h:5687
unsigned gap_control_sum(const T *buf) BMNOEXCEPT
Calculates sum of all words in GAP block. (For debugging purposes)
Definition bmfunc.h:3902
BMFORCEINLINE bool gap_is_all_zero(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Checks if GAP block is all-zero.
Definition bmfunc.h:1072
SIZE_TYPE gap_find_rank(const T *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
GAP block find position for the rank.
Definition bmfunc.h:2049
BMFORCEINLINE bool gap_is_all_one(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Checks if GAP block is all-one.
Definition bmfunc.h:1085
unsigned gap_bit_count(const T *buf, unsigned dsize=0) BMNOEXCEPT
Calculates number of bits ON in GAP buffer.
Definition bmfunc.h:1773
bool gap_shift_l1(T *BMRESTRICT buf, unsigned co_flag, unsigned *BMRESTRICT new_len) BMNOEXCEPT
Left shift GAP block by 1 bit.
Definition bmfunc.h:2946
void gap_add_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr, unsigned len) BMNOEXCEPT
Adds(OR) GAP block to bitblock.
Definition bmfunc.h:3436
unsigned gap_overhead(const T *length, const T *length_end, const T *glevel_len) BMNOEXCEPT
Calculates memory overhead for number of gap blocks sharing the same memory allocation table (level l...
Definition bmfunc.h:7905
bool gap_any_range(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if any bits are 1 in GAP buffer in the [left, right] range.
Definition bmfunc.h:1933
bool gap_is_all_one_range(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if all bits are 1 in GAP buffer in the [left, right] range.
Definition bmfunc.h:1910
BMFORCEINLINE gap_word_t * gap_operation_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP AND operation.
Definition bmfunc.h:5646
bool gap_find_first_diff(const T *BMRESTRICT buf1, const T *BMRESTRICT buf2, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two GAP-blocks.
Definition bmfunc.h:2314
void gap_convert_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT buf) BMNOEXCEPT
GAP block to bitblock conversion.
Definition bmfunc.h:3859
void gap_set_all(T *buf, unsigned set_max, unsigned value) BMNOEXCEPT
Sets all bits to 0 or 1 (GAP)
Definition bmfunc.h:3933
BMFORCEINLINE unsigned gap_count_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount SUB (AND NOT) operation test.
Definition bmfunc.h:5877
unsigned bit_array_compute_gaps(const T *arr, unsigned len) BMNOEXCEPT
Compute number of GAPs in bit-array.
Definition bmfunc.h:3066
unsigned gap_bit_count_range(const T *const buf, unsigned left, unsigned right) BMNOEXCEPT
Counts 1 bits in GAP buffer in the closed [left, right] range.
Definition bmfunc.h:1867
unsigned gap_add_value(T *buf, unsigned pos) BMNOEXCEPT
Add new value to the end of GAP buffer.
Definition bmfunc.h:2827
unsigned bit_block_to_gap(gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len) BMNOEXCEPT
Converts bit block to GAP.
Definition bmfunc.h:4163
int gap_calc_level(unsigned len, const T *glevel_len) BMNOEXCEPT
Calculates GAP block capacity level.
Definition bmfunc.h:4042
bool gap_is_interval(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if any bits are 1 in GAP buffer in the [left, right] range and flanked with 0s.
Definition bmfunc.h:1962
gap_word_t * gap_operation_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP SUB (AND NOT) operation.
Definition bmfunc.h:5835
bm::id_t gap_bitset_and_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT pcurr) BMNOEXCEPT
Bitcount test of bit block AND masked by GAP block.
Definition bmfunc.h:3611
bool improve_gap_levels(const T *length, const T *length_end, T *glevel_len) BMNOEXCEPT
Finds optimal gap blocks lengths.
Definition bmfunc.h:7932
unsigned gap_capacity(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returs GAP block capacity.
Definition bmfunc.h:1114
unsigned gap_limit(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returs GAP block capacity limit.
Definition bmfunc.h:1130
BMFORCEINLINE unsigned gap_count_or(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount OR operation test.
Definition bmfunc.h:5810
unsigned gap_buff_any_op(const T *BMRESTRICT vect1, unsigned vect1_mask, const T *BMRESTRICT vect2, unsigned vect2_mask, F f) BMNOEXCEPT2
Abstract distance test operation for GAP buffers. Receives functor F as a template argument.
Definition bmfunc.h:2507
BMFORCEINLINE bm::gap_word_t gap_length(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Returs GAP block length.
Definition bmfunc.h:1098
unsigned gap_buff_count_op(const T *vect1, const T *vect2, F f) BMNOEXCEPT2
Abstract distance(similarity) operation for GAP buffers. Receives functor F as a template argument.
Definition bmfunc.h:2576
Definition bm.h:77
const unsigned set_array_mask
Definition bmconst.h:96
bool block_any(const bm::word_t *const BMRESTRICT block) BMNOEXCEPT
Returns "true" if one bit is set in the block Function check for block varieties.
Definition bmfunc.h:5584
bm::id_t(* bit_operation_count_func_type)(const bm::word_t *BMRESTRICT, const bm::word_t *BMRESTRICT)
Definition bmfunc.h:8345
unsigned lower_bound_u32(const unsigned *arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
Hybrid, binary-linear lower bound search in unsigned array.
Definition bmfunc.h:8745
const id64_t all_bits_mask
Definition bmconst.h:128
const unsigned set_block_digest_wave_size
Definition bmconst.h:66
void for_each_nzblock(T ***root, unsigned size1, F &f)
Definition bmfunc.h:1382
void bit_block_change_bc(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT gc, unsigned *BMRESTRICT bc) BMNOEXCEPT
Definition bmfunc.h:4581
unsigned bitcount64_4way(bm::id64_t x, bm::id64_t y, bm::id64_t u, bm::id64_t v) BMNOEXCEPT
Definition bmfunc.h:254
void gap_buff_op(T *BMRESTRICT dest, const T *BMRESTRICT vect1, unsigned vect1_mask, const T *BMRESTRICT vect2, unsigned vect2_mask, F &f, unsigned &dlen) BMNOEXCEPT2
Abstract operation for GAP buffers. Receives functor F as a template argument.
Definition bmfunc.h:2355
void xor_swap(W &x, W &y) BMNOEXCEPT
XOR swap two scalar variables.
Definition bmfunc.h:909
bool gap_buff_dry_op(const T *BMRESTRICT vect1, const T *BMRESTRICT vect2, F &f, unsigned &dlen, unsigned limit) BMNOEXCEPT2
Abstract operation for GAP buffers (predicts legth) Receives functor F as a template argument.
Definition bmfunc.h:2434
bool block_is_interval(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if all bits are 1 in the block [left, right] and border bits are 0.
Definition bmfunc.h:5324
bool check_block_zero(const bm::word_t *blk, bool deep_scan) BMNOEXCEPT
Checks all conditions and returns true if block consists of only 0 bits.
Definition bmfunc.h:7858
bool block_is_all_one_range(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if all bits are 1 in the block [left, right] Function check for block varieties.
Definition bmfunc.h:5303
unsigned int word_t
Definition bmconst.h:38
const bm::gap_word_t * sse2_gap_sum_arr(const bm::gap_word_t *BMRESTRICT pbuf, unsigned sse_vect_waves, unsigned *sum)
Gap block population count (array sum) utility.
Definition bmsse_util.h:891
const unsigned set_block_mask
Definition bmconst.h:56
T * gap_2_dgap(const T *BMRESTRICT gap_buf, T *BMRESTRICT dgap_buf, bool copy_head=true) BMNOEXCEPT
Convert GAP buffer into D-GAP buffer.
Definition bmfunc.h:2188
bm::id_t bit_block_any_range(const bm::word_t *block, bm::word_t left, bm::word_t right) BMNOEXCEPT
BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP and functor.
Definition bmfunc.h:5603
bm::id64_t sum_arr(const T *first, const T *last) BMNOEXCEPT
Definition bmfunc.h:1695
unsigned short bitscan(V w, B *bits) BMNOEXCEPT
Definition bmfunc.h:554
bool check_block_one(const bm::word_t *blk, bool deep_scan) BMNOEXCEPT
Checks if block has only 1 bits.
Definition bmfunc.h:7882
unsigned gap_bfind(const T *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set) BMNOEXCEPT
Definition bmfunc.h:1220
BMFORCEINLINE RTYPE get_block_start(unsigned i, unsigned j) BMNOEXCEPT
Compute bit address of the first bit in a block.
Definition bmfunc.h:168
unsigned bit_to_gap(gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len) BMNOEXCEPT
Convert bit block to GAP representation.
Definition bmfunc.h:4248
const unsigned set_sub_array_size
Definition bmconst.h:94
void for_each_dgap(const T *gap_buf, Func &func)
Definition bmfunc.h:2147
unsigned bit_block_change32(const bm::word_t *block, unsigned size) BMNOEXCEPT
Definition bmfunc.h:4533
unsigned lower_bound_u64(const unsigned long long *arr, unsigned long long target, unsigned from, unsigned to) BMNOEXCEPT
Hybrid, binary-linear lower bound search in unsigned LONG array.
Definition bmfunc.h:8780
unsigned word_select64_bitscan(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition bmfunc.h:597
set_representation
set representation variants
Definition bmconst.h:202
@ set_bitset
Simple bitset.
Definition bmconst.h:203
@ set_gap
GAP-RLE compression.
Definition bmconst.h:204
@ set_array1
array of set 1 values
Definition bmconst.h:205
@ set_array0
array of 0 values
Definition bmconst.h:206
bool block_ptr_array_range(bm::word_t **arr, unsigned &left, unsigned &right) BMNOEXCEPT
array range detector
Definition bmfunc.h:8662
unsigned word_select64(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition bmfunc.h:624
unsigned block_find_interval_start(const bm::word_t *BMRESTRICT block, unsigned nbit_from, unsigned *BMRESTRICT found_nbit) BMNOEXCEPT
Find start of the current 111 interval.
Definition bmfunc.h:5528
gap_word_t *(* gap_operation_func_type)(const gap_word_t *BMRESTRICT, const gap_word_t *BMRESTRICT, gap_word_t *BMRESTRICT, unsigned &)
Definition bmfunc.h:8339
bool block_find_first_diff(const bm::word_t *BMRESTRICT blk, const bm::word_t *BMRESTRICT arg_blk, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two blocks (GAP or bit)
Definition bmfunc.h:8030
const unsigned gap_levels
Definition bmconst.h:84
void for_each_nzblock2(T ***root, unsigned size1, F &f)
Definition bmfunc.h:1491
F bmfor_each(T first, T last, F f)
Definition bmfunc.h:1682
bool find_not_null_ptr(bm::word_t ***arr, N start, N size, N *pos) BMNOEXCEPT
Definition bmfunc.h:923
void bit_recomb(It1 &it1, It2 &it2, BinaryOp &op, Encoder &enc, unsigned block_size=bm::set_block_size) BMNOEXCEPT
Definition bmfunc.h:8201
unsigned bit_scan_reverse32(unsigned value) BMNOEXCEPT
Definition bmutil.h:301
bm::operation setop2op(bm::set_operation op) BMNOEXCEPT
Convert set operation to operation.
Definition bmfunc.h:826
const unsigned set_word_shift
Definition bmconst.h:71
bm::id64_t idx_arr_block_lookup_u64(const bm::id64_t *idx, bm::id64_t size, bm::id64_t nb, bm::id64_t start) BMNOEXCEPT
block boundaries look ahead U32
Definition bmfunc.h:8546
const unsigned set_sub_total_bits
Definition bmconst.h:99
const unsigned set_block_digest_pos_shift
Definition bmconst.h:67
unsigned count_leading_zeros_u64(bm::id64_t w) BMNOEXCEPT
64-bit bit-scan reverse
Definition bmutil.h:353
const unsigned set_block_size
Definition bmconst.h:54
unsigned long long int id64_t
Definition bmconst.h:34
bool block_any_range(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if one bit is set in the block [left, right] Function check for block varieties.
Definition bmfunc.h:5563
unsigned lower_bound_linear_u64(const unsigned long long *arr, unsigned long long target, unsigned from, unsigned to) BMNOEXCEPT
Linear lower bound search in unsigned LONG array.
Definition bmfunc.h:8720
unsigned int id_t
Definition bmconst.h:37
const unsigned gap_max_buff_len
Definition bmconst.h:79
int parallel_popcnt_32(unsigned int n) BMNOEXCEPT
Definition bmfunc.h:215
BMFORCEINLINE RTYPE get_super_block_start(unsigned i) BMNOEXCEPT
Compute bit address of the first bit in a superblock.
Definition bmfunc.h:158
void set_block_bits_u32(bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop) BMNOEXCEPT
set bits in a bit-block using global index
Definition bmfunc.h:8635
unsigned idx_arr_block_lookup_u32(const unsigned *idx, unsigned size, unsigned nb, unsigned start) BMNOEXCEPT
block boundaries look ahead U32
Definition bmfunc.h:8572
BMFORCEINLINE unsigned long long bmi_bslr_u64(unsigned long long w) BMNOEXCEPT
Definition bmutil.h:332
bm::id64_t ptrp_test(ptr_payload_t ptr, bm::gap_word_t v) BMNOEXCEPT
Definition bmfunc.h:8857
const unsigned short set_bitscan_wave_size
Size of bit decode wave in words.
Definition bmfunc.h:8421
const unsigned set_array_shift
Definition bmconst.h:95
void dgap_2_gap(const T *BMRESTRICT dgap_buf, T *BMRESTRICT gap_buf, T gap_header=0) BMNOEXCEPT
Convert D-GAP buffer into GAP buffer.
Definition bmfunc.h:2214
bm::id_t block_to_global_index(unsigned i, unsigned j, unsigned block_idx) BMNOEXCEPT
calculate bvector<> global bit-index from block-local coords
Definition bmfunc.h:8828
unsigned short gap_word_t
Definition bmconst.h:77
unsigned count_trailing_zeros_u64(bm::id64_t w) BMNOEXCEPT
64-bit bit-scan fwd
Definition bmutil.h:381
void(* gap_operation_to_bitset_func_type)(unsigned *, const gap_word_t *)
Definition bmfunc.h:8335
const unsigned gap_max_bits
Definition bmconst.h:80
void get_block_coord(BI_TYPE nb, unsigned &i, unsigned &j) BMNOEXCEPT
Recalc linear bvector block index into 2D matrix coordinates.
Definition bmfunc.h:147
void sse4_bit_block_gather_scatter(unsigned *BMRESTRICT arr, const unsigned *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx)
Definition bmsse4.h:1343
unsigned word_select64_linear(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition bmfunc.h:574
void for_each_block(T ***root, unsigned size1, F &f, BLOCK_IDX start)
Definition bmfunc.h:1647
unsigned lower_bound_linear_u32(const unsigned *arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
Linear lower bound search in unsigned array.
Definition bmfunc.h:8697
const unsigned set_block_shift
Definition bmconst.h:55
void for_each_nzblock_range(T ***root, N top_size, N nb_from, N nb_to, F &f) BMNOEXCEPT
Definition bmfunc.h:1325
const unsigned set_word_mask
Definition bmconst.h:72
BMFORCEINLINE unsigned long long bmi_blsi_u64(unsigned long long w)
Definition bmutil.h:342
const unsigned bits_in_block
Definition bmconst.h:113
id64_t wordop_t
Definition bmconst.h:127
unsigned block_find_interval_end(const bm::word_t *BMRESTRICT block, unsigned nbit_from, unsigned *BMRESTRICT found_nbit) BMNOEXCEPT
Find end of the current 111 interval.
Definition bmfunc.h:5428
BMFORCEINLINE unsigned or_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP or functor.
Definition bmfunc.h:5617
BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP xor functor.
Definition bmfunc.h:5610
unsigned bit_scan_forward32(unsigned value) BMNOEXCEPT
Definition bmutil.h:316
BMFORCEINLINE unsigned sub_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP or functor.
Definition bmfunc.h:5623
void set_block_bits_u64(bm::word_t *BMRESTRICT block, const bm::id64_t *BMRESTRICT idx, bm::id64_t start, bm::id64_t stop) BMNOEXCEPT
set bits in a bit-block using global index
Definition bmfunc.h:8605
bool for_each_nzblock_if(T ***root, BI size1, F &f) BMNOEXCEPT
Definition bmfunc.h:1613
SIZE_TYPE block_find_rank(const bm::word_t *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
Find rank in block (GAP or BIT)
Definition bmfunc.h:7761
bool is_const_set_operation(set_operation op) BMNOEXCEPT
Returns true if set operation is constant (bitcount)
Definition bmfunc.h:817
bm::word_t * _p_fullp
Definition bmfunc.h:848
bm::word_t BM_VECT_ALIGN _p[bm::set_block_size] BM_VECT_ALIGN_ATTR
Definition bmfunc.h:847
bm::word_t BM_VECT_ALIGN *_s[bm::set_sub_array_size] BM_VECT_ALIGN_ATTR
Definition bmfunc.h:846
Structure carries pointer on bit block with all bits 1.
Definition bmfunc.h:843
static BMFORCEINLINE bool is_valid_block_addr(const bm::word_t *bp) BMNOEXCEPT
Definition bmfunc.h:898
static all_set_block _block
Definition bmfunc.h:901
static BMFORCEINLINE bool is_full_block(const bm::word_t *bp) BMNOEXCEPT
Definition bmfunc.h:894
static bm::id64_t block_type(const bm::word_t *bp) BMNOEXCEPT
Definition bmfunc.h:873
Bit AND functor.
Definition bmfunc.h:8217
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8218
Bit ASSIGN functor.
Definition bmfunc.h:8241
W operator()(W, W w2) BMNOEXCEPT
Definition bmfunc.h:8242
Bit COUNT AND functor.
Definition bmfunc.h:8258
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8259
Bit COUNT A functor.
Definition bmfunc.h:8314
W operator()(W w1, W) BMNOEXCEPT
Definition bmfunc.h:8315
Bit COUNT B functor.
Definition bmfunc.h:8325
W operator()(W, W w2) BMNOEXCEPT
Definition bmfunc.h:8326
Bit COUNT OR functor.
Definition bmfunc.h:8280
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8281
Bit COUNT SUB AB functor.
Definition bmfunc.h:8292
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8293
Bit SUB BA functor.
Definition bmfunc.h:8303
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8304
Bit COUNT XOR functor.
Definition bmfunc.h:8269
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8270
Bit COUNT functor.
Definition bmfunc.h:8247
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8248
Bit OR functor.
Definition bmfunc.h:8223
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8224
Bit SUB functor.
Definition bmfunc.h:8229
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8230
Bit XOR functor.
Definition bmfunc.h:8235
W operator()(W w1, W w2) BMNOEXCEPT
Definition bmfunc.h:8236
bit-block array wrapped into union for correct interpretation of 32-bit vs 64-bit access vs SIMD
Definition bmutil.h:57
Structure to aid in counting bits table contains count of bits in 0-255 diapason of numbers.
Definition bmconst.h:288
bit-decode cache structure
Definition bmfunc.h:133
bm::id64_t cvalue
Definition bmfunc.h:136
unsigned short bits[65]
Definition bmfunc.h:134
Structure keeps all-left/right ON bits masks.
Definition bmconst.h:343
Structure with statistical information about memory allocation footprint, serialization projection,...
Definition bmfunc.h:55
size_t gap_cap_overhead
gap memory overhead between length and capacity
Definition bmfunc.h:62
size_t ptr_sub_blocks
Number of sub-blocks.
Definition bmfunc.h:58
unsigned long long gaps_by_level[bm::gap_levels]
number of GAP blocks at each level
Definition bmfunc.h:64
size_t gap_blocks
Number of GAP blocks.
Definition bmfunc.h:57
size_t bit_blocks
Number of bit blocks.
Definition bmfunc.h:56
size_t bv_count
Number of bit-vectors.
Definition bmfunc.h:59
void add_gap_block(unsigned capacity, unsigned length) BMNOEXCEPT
count gap block
Definition bmfunc.h:76
gap_word_t gap_levels[bm::gap_levels]
GAP block lengths in the bvect.
Definition bmfunc.h:63
size_t max_serialize_mem
estimated maximum memory for serialization
Definition bmfunc.h:60
void reset() BMNOEXCEPT
Reset statisctics.
Definition bmfunc.h:96
void add_bit_block() BMNOEXCEPT
cound bit block
Definition bmfunc.h:67
size_t memory_used
memory usage for all blocks and service tables
Definition bmfunc.h:61
void add(const bv_statistics &st) BMNOEXCEPT
Sum data from another sttructure.
Definition bmfunc.h:105
ad-hoc conditional expressions
Definition bmutil.h:111
d-Gap copy functor
Definition bmfunc.h:2167
d_copy_func(T *dg_buf)
Definition bmfunc.h:2168
void operator()(T dgap)
Definition bmfunc.h:2169
static bit_operation_count_func_type bit_operation_count(unsigned i)
Definition bmfunc.h:8372
static gap_operation_to_bitset_func_type gap_op_to_bit(unsigned i)
Definition bmfunc.h:8360
static bit_operation_count_func_type bit_op_count_table_[bm::set_END]
Definition bmfunc.h:8357
static gap_operation_func_type gap_operation(unsigned i)
Definition bmfunc.h:8366
static gap_operation_func_type gapop_table_[bm::set_END]
Definition bmfunc.h:8355
static gap_operation_to_bitset_func_type gap2bit_table_[bm::set_END]
Definition bmfunc.h:8353
Pair type.
Definition bmfunc.h:122
Second second
Definition bmfunc.h:124
pair(First f, Second s)
Definition bmfunc.h:126
First first
Definition bmfunc.h:123
helper union to interpret pointer as integers
Definition bmfunc.h:8846
bm::word_t * blk
Definition bmfunc.h:8847
bm::id64_t i64
Definition bmfunc.h:8848
unsigned short i16[4]
Definition bmfunc.h:8849