1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static com.google.common.base.Preconditions.checkArgument;
21 import static com.google.common.base.Preconditions.checkNotNull;
22 import static com.google.common.base.Preconditions.checkPositionIndex;
23
24 import java.io.DataInput;
25 import java.io.DataOutput;
26 import java.io.IOException;
27 import java.lang.reflect.Field;
28 import java.math.BigDecimal;
29 import java.math.BigInteger;
30 import java.nio.ByteBuffer;
31 import java.nio.ByteOrder;
32 import java.nio.charset.Charset;
33 import java.security.AccessController;
34 import java.security.PrivilegedAction;
35 import java.security.SecureRandom;
36 import java.util.Arrays;
37 import java.util.Collection;
38 import java.util.Comparator;
39 import java.util.Iterator;
40 import java.util.List;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.hadoop.classification.InterfaceAudience;
45 import org.apache.hadoop.classification.InterfaceStability;
46 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
47 import org.apache.hadoop.io.RawComparator;
48 import org.apache.hadoop.io.WritableComparator;
49 import org.apache.hadoop.io.WritableUtils;
50
51 import sun.misc.Unsafe;
52
53 import com.google.common.annotations.VisibleForTesting;
54 import com.google.common.collect.Lists;
55
56
57
58
59
60
61 @InterfaceAudience.Public
62 @InterfaceStability.Stable
63 public class Bytes {
64
65
66 private static final String UTF8_ENCODING = "UTF-8";
67
68
69
70 private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
71
72
73 private static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
74
75 private static final Log LOG = LogFactory.getLog(Bytes.class);
76
77
78
79
80 public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
81
82
83
84
85 public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
86
87
88
89
90 public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
91
92
93
94
95 public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
96
97
98
99
100 public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
101
102
103
104
105 public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
106
107
108
109
110 public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
111
112
113
114
115 public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
116
117
118
119
120
121
122
123
124 public static final int ESTIMATED_HEAP_TAX = 16;
125
126
127
128
129
130
131
132
133 final public static int len(byte[] b) {
134 return b == null ? 0 : b.length;
135 }
136
137
138
139
140 @InterfaceAudience.Public
141 @InterfaceStability.Stable
142 public static class ByteArrayComparator implements RawComparator<byte []> {
143
144
145
146 public ByteArrayComparator() {
147 super();
148 }
149 @Override
150 public int compare(byte [] left, byte [] right) {
151 return compareTo(left, right);
152 }
153 @Override
154 public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
155 return LexicographicalComparerHolder.BEST_COMPARER.
156 compareTo(b1, s1, l1, b2, s2, l2);
157 }
158 }
159
160
161
162
163
164
165
166
167
168 @InterfaceAudience.Public
169 @InterfaceStability.Stable
170 public static class RowEndKeyComparator extends ByteArrayComparator {
171 @Override
172 public int compare(byte[] left, byte[] right) {
173 return compare(left, 0, left.length, right, 0, right.length);
174 }
175 @Override
176 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
177 if (b1 == b2 && s1 == s2 && l1 == l2) {
178 return 0;
179 }
180 if (l1 == 0) {
181 return l2;
182 }
183 if (l2 == 0) {
184 return -1;
185 }
186 return super.compare(b1, s1, l1, b2, s2, l2);
187 }
188 }
189
190
191
192
193 public final static Comparator<byte []> BYTES_COMPARATOR = new ByteArrayComparator();
194
195
196
197
198 public final static RawComparator<byte []> BYTES_RAWCOMPARATOR = new ByteArrayComparator();
199
200
201
202
203
204
205
206 public static byte [] readByteArray(final DataInput in)
207 throws IOException {
208 int len = WritableUtils.readVInt(in);
209 if (len < 0) {
210 throw new NegativeArraySizeException(Integer.toString(len));
211 }
212 byte [] result = new byte[len];
213 in.readFully(result, 0, len);
214 return result;
215 }
216
217
218
219
220
221
222
223 public static byte [] readByteArrayThrowsRuntime(final DataInput in) {
224 try {
225 return readByteArray(in);
226 } catch (Exception e) {
227 throw new RuntimeException(e);
228 }
229 }
230
231
232
233
234
235
236
237 public static void writeByteArray(final DataOutput out, final byte [] b)
238 throws IOException {
239 if(b == null) {
240 WritableUtils.writeVInt(out, 0);
241 } else {
242 writeByteArray(out, b, 0, b.length);
243 }
244 }
245
246
247
248
249
250
251
252
253
254 public static void writeByteArray(final DataOutput out, final byte [] b,
255 final int offset, final int length)
256 throws IOException {
257 WritableUtils.writeVInt(out, length);
258 out.write(b, offset, length);
259 }
260
261
262
263
264
265
266
267
268
269
270 public static int writeByteArray(final byte [] tgt, final int tgtOffset,
271 final byte [] src, final int srcOffset, final int srcLength) {
272 byte [] vint = vintToBytes(srcLength);
273 System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
274 int offset = tgtOffset + vint.length;
275 System.arraycopy(src, srcOffset, tgt, offset, srcLength);
276 return offset + srcLength;
277 }
278
279
280
281
282
283
284
285
286
287
288 public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
289 int srcOffset, int srcLength) {
290 System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
291 return tgtOffset + srcLength;
292 }
293
294
295
296
297
298
299
300
301 public static int putByte(byte[] bytes, int offset, byte b) {
302 bytes[offset] = b;
303 return offset + 1;
304 }
305
306
307
308
309
310
311
312
313 public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf) {
314 int len = buf.remaining();
315 buf.get(bytes, offset, len);
316 return offset + len;
317 }
318
319
320
321
322
323
324
325
326
327
328
329 public static byte[] toBytes(ByteBuffer buf) {
330 ByteBuffer dup = buf.duplicate();
331 dup.position(0);
332 return readBytes(dup);
333 }
334
335 private static byte[] readBytes(ByteBuffer buf) {
336 byte [] result = new byte[buf.remaining()];
337 buf.get(result);
338 return result;
339 }
340
341
342
343
344
345 public static String toString(final byte [] b) {
346 if (b == null) {
347 return null;
348 }
349 return toString(b, 0, b.length);
350 }
351
352
353
354
355
356
357
358 public static String toString(final byte [] b1,
359 String sep,
360 final byte [] b2) {
361 return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
362 }
363
364
365
366
367
368
369
370
371
372
373 public static String toString(final byte [] b, int off, int len) {
374 if (b == null) {
375 return null;
376 }
377 if (len == 0) {
378 return "";
379 }
380 return new String(b, off, len, UTF8_CHARSET);
381 }
382
383
384
385
386
387
388
389
390 public static String toStringBinary(final byte [] b) {
391 if (b == null)
392 return "null";
393 return toStringBinary(b, 0, b.length);
394 }
395
396
397
398
399
400
401
402
403
404
405
406
407 public static String toStringBinary(ByteBuffer buf) {
408 if (buf == null)
409 return "null";
410 if (buf.hasArray()) {
411 return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
412 }
413 return toStringBinary(toBytes(buf));
414 }
415
416
417
418
419
420
421
422
423
424
425
426 public static String toStringBinary(final byte [] b, int off, int len) {
427 StringBuilder result = new StringBuilder();
428
429 if (off >= b.length) return result.toString();
430 if (off + len > b.length) len = b.length - off;
431 for (int i = off; i < off + len ; ++i ) {
432 int ch = b[i] & 0xFF;
433 if ( (ch >= '0' && ch <= '9')
434 || (ch >= 'A' && ch <= 'Z')
435 || (ch >= 'a' && ch <= 'z')
436 || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
437 result.append((char)ch);
438 } else {
439 result.append(String.format("\\x%02X", ch));
440 }
441 }
442 return result.toString();
443 }
444
445 private static boolean isHexDigit(char c) {
446 return
447 (c >= 'A' && c <= 'F') ||
448 (c >= '0' && c <= '9');
449 }
450
451
452
453
454
455
456
457 public static byte toBinaryFromHex(byte ch) {
458 if ( ch >= 'A' && ch <= 'F' )
459 return (byte) ((byte)10 + (byte) (ch - 'A'));
460
461 return (byte) (ch - '0');
462 }
463
464 public static byte [] toBytesBinary(String in) {
465
466 byte [] b = new byte[in.length()];
467 int size = 0;
468 for (int i = 0; i < in.length(); ++i) {
469 char ch = in.charAt(i);
470 if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
471
472 char hd1 = in.charAt(i+2);
473 char hd2 = in.charAt(i+3);
474
475
476 if (!isHexDigit(hd1) ||
477 !isHexDigit(hd2)) {
478
479 continue;
480 }
481
482 byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2));
483
484 b[size++] = d;
485 i += 3;
486 } else {
487 b[size++] = (byte) ch;
488 }
489 }
490
491 byte [] b2 = new byte[size];
492 System.arraycopy(b, 0, b2, 0, size);
493 return b2;
494 }
495
496
497
498
499
500
501 public static byte[] toBytes(String s) {
502 return s.getBytes(UTF8_CHARSET);
503 }
504
505
506
507
508
509
510
511
512 public static byte [] toBytes(final boolean b) {
513 return new byte[] { b ? (byte) -1 : (byte) 0 };
514 }
515
516
517
518
519
520
521 public static boolean toBoolean(final byte [] b) {
522 if (b.length != 1) {
523 throw new IllegalArgumentException("Array has wrong size: " + b.length);
524 }
525 return b[0] != (byte) 0;
526 }
527
528
529
530
531
532
533
534 public static byte[] toBytes(long val) {
535 byte [] b = new byte[8];
536 for (int i = 7; i > 0; i--) {
537 b[i] = (byte) val;
538 val >>>= 8;
539 }
540 b[0] = (byte) val;
541 return b;
542 }
543
544
545
546
547
548
549
550 public static long toLong(byte[] bytes) {
551 return toLong(bytes, 0, SIZEOF_LONG);
552 }
553
554
555
556
557
558
559
560
561
562 public static long toLong(byte[] bytes, int offset) {
563 return toLong(bytes, offset, SIZEOF_LONG);
564 }
565
566
567
568
569
570
571
572
573
574
575
576 public static long toLong(byte[] bytes, int offset, final int length) {
577 if (length != SIZEOF_LONG || offset + length > bytes.length) {
578 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
579 }
580 long l = 0;
581 for(int i = offset; i < offset + length; i++) {
582 l <<= 8;
583 l ^= bytes[i] & 0xFF;
584 }
585 return l;
586 }
587
588 private static IllegalArgumentException
589 explainWrongLengthOrOffset(final byte[] bytes,
590 final int offset,
591 final int length,
592 final int expectedLength) {
593 String reason;
594 if (length != expectedLength) {
595 reason = "Wrong length: " + length + ", expected " + expectedLength;
596 } else {
597 reason = "offset (" + offset + ") + length (" + length + ") exceed the"
598 + " capacity of the array: " + bytes.length;
599 }
600 return new IllegalArgumentException(reason);
601 }
602
603
604
605
606
607
608
609
610
611
612 public static int putLong(byte[] bytes, int offset, long val) {
613 if (bytes.length - offset < SIZEOF_LONG) {
614 throw new IllegalArgumentException("Not enough room to put a long at"
615 + " offset " + offset + " in a " + bytes.length + " byte array");
616 }
617 for(int i = offset + 7; i > offset; i--) {
618 bytes[i] = (byte) val;
619 val >>>= 8;
620 }
621 bytes[offset] = (byte) val;
622 return offset + SIZEOF_LONG;
623 }
624
625
626
627
628
629
630 public static float toFloat(byte [] bytes) {
631 return toFloat(bytes, 0);
632 }
633
634
635
636
637
638
639
640 public static float toFloat(byte [] bytes, int offset) {
641 return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
642 }
643
644
645
646
647
648
649
650 public static int putFloat(byte [] bytes, int offset, float f) {
651 return putInt(bytes, offset, Float.floatToRawIntBits(f));
652 }
653
654
655
656
657
658 public static byte [] toBytes(final float f) {
659
660 return Bytes.toBytes(Float.floatToRawIntBits(f));
661 }
662
663
664
665
666
667 public static double toDouble(final byte [] bytes) {
668 return toDouble(bytes, 0);
669 }
670
671
672
673
674
675
676 public static double toDouble(final byte [] bytes, final int offset) {
677 return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
678 }
679
680
681
682
683
684
685
686 public static int putDouble(byte [] bytes, int offset, double d) {
687 return putLong(bytes, offset, Double.doubleToLongBits(d));
688 }
689
690
691
692
693
694
695
696
697 public static byte [] toBytes(final double d) {
698
699 return Bytes.toBytes(Double.doubleToRawLongBits(d));
700 }
701
702
703
704
705
706
707
708
709 public static byte[] toBytes(int val) {
710 byte [] b = new byte[4];
711 for(int i = 3; i > 0; i--) {
712 b[i] = (byte) val;
713 val >>>= 8;
714 }
715 b[0] = (byte) val;
716 return b;
717 }
718
719
720
721
722
723
724 public static int toInt(byte[] bytes) {
725 return toInt(bytes, 0, SIZEOF_INT);
726 }
727
728
729
730
731
732
733
734 public static int toInt(byte[] bytes, int offset) {
735 return toInt(bytes, offset, SIZEOF_INT);
736 }
737
738
739
740
741
742
743
744
745
746
747 public static int toInt(byte[] bytes, int offset, final int length) {
748 if (length != SIZEOF_INT || offset + length > bytes.length) {
749 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
750 }
751 int n = 0;
752 for(int i = offset; i < (offset + length); i++) {
753 n <<= 8;
754 n ^= bytes[i] & 0xFF;
755 }
756 return n;
757 }
758
759
760
761
762
763
764
765
766
767
768 public static int readAsInt(byte[] bytes, int offset, final int length) {
769 if (offset + length > bytes.length) {
770 throw new IllegalArgumentException("offset (" + offset + ") + length (" + length
771 + ") exceed the" + " capacity of the array: " + bytes.length);
772 }
773 int n = 0;
774 for(int i = offset; i < (offset + length); i++) {
775 n <<= 8;
776 n ^= bytes[i] & 0xFF;
777 }
778 return n;
779 }
780
781
782
783
784
785
786
787
788
789
790 public static int putInt(byte[] bytes, int offset, int val) {
791 if (bytes.length - offset < SIZEOF_INT) {
792 throw new IllegalArgumentException("Not enough room to put an int at"
793 + " offset " + offset + " in a " + bytes.length + " byte array");
794 }
795 for(int i= offset + 3; i > offset; i--) {
796 bytes[i] = (byte) val;
797 val >>>= 8;
798 }
799 bytes[offset] = (byte) val;
800 return offset + SIZEOF_INT;
801 }
802
803
804
805
806
807
808 public static byte[] toBytes(short val) {
809 byte[] b = new byte[SIZEOF_SHORT];
810 b[1] = (byte) val;
811 val >>= 8;
812 b[0] = (byte) val;
813 return b;
814 }
815
816
817
818
819
820
821 public static short toShort(byte[] bytes) {
822 return toShort(bytes, 0, SIZEOF_SHORT);
823 }
824
825
826
827
828
829
830
831 public static short toShort(byte[] bytes, int offset) {
832 return toShort(bytes, offset, SIZEOF_SHORT);
833 }
834
835
836
837
838
839
840
841
842
843
844 public static short toShort(byte[] bytes, int offset, final int length) {
845 if (length != SIZEOF_SHORT || offset + length > bytes.length) {
846 throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
847 }
848 short n = 0;
849 n ^= bytes[offset] & 0xFF;
850 n <<= 8;
851 n ^= bytes[offset+1] & 0xFF;
852 return n;
853 }
854
855
856
857
858
859
860
861
862
863
864 public static byte[] getBytes(ByteBuffer buf) {
865 return readBytes(buf.duplicate());
866 }
867
868
869
870
871
872
873
874
875
876
877 public static int putShort(byte[] bytes, int offset, short val) {
878 if (bytes.length - offset < SIZEOF_SHORT) {
879 throw new IllegalArgumentException("Not enough room to put a short at"
880 + " offset " + offset + " in a " + bytes.length + " byte array");
881 }
882 bytes[offset+1] = (byte) val;
883 val >>= 8;
884 bytes[offset] = (byte) val;
885 return offset + SIZEOF_SHORT;
886 }
887
888
889
890
891
892
893
894
895
896
897
898
899
900 public static int putAsShort(byte[] bytes, int offset, int val) {
901 if (bytes.length - offset < SIZEOF_SHORT) {
902 throw new IllegalArgumentException("Not enough room to put a short at"
903 + " offset " + offset + " in a " + bytes.length + " byte array");
904 }
905 bytes[offset+1] = (byte) val;
906 val >>= 8;
907 bytes[offset] = (byte) val;
908 return offset + SIZEOF_SHORT;
909 }
910
911
912
913
914
915
916
917 public static byte[] toBytes(BigDecimal val) {
918 byte[] valueBytes = val.unscaledValue().toByteArray();
919 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
920 int offset = putInt(result, 0, val.scale());
921 putBytes(result, offset, valueBytes, 0, valueBytes.length);
922 return result;
923 }
924
925
926
927
928
929
930
931
932 public static BigDecimal toBigDecimal(byte[] bytes) {
933 return toBigDecimal(bytes, 0, bytes.length);
934 }
935
936
937
938
939
940
941
942
943
944 public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
945 if (bytes == null || length < SIZEOF_INT + 1 ||
946 (offset + length > bytes.length)) {
947 return null;
948 }
949
950 int scale = toInt(bytes, offset);
951 byte[] tcBytes = new byte[length - SIZEOF_INT];
952 System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
953 return new BigDecimal(new BigInteger(tcBytes), scale);
954 }
955
956
957
958
959
960
961
962
963
964 public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
965 if (bytes == null) {
966 return offset;
967 }
968
969 byte[] valueBytes = val.unscaledValue().toByteArray();
970 byte[] result = new byte[valueBytes.length + SIZEOF_INT];
971 offset = putInt(result, offset, val.scale());
972 return putBytes(result, offset, valueBytes, 0, valueBytes.length);
973 }
974
975
976
977
978
979 public static byte [] vintToBytes(final long vint) {
980 long i = vint;
981 int size = WritableUtils.getVIntSize(i);
982 byte [] result = new byte[size];
983 int offset = 0;
984 if (i >= -112 && i <= 127) {
985 result[offset] = (byte) i;
986 return result;
987 }
988
989 int len = -112;
990 if (i < 0) {
991 i ^= -1L;
992 len = -120;
993 }
994
995 long tmp = i;
996 while (tmp != 0) {
997 tmp = tmp >> 8;
998 len--;
999 }
1000
1001 result[offset++] = (byte) len;
1002
1003 len = (len < -120) ? -(len + 120) : -(len + 112);
1004
1005 for (int idx = len; idx != 0; idx--) {
1006 int shiftbits = (idx - 1) * 8;
1007 long mask = 0xFFL << shiftbits;
1008 result[offset++] = (byte)((i & mask) >> shiftbits);
1009 }
1010 return result;
1011 }
1012
1013
1014
1015
1016
1017 public static long bytesToVint(final byte [] buffer) {
1018 int offset = 0;
1019 byte firstByte = buffer[offset++];
1020 int len = WritableUtils.decodeVIntSize(firstByte);
1021 if (len == 1) {
1022 return firstByte;
1023 }
1024 long i = 0;
1025 for (int idx = 0; idx < len-1; idx++) {
1026 byte b = buffer[offset++];
1027 i = i << 8;
1028 i = i | (b & 0xFF);
1029 }
1030 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
1031 }
1032
1033
1034
1035
1036
1037
1038
1039
1040 public static long readVLong(final byte [] buffer, final int offset)
1041 throws IOException {
1042 byte firstByte = buffer[offset];
1043 int len = WritableUtils.decodeVIntSize(firstByte);
1044 if (len == 1) {
1045 return firstByte;
1046 }
1047 long i = 0;
1048 for (int idx = 0; idx < len-1; idx++) {
1049 byte b = buffer[offset + 1 + idx];
1050 i = i << 8;
1051 i = i | (b & 0xFF);
1052 }
1053 return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
1054 }
1055
1056
1057
1058
1059
1060
1061 public static int compareTo(final byte [] left, final byte [] right) {
1062 return LexicographicalComparerHolder.BEST_COMPARER.
1063 compareTo(left, 0, left.length, right, 0, right.length);
1064 }
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077 public static int compareTo(byte[] buffer1, int offset1, int length1,
1078 byte[] buffer2, int offset2, int length2) {
1079 return LexicographicalComparerHolder.BEST_COMPARER.
1080 compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
1081 }
1082
1083 interface Comparer<T> {
1084 int compareTo(
1085 T buffer1, int offset1, int length1, T buffer2, int offset2, int length2
1086 );
1087 }
1088
1089 @VisibleForTesting
1090 static Comparer<byte[]> lexicographicalComparerJavaImpl() {
1091 return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
1092 }
1093
1094
1095
1096
1097
1098
1099
1100
1101 @VisibleForTesting
1102 static class LexicographicalComparerHolder {
1103 static final String UNSAFE_COMPARER_NAME =
1104 LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
1105
1106 static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
1107
1108
1109
1110
1111 static Comparer<byte[]> getBestComparer() {
1112 try {
1113 Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
1114
1115
1116 @SuppressWarnings("unchecked")
1117 Comparer<byte[]> comparer =
1118 (Comparer<byte[]>) theClass.getEnumConstants()[0];
1119 return comparer;
1120 } catch (Throwable t) {
1121 return lexicographicalComparerJavaImpl();
1122 }
1123 }
1124
1125 enum PureJavaComparer implements Comparer<byte[]> {
1126 INSTANCE;
1127
1128 @Override
1129 public int compareTo(byte[] buffer1, int offset1, int length1,
1130 byte[] buffer2, int offset2, int length2) {
1131
1132 if (buffer1 == buffer2 &&
1133 offset1 == offset2 &&
1134 length1 == length2) {
1135 return 0;
1136 }
1137
1138 int end1 = offset1 + length1;
1139 int end2 = offset2 + length2;
1140 for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
1141 int a = (buffer1[i] & 0xff);
1142 int b = (buffer2[j] & 0xff);
1143 if (a != b) {
1144 return a - b;
1145 }
1146 }
1147 return length1 - length2;
1148 }
1149 }
1150
1151 @VisibleForTesting
1152 enum UnsafeComparer implements Comparer<byte[]> {
1153 INSTANCE;
1154
1155 static final Unsafe theUnsafe;
1156
1157
1158 static final int BYTE_ARRAY_BASE_OFFSET;
1159
1160 static {
1161 theUnsafe = (Unsafe) AccessController.doPrivileged(
1162 new PrivilegedAction<Object>() {
1163 @Override
1164 public Object run() {
1165 try {
1166 Field f = Unsafe.class.getDeclaredField("theUnsafe");
1167 f.setAccessible(true);
1168 return f.get(null);
1169 } catch (NoSuchFieldException e) {
1170
1171
1172 throw new Error();
1173 } catch (IllegalAccessException e) {
1174 throw new Error();
1175 }
1176 }
1177 });
1178
1179 BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
1180
1181
1182 if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
1183 throw new AssertionError();
1184 }
1185 }
1186
1187 static final boolean littleEndian =
1188 ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
1189
1190
1191
1192
1193
1194 static boolean lessThanUnsigned(long x1, long x2) {
1195 return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
1196 }
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209 @Override
1210 public int compareTo(byte[] buffer1, int offset1, int length1,
1211 byte[] buffer2, int offset2, int length2) {
1212
1213 if (buffer1 == buffer2 &&
1214 offset1 == offset2 &&
1215 length1 == length2) {
1216 return 0;
1217 }
1218 int minLength = Math.min(length1, length2);
1219 int minWords = minLength / SIZEOF_LONG;
1220 int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
1221 int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
1222
1223
1224
1225
1226
1227
1228 for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
1229 long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
1230 long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
1231 long diff = lw ^ rw;
1232
1233 if (diff != 0) {
1234 if (!littleEndian) {
1235 return lessThanUnsigned(lw, rw) ? -1 : 1;
1236 }
1237
1238
1239 int n = 0;
1240 int y;
1241 int x = (int) diff;
1242 if (x == 0) {
1243 x = (int) (diff >>> 32);
1244 n = 32;
1245 }
1246
1247 y = x << 16;
1248 if (y == 0) {
1249 n += 16;
1250 } else {
1251 x = y;
1252 }
1253
1254 y = x << 8;
1255 if (y == 0) {
1256 n += 8;
1257 }
1258 return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
1259 }
1260 }
1261
1262
1263 for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
1264 int a = (buffer1[offset1 + i] & 0xff);
1265 int b = (buffer2[offset2 + i] & 0xff);
1266 if (a != b) {
1267 return a - b;
1268 }
1269 }
1270 return length1 - length2;
1271 }
1272 }
1273 }
1274
1275
1276
1277
1278
1279
1280 public static boolean equals(final byte [] left, final byte [] right) {
1281
1282
1283 if (left == right) return true;
1284 if (left == null || right == null) return false;
1285 if (left.length != right.length) return false;
1286 if (left.length == 0) return true;
1287
1288
1289
1290
1291 if (left[left.length - 1] != right[right.length - 1]) return false;
1292
1293 return compareTo(left, right) == 0;
1294 }
1295
1296 public static boolean equals(final byte[] left, int leftOffset, int leftLen,
1297 final byte[] right, int rightOffset, int rightLen) {
1298
1299 if (left == right &&
1300 leftOffset == rightOffset &&
1301 leftLen == rightLen) {
1302 return true;
1303 }
1304
1305 if (leftLen != rightLen) {
1306 return false;
1307 }
1308 if (leftLen == 0) {
1309 return true;
1310 }
1311
1312
1313
1314
1315 if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
1316
1317 return LexicographicalComparerHolder.BEST_COMPARER.
1318 compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
1319 }
1320
1321
1322
1323
1324
1325
1326
1327 public static boolean equals(byte[] a, ByteBuffer buf) {
1328 if (a == null) return buf == null;
1329 if (buf == null) return false;
1330 if (a.length != buf.remaining()) return false;
1331
1332
1333 ByteBuffer b = buf.duplicate();
1334 for (byte anA : a) {
1335 if (anA != b.get()) {
1336 return false;
1337 }
1338 }
1339 return true;
1340 }
1341
1342
1343
1344
1345
1346
1347 public static boolean startsWith(byte[] bytes, byte[] prefix) {
1348 return bytes != null && prefix != null &&
1349 bytes.length >= prefix.length &&
1350 LexicographicalComparerHolder.BEST_COMPARER.
1351 compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
1352 }
1353
1354
1355
1356
1357
1358
1359
1360 public static int hashCode(final byte [] b) {
1361 return hashCode(b, b.length);
1362 }
1363
1364
1365
1366
1367
1368
1369
1370
1371 public static int hashCode(final byte [] b, final int length) {
1372 return WritableComparator.hashBytes(b, length);
1373 }
1374
1375
1376
1377
1378
1379
1380 public static Integer mapKey(final byte [] b) {
1381 return hashCode(b);
1382 }
1383
1384
1385
1386
1387
1388
1389
1390 public static Integer mapKey(final byte [] b, final int length) {
1391 return hashCode(b, length);
1392 }
1393
1394
1395
1396
1397
1398
1399 public static byte [] add(final byte [] a, final byte [] b) {
1400 return add(a, b, EMPTY_BYTE_ARRAY);
1401 }
1402
1403
1404
1405
1406
1407
1408
1409 public static byte [] add(final byte [] a, final byte [] b, final byte [] c) {
1410 byte [] result = new byte[a.length + b.length + c.length];
1411 System.arraycopy(a, 0, result, 0, a.length);
1412 System.arraycopy(b, 0, result, a.length, b.length);
1413 System.arraycopy(c, 0, result, a.length + b.length, c.length);
1414 return result;
1415 }
1416
1417
1418
1419
1420
1421
1422 public static byte [] head(final byte [] a, final int length) {
1423 if (a.length < length) {
1424 return null;
1425 }
1426 byte [] result = new byte[length];
1427 System.arraycopy(a, 0, result, 0, length);
1428 return result;
1429 }
1430
1431
1432
1433
1434
1435
1436 public static byte [] tail(final byte [] a, final int length) {
1437 if (a.length < length) {
1438 return null;
1439 }
1440 byte [] result = new byte[length];
1441 System.arraycopy(a, a.length - length, result, 0, length);
1442 return result;
1443 }
1444
1445
1446
1447
1448
1449
1450 public static byte [] padHead(final byte [] a, final int length) {
1451 byte [] padding = new byte[length];
1452 for (int i = 0; i < length; i++) {
1453 padding[i] = 0;
1454 }
1455 return add(padding,a);
1456 }
1457
1458
1459
1460
1461
1462
1463 public static byte [] padTail(final byte [] a, final int length) {
1464 byte [] padding = new byte[length];
1465 for (int i = 0; i < length; i++) {
1466 padding[i] = 0;
1467 }
1468 return add(a,padding);
1469 }
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480 public static byte [][] split(final byte [] a, final byte [] b, final int num) {
1481 return split(a, b, false, num);
1482 }
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496 public static byte[][] split(final byte[] a, final byte[] b,
1497 boolean inclusive, final int num) {
1498 byte[][] ret = new byte[num + 2][];
1499 int i = 0;
1500 Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
1501 if (iter == null)
1502 return null;
1503 for (byte[] elem : iter) {
1504 ret[i++] = elem;
1505 }
1506 return ret;
1507 }
1508
1509
1510
1511
1512 public static Iterable<byte[]> iterateOnSplits(final byte[] a,
1513 final byte[] b, final int num)
1514 {
1515 return iterateOnSplits(a, b, false, num);
1516 }
1517
1518
1519
1520
1521 public static Iterable<byte[]> iterateOnSplits(
1522 final byte[] a, final byte[]b, boolean inclusive, final int num)
1523 {
1524 byte [] aPadded;
1525 byte [] bPadded;
1526 if (a.length < b.length) {
1527 aPadded = padTail(a, b.length - a.length);
1528 bPadded = b;
1529 } else if (b.length < a.length) {
1530 aPadded = a;
1531 bPadded = padTail(b, a.length - b.length);
1532 } else {
1533 aPadded = a;
1534 bPadded = b;
1535 }
1536 if (compareTo(aPadded,bPadded) >= 0) {
1537 throw new IllegalArgumentException("b <= a");
1538 }
1539 if (num <= 0) {
1540 throw new IllegalArgumentException("num cannot be <= 0");
1541 }
1542 byte [] prependHeader = {1, 0};
1543 final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
1544 final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
1545 BigInteger diffBI = stopBI.subtract(startBI);
1546 if (inclusive) {
1547 diffBI = diffBI.add(BigInteger.ONE);
1548 }
1549 final BigInteger splitsBI = BigInteger.valueOf(num + 1);
1550 if(diffBI.compareTo(splitsBI) < 0) {
1551 return null;
1552 }
1553 final BigInteger intervalBI;
1554 try {
1555 intervalBI = diffBI.divide(splitsBI);
1556 } catch(Exception e) {
1557 LOG.error("Exception caught during division", e);
1558 return null;
1559 }
1560
1561 final Iterator<byte[]> iterator = new Iterator<byte[]>() {
1562 private int i = -1;
1563
1564 @Override
1565 public boolean hasNext() {
1566 return i < num+1;
1567 }
1568
1569 @Override
1570 public byte[] next() {
1571 i++;
1572 if (i == 0) return a;
1573 if (i == num + 1) return b;
1574
1575 BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
1576 byte [] padded = curBI.toByteArray();
1577 if (padded[1] == 0)
1578 padded = tail(padded, padded.length - 2);
1579 else
1580 padded = tail(padded, padded.length - 1);
1581 return padded;
1582 }
1583
1584 @Override
1585 public void remove() {
1586 throw new UnsupportedOperationException();
1587 }
1588
1589 };
1590
1591 return new Iterable<byte[]>() {
1592 @Override
1593 public Iterator<byte[]> iterator() {
1594 return iterator;
1595 }
1596 };
1597 }
1598
1599
1600
1601
1602
1603
1604 public static int hashCode(byte[] bytes, int offset, int length) {
1605 int hash = 1;
1606 for (int i = offset; i < offset + length; i++)
1607 hash = (31 * hash) + (int) bytes[i];
1608 return hash;
1609 }
1610
1611
1612
1613
1614
1615 public static byte [][] toByteArrays(final String [] t) {
1616 byte [][] result = new byte[t.length][];
1617 for (int i = 0; i < t.length; i++) {
1618 result[i] = Bytes.toBytes(t[i]);
1619 }
1620 return result;
1621 }
1622
1623
1624
1625
1626
1627
1628 public static byte [][] toByteArrays(final String column) {
1629 return toByteArrays(toBytes(column));
1630 }
1631
1632
1633
1634
1635
1636
1637 public static byte [][] toByteArrays(final byte [] column) {
1638 byte [][] result = new byte[1][];
1639 result[0] = column;
1640 return result;
1641 }
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658 public static int binarySearch(byte [][]arr, byte []key, int offset,
1659 int length, RawComparator<?> comparator) {
1660 int low = 0;
1661 int high = arr.length - 1;
1662
1663 while (low <= high) {
1664 int mid = (low+high) >>> 1;
1665
1666
1667 int cmp = comparator.compare(key, offset, length,
1668 arr[mid], 0, arr[mid].length);
1669
1670 if (cmp > 0)
1671 low = mid + 1;
1672
1673 else if (cmp < 0)
1674 high = mid - 1;
1675
1676 else
1677 return mid;
1678 }
1679 return - (low+1);
1680 }
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690 public static byte [] incrementBytes(byte[] value, long amount)
1691 {
1692 byte[] val = value;
1693 if (val.length < SIZEOF_LONG) {
1694
1695 byte [] newvalue;
1696 if (val[0] < 0) {
1697 newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1};
1698 } else {
1699 newvalue = new byte[SIZEOF_LONG];
1700 }
1701 System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
1702 val.length);
1703 val = newvalue;
1704 } else if (val.length > SIZEOF_LONG) {
1705 throw new IllegalArgumentException("Increment Bytes - value too big: " +
1706 val.length);
1707 }
1708 if(amount == 0) return val;
1709 if(val[0] < 0){
1710 return binaryIncrementNeg(val, amount);
1711 }
1712 return binaryIncrementPos(val, amount);
1713 }
1714
1715
1716 private static byte [] binaryIncrementPos(byte [] value, long amount) {
1717 long amo = amount;
1718 int sign = 1;
1719 if (amount < 0) {
1720 amo = -amount;
1721 sign = -1;
1722 }
1723 for(int i=0;i<value.length;i++) {
1724 int cur = ((int)amo % 256) * sign;
1725 amo = (amo >> 8);
1726 int val = value[value.length-i-1] & 0x0ff;
1727 int total = val + cur;
1728 if(total > 255) {
1729 amo += sign;
1730 total %= 256;
1731 } else if (total < 0) {
1732 amo -= sign;
1733 }
1734 value[value.length-i-1] = (byte)total;
1735 if (amo == 0) return value;
1736 }
1737 return value;
1738 }
1739
1740
1741 private static byte [] binaryIncrementNeg(byte [] value, long amount) {
1742 long amo = amount;
1743 int sign = 1;
1744 if (amount < 0) {
1745 amo = -amount;
1746 sign = -1;
1747 }
1748 for(int i=0;i<value.length;i++) {
1749 int cur = ((int)amo % 256) * sign;
1750 amo = (amo >> 8);
1751 int val = ((~value[value.length-i-1]) & 0x0ff) + 1;
1752 int total = cur - val;
1753 if(total >= 0) {
1754 amo += sign;
1755 } else if (total < -256) {
1756 amo -= sign;
1757 total %= 256;
1758 }
1759 value[value.length-i-1] = (byte)total;
1760 if (amo == 0) return value;
1761 }
1762 return value;
1763 }
1764
1765
1766
1767
1768 public static void writeStringFixedSize(final DataOutput out, String s,
1769 int size) throws IOException {
1770 byte[] b = toBytes(s);
1771 if (b.length > size) {
1772 throw new IOException("Trying to write " + b.length + " bytes (" +
1773 toStringBinary(b) + ") into a field of length " + size);
1774 }
1775
1776 out.writeBytes(s);
1777 for (int i = 0; i < size - s.length(); ++i)
1778 out.writeByte(0);
1779 }
1780
1781
1782
1783
1784 public static String readStringFixedSize(final DataInput in, int size)
1785 throws IOException {
1786 byte[] b = new byte[size];
1787 in.readFully(b);
1788 int n = b.length;
1789 while (n > 0 && b[n - 1] == 0)
1790 --n;
1791
1792 return toString(b, 0, n);
1793 }
1794
1795
1796
1797
1798
1799
1800
1801 public static byte [] copy(byte [] bytes) {
1802 if (bytes == null) return null;
1803 byte [] result = new byte[bytes.length];
1804 System.arraycopy(bytes, 0, result, 0, bytes.length);
1805 return result;
1806 }
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816 public static byte [] copy(byte [] bytes, final int offset, final int length) {
1817 if (bytes == null) return null;
1818 byte [] result = new byte[length];
1819 System.arraycopy(bytes, offset, result, 0, length);
1820 return result;
1821 }
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833 public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
1834 int unsignedKey = key & 0xff;
1835 int low = fromIndex;
1836 int high = toIndex - 1;
1837
1838 while (low <= high) {
1839 int mid = (low + high) >>> 1;
1840 int midVal = a[mid] & 0xff;
1841
1842 if (midVal < unsignedKey) {
1843 low = mid + 1;
1844 } else if (midVal > unsignedKey) {
1845 high = mid - 1;
1846 } else {
1847 return mid;
1848 }
1849 }
1850 return -(low + 1);
1851 }
1852
1853
1854
1855
1856
1857
1858
1859
1860 public static byte[] unsignedCopyAndIncrement(final byte[] input) {
1861 byte[] copy = copy(input);
1862 if (copy == null) {
1863 throw new IllegalArgumentException("cannot increment null array");
1864 }
1865 for (int i = copy.length - 1; i >= 0; --i) {
1866 if (copy[i] == -1) {
1867 copy[i] = 0;
1868 } else {
1869 ++copy[i];
1870 return copy;
1871 }
1872 }
1873
1874 byte[] out = new byte[copy.length + 1];
1875 out[0] = 1;
1876 System.arraycopy(copy, 0, out, 1, copy.length);
1877 return out;
1878 }
1879
1880 public static boolean equals(List<byte[]> a, List<byte[]> b) {
1881 if (a == null) {
1882 if (b == null) {
1883 return true;
1884 }
1885 return false;
1886 }
1887 if (b == null) {
1888 return false;
1889 }
1890 if (a.size() != b.size()) {
1891 return false;
1892 }
1893 for (int i = 0; i < a.size(); ++i) {
1894 if (!Bytes.equals(a.get(i), b.get(i))) {
1895 return false;
1896 }
1897 }
1898 return true;
1899 }
1900
1901 public static boolean isSorted(Collection<byte[]> arrays) {
1902 byte[] previous = new byte[0];
1903 for (byte[] array : IterableUtils.nullSafe(arrays)) {
1904 if (Bytes.compareTo(previous, array) > 0) {
1905 return false;
1906 }
1907 previous = array;
1908 }
1909 return true;
1910 }
1911
1912 public static List<byte[]> getUtf8ByteArrays(List<String> strings) {
1913 List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(strings));
1914 for (String s : IterableUtils.nullSafe(strings)) {
1915 byteArrays.add(Bytes.toBytes(s));
1916 }
1917 return byteArrays;
1918 }
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929 public static int indexOf(byte[] array, byte target) {
1930 for (int i = 0; i < array.length; i++) {
1931 if (array[i] == target) {
1932 return i;
1933 }
1934 }
1935 return -1;
1936 }
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949 public static int indexOf(byte[] array, byte[] target) {
1950 checkNotNull(array, "array");
1951 checkNotNull(target, "target");
1952 if (target.length == 0) {
1953 return 0;
1954 }
1955
1956 outer:
1957 for (int i = 0; i < array.length - target.length + 1; i++) {
1958 for (int j = 0; j < target.length; j++) {
1959 if (array[i + j] != target[j]) {
1960 continue outer;
1961 }
1962 }
1963 return i;
1964 }
1965 return -1;
1966 }
1967
1968
1969
1970
1971
1972
1973 public static boolean contains(byte[] array, byte target) {
1974 return indexOf(array, target) > -1;
1975 }
1976
1977
1978
1979
1980
1981
1982 public static boolean contains(byte[] array, byte[] target) {
1983 return indexOf(array, target) > -1;
1984 }
1985
1986
1987
1988
1989
1990 public static void zero(byte[] b) {
1991 zero(b, 0, b.length);
1992 }
1993
1994
1995
1996
1997
1998
1999
2000 public static void zero(byte[] b, int offset, int length) {
2001 checkPositionIndex(offset, b.length, "offset");
2002 checkArgument(length > 0, "length must be greater than 0");
2003 checkPositionIndex(offset + length, b.length, "offset + length");
2004 Arrays.fill(b, offset, offset + length, (byte) 0);
2005 }
2006
2007 private static final SecureRandom RNG = new SecureRandom();
2008
2009
2010
2011
2012
2013 public static void random(byte[] b) {
2014 RNG.nextBytes(b);
2015 }
2016
2017
2018
2019
2020
2021
2022
2023 public static void random(byte[] b, int offset, int length) {
2024 checkPositionIndex(offset, b.length, "offset");
2025 checkArgument(length > 0, "length must be greater than 0");
2026 checkPositionIndex(offset + length, b.length, "offset + length");
2027 byte[] buf = new byte[length];
2028 RNG.nextBytes(buf);
2029 System.arraycopy(buf, 0, b, offset, length);
2030 }
2031
2032
2033
2034
2035
2036
2037 public static byte[] createMaxByteArray(int maxByteCount) {
2038 byte[] maxByteArray = new byte[maxByteCount];
2039 for (int i = 0; i < maxByteArray.length; i++) {
2040 maxByteArray[i] = (byte) 0xff;
2041 }
2042 return maxByteArray;
2043 }
2044
2045
2046
2047
2048
2049
2050
2051 public static byte[] multiple(byte[] srcBytes, int multiNum) {
2052 if (multiNum <= 0) {
2053 return new byte[0];
2054 }
2055 byte[] result = new byte[srcBytes.length * multiNum];
2056 for (int i = 0; i < multiNum; i++) {
2057 System.arraycopy(srcBytes, 0, result, i * srcBytes.length,
2058 srcBytes.length);
2059 }
2060 return result;
2061 }
2062
2063
2064
2065
2066
2067 public static String toHex(byte[] b) {
2068 checkArgument(b.length > 0, "length must be greater than 0");
2069 return String.format("%x", new BigInteger(1, b));
2070 }
2071
2072
2073
2074
2075
2076
2077 public static byte[] fromHex(String hex) {
2078 checkArgument(hex.length() > 0, "length must be greater than 0");
2079 checkArgument(hex.length() % 2 == 0, "length must be a multiple of 2");
2080
2081 hex = hex.toUpperCase();
2082 byte[] b = new byte[hex.length() / 2];
2083 for (int i = 0; i < b.length; i++) {
2084 b[i] = (byte)((toBinaryFromHex((byte)hex.charAt(2 * i)) << 4) +
2085 toBinaryFromHex((byte)hex.charAt((2 * i + 1))));
2086 }
2087 return b;
2088 }
2089
2090 }