1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.*;
22
23 import java.io.IOException;
24 import java.util.NavigableSet;
25
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FSDataOutputStream;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
32 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.EntryBuffers;
33 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.RegionEntryBuffer;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.junit.Test;
36 import org.junit.experimental.categories.Category;
37
38 import static org.mockito.Mockito.mock;
39
40
41
42
43 @Category(SmallTests.class)
44 public class TestHLogMethods {
45 private static final byte[] TEST_REGION = Bytes.toBytes("test_region");;
46 private static final TableName TEST_TABLE =
47 TableName.valueOf("test_table");
48
49 private final HBaseTestingUtility util = new HBaseTestingUtility();
50
51
52
53
54
55
56 @Test public void testGetSplitEditFilesSorted() throws IOException {
57 FileSystem fs = FileSystem.get(util.getConfiguration());
58 Path regiondir = util.getDataTestDir("regiondir");
59 fs.delete(regiondir, true);
60 fs.mkdirs(regiondir);
61 Path recoverededits = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
62 String first = HLogSplitter.formatRecoveredEditsFileName(-1);
63 createFile(fs, recoverededits, first);
64 createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(0));
65 createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(1));
66 createFile(fs, recoverededits, HLogSplitter
67 .formatRecoveredEditsFileName(11));
68 createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(2));
69 createFile(fs, recoverededits, HLogSplitter
70 .formatRecoveredEditsFileName(50));
71 String last = HLogSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE);
72 createFile(fs, recoverededits, last);
73 createFile(fs, recoverededits,
74 Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis());
75
76 HLogFactory.createHLog(fs, regiondir, "dummyLogName", util.getConfiguration());
77 NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regiondir);
78 assertEquals(7, files.size());
79 assertEquals(files.pollFirst().getName(), first);
80 assertEquals(files.pollLast().getName(), last);
81 assertEquals(files.pollFirst().getName(),
82 HLogSplitter
83 .formatRecoveredEditsFileName(0));
84 assertEquals(files.pollFirst().getName(),
85 HLogSplitter
86 .formatRecoveredEditsFileName(1));
87 assertEquals(files.pollFirst().getName(),
88 HLogSplitter
89 .formatRecoveredEditsFileName(2));
90 assertEquals(files.pollFirst().getName(),
91 HLogSplitter
92 .formatRecoveredEditsFileName(11));
93 }
94
95 private void createFile(final FileSystem fs, final Path testdir,
96 final String name)
97 throws IOException {
98 FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
99 fdos.close();
100 }
101
102 @Test
103 public void testRegionEntryBuffer() throws Exception {
104 HLogSplitter.RegionEntryBuffer reb = new HLogSplitter.RegionEntryBuffer(
105 TEST_TABLE, TEST_REGION);
106 assertEquals(0, reb.heapSize());
107
108 reb.appendEntry(createTestLogEntry(1));
109 assertTrue(reb.heapSize() > 0);
110 }
111
112 @Test
113 public void testEntrySink() throws Exception {
114 Configuration conf = new Configuration();
115 RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
116 RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
117 HLogSplitter splitter = new HLogSplitter(
118 conf, mock(Path.class), mock(FileSystem.class), null, null, mode);
119
120 EntryBuffers sink = splitter.new EntryBuffers(1*1024*1024);
121 for (int i = 0; i < 1000; i++) {
122 HLog.Entry entry = createTestLogEntry(i);
123 sink.appendEntry(entry);
124 }
125
126 assertTrue(sink.totalBuffered > 0);
127 long amountInChunk = sink.totalBuffered;
128
129 RegionEntryBuffer chunk = sink.getChunkToWrite();
130 assertEquals(chunk.heapSize(), amountInChunk);
131
132
133 assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));
134
135
136 for (int i = 0; i < 500; i++) {
137 HLog.Entry entry = createTestLogEntry(i);
138 sink.appendEntry(entry);
139 }
140
141
142 assertNull(sink.getChunkToWrite());
143
144
145
146 sink.doneWriting(chunk);
147
148 RegionEntryBuffer chunk2 = sink.getChunkToWrite();
149 assertNotNull(chunk2);
150 assertNotSame(chunk, chunk2);
151 long amountInChunk2 = sink.totalBuffered;
152
153 assertTrue(amountInChunk2 < amountInChunk);
154
155 sink.doneWriting(chunk2);
156 assertEquals(0, sink.totalBuffered);
157 }
158
159 private HLog.Entry createTestLogEntry(int i) {
160 long seq = i;
161 long now = i * 1000;
162
163 WALEdit edit = new WALEdit();
164 edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
165 HLogKey key = new HLogKey(TEST_REGION, TEST_TABLE, seq, now,
166 HConstants.DEFAULT_CLUSTER_ID);
167 HLog.Entry entry = new HLog.Entry(key, edit);
168 return entry;
169 }
170
171 }
172