1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.hbase.*;
24 import org.apache.hadoop.hbase.client.*;
25 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
26 import org.apache.hadoop.hbase.util.Bytes;
27 import org.apache.hadoop.hbase.util.Threads;
28 import org.apache.log4j.Level;
29 import org.apache.log4j.Logger;
30 import org.junit.AfterClass;
31 import org.junit.BeforeClass;
32 import org.junit.Test;
33 import org.junit.experimental.categories.Category;
34 import static org.junit.Assert.*;
35
36 import java.io.IOException;
37 import java.util.ArrayList;
38 import java.util.List;
39
40
41 @Category(MediumTests.class)
42 public class TestRegionServerMetrics {
43 private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
44 private static MetricsAssertHelper metricsHelper;
45
46 static {
47 Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
48 }
49
50 private static MiniHBaseCluster cluster;
51 private static HRegionServer rs;
52 private static Configuration conf;
53 private static HBaseTestingUtility TEST_UTIL;
54 private static MetricsRegionServer metricsRegionServer;
55 private static MetricsRegionServerSource serverSource;
56
57 @BeforeClass
58 public static void startCluster() throws Exception {
59 metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
60 TEST_UTIL = new HBaseTestingUtility();
61 conf = TEST_UTIL.getConfiguration();
62 conf.getLong("hbase.splitlog.max.resubmit", 0);
63
64 conf.setInt("zookeeper.recovery.retry", 0);
65 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
66
67 TEST_UTIL.startMiniCluster(1, 1);
68 cluster = TEST_UTIL.getHBaseCluster();
69
70 cluster.waitForActiveAndReadyMaster();
71
72 while (cluster.getLiveRegionServerThreads().size() < 1) {
73 Threads.sleep(100);
74 }
75
76 rs = cluster.getRegionServer(0);
77 metricsRegionServer = rs.getMetrics();
78 serverSource = metricsRegionServer.getMetricsSource();
79 }
80
81 @AfterClass
82 public static void after() throws Exception {
83 if (TEST_UTIL != null) {
84 TEST_UTIL.shutdownMiniCluster();
85 }
86 }
87
88 @Test(timeout = 300000)
89 public void testRegionCount() throws Exception {
90 String regionMetricsKey = "regionCount";
91 long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
92
93 TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf"));
94 metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
95 }
96
97 @Test
98 public void testLocalFiles() throws Exception {
99 metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
100 }
101
102 @Test
103 public void testRequestCount() throws Exception {
104 String tableNameString = "testRequestCount";
105 byte[] tName = Bytes.toBytes(tableNameString);
106 byte[] cfName = Bytes.toBytes("d");
107 byte[] row = Bytes.toBytes("rk");
108 byte[] qualifier = Bytes.toBytes("qual");
109 byte[] initValue = Bytes.toBytes("Value");
110 byte[] nextValue = Bytes.toBytes("NEXT VAL");
111
112
113 TEST_UTIL.createTable(tName, cfName);
114
115 new HTable(conf, tName).close();
116
117
118 HTable table = new HTable(conf, tName);
119 Put p = new Put(row);
120 p.add(cfName, qualifier, initValue);
121 table.put(p);
122
123 metricsRegionServer.getRegionServerWrapper().forceRecompute();
124 long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
125 long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
126 long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
127
128 for (int i=0; i< 30; i++) {
129 table.put(p);
130 }
131
132 metricsRegionServer.getRegionServerWrapper().forceRecompute();
133 metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
134 metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
135 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
136
137 Get g = new Get(row);
138 for (int i=0; i< 10; i++) {
139 table.get(g);
140 }
141
142 metricsRegionServer.getRegionServerWrapper().forceRecompute();
143 metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
144 metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
145 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
146
147 for ( HRegionInfo i:table.getRegionLocations().keySet()) {
148 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
149 .getMetrics()
150 .getSource()
151 .getAggregateSource();
152 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
153 "_table_"+tableNameString +
154 "_region_" + i.getEncodedName()+
155 "_metric";
156 metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
157 metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
158 }
159
160
161 metricsRegionServer.getRegionServerWrapper().forceRecompute();
162 metricsHelper.assertCounter("totalRequestCount", requests + 40 + 3, serverSource);
163 metricsHelper.assertCounter("readRequestCount", readRequests + 10 + 1, serverSource);
164
165
166 List<Get> gets = new ArrayList<Get>();
167 for (int i=0; i< 10; i++) {
168 gets.add(new Get(row));
169 }
170 table.get(gets);
171
172 metricsRegionServer.getRegionServerWrapper().forceRecompute();
173 metricsHelper.assertCounter("totalRequestCount", requests + 50 + 3, serverSource);
174 metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
175 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
176
177 table.setAutoFlushTo(false);
178 for (int i=0; i< 30; i++) {
179 table.put(p);
180 }
181 table.flushCommits();
182
183 metricsRegionServer.getRegionServerWrapper().forceRecompute();
184 metricsHelper.assertCounter("totalRequestCount", requests + 80 + 3, serverSource);
185 metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
186 metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
187
188 table.close();
189 }
190
191 @Test
192 public void testMutationsWithoutWal() throws Exception {
193 byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
194 byte[] cf = Bytes.toBytes("d");
195 byte[] row = Bytes.toBytes("rk");
196 byte[] qualifier = Bytes.toBytes("qual");
197 byte[] val = Bytes.toBytes("Value");
198
199 metricsRegionServer.getRegionServerWrapper().forceRecompute();
200
201 TEST_UTIL.createTable(tableName, cf);
202
203 HTable t = new HTable(conf, tableName);
204
205 Put p = new Put(row);
206 p.add(cf, qualifier, val);
207 p.setDurability(Durability.SKIP_WAL);
208
209 t.put(p);
210 t.flushCommits();
211
212 metricsRegionServer.getRegionServerWrapper().forceRecompute();
213 metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
214 long minLength = row.length + cf.length + qualifier.length + val.length;
215 metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
216
217 t.close();
218 }
219
220 @Test
221 public void testStoreCount() throws Exception {
222 byte[] tableName = Bytes.toBytes("testStoreCount");
223 byte[] cf = Bytes.toBytes("d");
224 byte[] row = Bytes.toBytes("rk");
225 byte[] qualifier = Bytes.toBytes("qual");
226 byte[] val = Bytes.toBytes("Value");
227
228 metricsRegionServer.getRegionServerWrapper().forceRecompute();
229 long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
230 long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
231
232 TEST_UTIL.createTable(tableName, cf);
233
234
235 HTable t = new HTable(conf, tableName);
236 Put p = new Put(row);
237 p.add(cf, qualifier, val);
238 t.put(p);
239 t.flushCommits();
240 TEST_UTIL.getHBaseAdmin().flush(tableName);
241
242 metricsRegionServer.getRegionServerWrapper().forceRecompute();
243 metricsHelper.assertGauge("storeCount", stores +1, serverSource);
244 metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
245
246 t.close();
247 }
248
249 @Test
250 public void testCheckAndPutCount() throws Exception {
251 String tableNameString = "testCheckAndPutCount";
252 byte[] tableName = Bytes.toBytes(tableNameString);
253 byte[] cf = Bytes.toBytes("d");
254 byte[] row = Bytes.toBytes("rk");
255 byte[] qualifier = Bytes.toBytes("qual");
256 byte[] valOne = Bytes.toBytes("Value");
257 byte[] valTwo = Bytes.toBytes("ValueTwo");
258 byte[] valThree = Bytes.toBytes("ValueThree");
259
260 TEST_UTIL.createTable(tableName, cf);
261 HTable t = new HTable(conf, tableName);
262 Put p = new Put(row);
263 p.add(cf, qualifier, valOne);
264 t.put(p);
265 t.flushCommits();
266
267 Put pTwo = new Put(row);
268 pTwo.add(cf, qualifier, valTwo);
269 t.checkAndPut(row, cf, qualifier, valOne, pTwo);
270 t.flushCommits();
271
272 Put pThree = new Put(row);
273 pThree.add(cf, qualifier, valThree);
274 t.checkAndPut(row, cf, qualifier, valOne, pThree);
275 t.flushCommits();
276
277
278 metricsRegionServer.getRegionServerWrapper().forceRecompute();
279 metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
280 metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
281
282 t.close();
283 }
284
285 @Test
286 public void testIncrement() throws Exception {
287 String tableNameString = "testIncrement";
288 byte[] tableName = Bytes.toBytes(tableNameString);
289 byte[] cf = Bytes.toBytes("d");
290 byte[] row = Bytes.toBytes("rk");
291 byte[] qualifier = Bytes.toBytes("qual");
292 byte[] val = Bytes.toBytes(0l);
293
294
295 TEST_UTIL.createTable(tableName, cf);
296 HTable t = new HTable(conf, tableName);
297
298 Put p = new Put(row);
299 p.add(cf, qualifier, val);
300 t.put(p);
301 t.flushCommits();
302
303 for(int count = 0; count< 13; count++) {
304 Increment inc = new Increment(row);
305 inc.addColumn(cf, qualifier, 100);
306 t.increment(inc);
307 }
308
309 t.flushCommits();
310
311 metricsRegionServer.getRegionServerWrapper().forceRecompute();
312 metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
313
314 t.close();
315 }
316
317 @Test
318 public void testAppend() throws Exception {
319 String tableNameString = "testAppend";
320 byte[] tableName = Bytes.toBytes(tableNameString);
321 byte[] cf = Bytes.toBytes("d");
322 byte[] row = Bytes.toBytes("rk");
323 byte[] qualifier = Bytes.toBytes("qual");
324 byte[] val = Bytes.toBytes("One");
325
326
327 TEST_UTIL.createTable(tableName, cf);
328 HTable t = new HTable(conf, tableName);
329
330 Put p = new Put(row);
331 p.add(cf, qualifier, val);
332 t.put(p);
333 t.flushCommits();
334
335 for(int count = 0; count< 73; count++) {
336 Append append = new Append(row);
337 append.add(cf, qualifier, Bytes.toBytes(",Test"));
338 t.append(append);
339 }
340
341 t.flushCommits();
342
343 metricsRegionServer.getRegionServerWrapper().forceRecompute();
344 metricsHelper.assertCounter("appendNumOps", 73, serverSource);
345
346 t.close();
347 }
348
349 @Test
350 public void testScanNext() throws IOException {
351 String tableNameString = "testScanNext";
352 byte[] tableName = Bytes.toBytes(tableNameString);
353 byte[] cf = Bytes.toBytes("d");
354 byte[] qualifier = Bytes.toBytes("qual");
355 byte[] val = Bytes.toBytes("One");
356
357
358 TEST_UTIL.createTable(tableName, cf);
359 HTable t = new HTable(conf, tableName);
360 t.setAutoFlush(false, true);
361 for (int insertCount =0; insertCount < 100; insertCount++) {
362 Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
363 p.add(cf, qualifier, val);
364 t.put(p);
365 }
366 t.flushCommits();
367
368 Scan s = new Scan();
369 s.setBatch(1);
370 s.setCaching(1);
371 ResultScanner resultScanners = t.getScanner(s);
372
373 for (int nextCount = 0; nextCount < 30; nextCount++) {
374 Result result = resultScanners.next();
375 assertNotNull(result);
376 assertEquals(1, result.size());
377 }
378 for ( HRegionInfo i:t.getRegionLocations().keySet()) {
379 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
380 .getMetrics()
381 .getSource()
382 .getAggregateSource();
383 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
384 "_table_"+tableNameString +
385 "_region_" + i.getEncodedName()+
386 "_metric";
387 metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
388 }
389 }
390 }