View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.hbase.*;
24  import org.apache.hadoop.hbase.client.*;
25  import org.apache.hadoop.hbase.test.MetricsAssertHelper;
26  import org.apache.hadoop.hbase.util.Bytes;
27  import org.apache.hadoop.hbase.util.Threads;
28  import org.apache.log4j.Level;
29  import org.apache.log4j.Logger;
30  import org.junit.AfterClass;
31  import org.junit.BeforeClass;
32  import org.junit.Test;
33  import org.junit.experimental.categories.Category;
34  import static org.junit.Assert.*;
35  
36  import java.io.IOException;
37  import java.util.ArrayList;
38  import java.util.List;
39  
40  
41  @Category(MediumTests.class)
42  public class TestRegionServerMetrics {
43    private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
44    private static MetricsAssertHelper metricsHelper;
45  
46    static {
47      Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
48    }
49  
50    private static MiniHBaseCluster cluster;
51    private static HRegionServer rs;
52    private static Configuration conf;
53    private static HBaseTestingUtility TEST_UTIL;
54    private static MetricsRegionServer metricsRegionServer;
55    private static MetricsRegionServerSource serverSource;
56  
57    @BeforeClass
58    public static void startCluster() throws Exception {
59      metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
60      TEST_UTIL = new HBaseTestingUtility();
61      conf = TEST_UTIL.getConfiguration();
62      conf.getLong("hbase.splitlog.max.resubmit", 0);
63      // Make the failure test faster
64      conf.setInt("zookeeper.recovery.retry", 0);
65      conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
66  
67      TEST_UTIL.startMiniCluster(1, 1);
68      cluster = TEST_UTIL.getHBaseCluster();
69  
70      cluster.waitForActiveAndReadyMaster();
71  
72      while (cluster.getLiveRegionServerThreads().size() < 1) {
73        Threads.sleep(100);
74      }
75  
76      rs = cluster.getRegionServer(0);
77      metricsRegionServer = rs.getMetrics();
78      serverSource = metricsRegionServer.getMetricsSource();
79    }
80  
81    @AfterClass
82    public static void after() throws Exception {
83      if (TEST_UTIL != null) {
84        TEST_UTIL.shutdownMiniCluster();
85      }
86    }
87  
88    @Test(timeout = 300000)
89    public void testRegionCount() throws Exception {
90      String regionMetricsKey = "regionCount";
91      long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
92      // Creating a table should add one region
93      TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf"));
94      metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
95    }
96  
97    @Test
98    public void testLocalFiles() throws Exception {
99      metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
100   }
101 
102   @Test
103   public void testRequestCount() throws Exception {
104     String tableNameString = "testRequestCount";
105     byte[] tName = Bytes.toBytes(tableNameString);
106     byte[] cfName = Bytes.toBytes("d");
107     byte[] row = Bytes.toBytes("rk");
108     byte[] qualifier = Bytes.toBytes("qual");
109     byte[] initValue = Bytes.toBytes("Value");
110     byte[] nextValue = Bytes.toBytes("NEXT VAL");
111 
112 
113     TEST_UTIL.createTable(tName, cfName);
114 
115     new HTable(conf, tName).close(); //wait for the table to come up.
116 
117     // Do a first put to be sure that the connection is established, meta is there and so on.
118     HTable table = new HTable(conf, tName);
119     Put p = new Put(row);
120     p.add(cfName, qualifier, initValue);
121     table.put(p);
122 
123     metricsRegionServer.getRegionServerWrapper().forceRecompute();
124     long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
125     long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
126     long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
127 
128     for (int i=0; i< 30; i++) {
129       table.put(p);
130     }
131 
132     metricsRegionServer.getRegionServerWrapper().forceRecompute();
133     metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
134     metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
135     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
136 
137     Get g = new Get(row);
138     for (int i=0; i< 10; i++) {
139       table.get(g);
140     }
141 
142     metricsRegionServer.getRegionServerWrapper().forceRecompute();
143     metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
144     metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
145     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
146 
147     for ( HRegionInfo i:table.getRegionLocations().keySet()) {
148       MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
149           .getMetrics()
150           .getSource()
151           .getAggregateSource();
152       String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
153           "_table_"+tableNameString +
154           "_region_" + i.getEncodedName()+
155           "_metric";
156       metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
157       metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
158     }
159 
160     // 0.98 specific; the loop above does reads. The 0.99 does not
161     metricsRegionServer.getRegionServerWrapper().forceRecompute();
162     metricsHelper.assertCounter("totalRequestCount", requests + 40 + 3, serverSource);
163     metricsHelper.assertCounter("readRequestCount", readRequests + 10 + 1, serverSource);
164     // end of 0.98 specific
165 
166     List<Get> gets = new ArrayList<Get>();
167     for (int i=0; i< 10; i++) {
168       gets.add(new Get(row));
169     }
170     table.get(gets);
171 
172     metricsRegionServer.getRegionServerWrapper().forceRecompute();
173     metricsHelper.assertCounter("totalRequestCount", requests + 50 + 3, serverSource);
174     metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
175     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
176 
177     table.setAutoFlushTo(false);
178     for (int i=0; i< 30; i++) {
179       table.put(p);
180     }
181     table.flushCommits();
182 
183     metricsRegionServer.getRegionServerWrapper().forceRecompute();
184     metricsHelper.assertCounter("totalRequestCount", requests + 80 + 3, serverSource);
185     metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
186     metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
187 
188     table.close();
189   }
190 
191   @Test
192   public void testMutationsWithoutWal() throws Exception {
193     byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
194     byte[] cf = Bytes.toBytes("d");
195     byte[] row = Bytes.toBytes("rk");
196     byte[] qualifier = Bytes.toBytes("qual");
197     byte[] val = Bytes.toBytes("Value");
198 
199     metricsRegionServer.getRegionServerWrapper().forceRecompute();
200 
201     TEST_UTIL.createTable(tableName, cf);
202 
203     HTable t = new HTable(conf, tableName);
204 
205     Put p = new Put(row);
206     p.add(cf, qualifier, val);
207     p.setDurability(Durability.SKIP_WAL);
208 
209     t.put(p);
210     t.flushCommits();
211 
212     metricsRegionServer.getRegionServerWrapper().forceRecompute();
213     metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
214     long minLength = row.length + cf.length + qualifier.length + val.length;
215     metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
216 
217     t.close();
218   }
219 
220   @Test
221   public void testStoreCount() throws Exception {
222     byte[] tableName = Bytes.toBytes("testStoreCount");
223     byte[] cf = Bytes.toBytes("d");
224     byte[] row = Bytes.toBytes("rk");
225     byte[] qualifier = Bytes.toBytes("qual");
226     byte[] val = Bytes.toBytes("Value");
227 
228     metricsRegionServer.getRegionServerWrapper().forceRecompute();
229     long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
230     long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
231 
232     TEST_UTIL.createTable(tableName, cf);
233 
234     //Force a hfile.
235     HTable t = new HTable(conf, tableName);
236     Put p = new Put(row);
237     p.add(cf, qualifier, val);
238     t.put(p);
239     t.flushCommits();
240     TEST_UTIL.getHBaseAdmin().flush(tableName);
241 
242     metricsRegionServer.getRegionServerWrapper().forceRecompute();
243     metricsHelper.assertGauge("storeCount", stores +1, serverSource);
244     metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
245 
246     t.close();
247   }
248 
249   @Test
250   public void testCheckAndPutCount() throws Exception {
251     String tableNameString = "testCheckAndPutCount";
252     byte[] tableName = Bytes.toBytes(tableNameString);
253     byte[] cf = Bytes.toBytes("d");
254     byte[] row = Bytes.toBytes("rk");
255     byte[] qualifier = Bytes.toBytes("qual");
256     byte[] valOne = Bytes.toBytes("Value");
257     byte[] valTwo = Bytes.toBytes("ValueTwo");
258     byte[] valThree = Bytes.toBytes("ValueThree");
259 
260     TEST_UTIL.createTable(tableName, cf);
261     HTable t = new HTable(conf, tableName);
262     Put p = new Put(row);
263     p.add(cf, qualifier, valOne);
264     t.put(p);
265     t.flushCommits();
266 
267     Put pTwo = new Put(row);
268     pTwo.add(cf, qualifier, valTwo);
269     t.checkAndPut(row, cf, qualifier, valOne, pTwo);
270     t.flushCommits();
271 
272     Put pThree = new Put(row);
273     pThree.add(cf, qualifier, valThree);
274     t.checkAndPut(row, cf, qualifier, valOne, pThree);
275     t.flushCommits();
276 
277 
278     metricsRegionServer.getRegionServerWrapper().forceRecompute();
279     metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
280     metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
281 
282     t.close();
283   }
284 
285   @Test
286   public void testIncrement() throws Exception {
287     String tableNameString = "testIncrement";
288     byte[] tableName = Bytes.toBytes(tableNameString);
289     byte[] cf = Bytes.toBytes("d");
290     byte[] row = Bytes.toBytes("rk");
291     byte[] qualifier = Bytes.toBytes("qual");
292     byte[] val = Bytes.toBytes(0l);
293 
294 
295     TEST_UTIL.createTable(tableName, cf);
296     HTable t = new HTable(conf, tableName);
297 
298     Put p = new Put(row);
299     p.add(cf, qualifier, val);
300     t.put(p);
301     t.flushCommits();
302 
303     for(int count = 0; count< 13; count++) {
304       Increment inc = new Increment(row);
305       inc.addColumn(cf, qualifier, 100);
306       t.increment(inc);
307     }
308 
309     t.flushCommits();
310 
311     metricsRegionServer.getRegionServerWrapper().forceRecompute();
312     metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
313 
314     t.close();
315   }
316 
317   @Test
318   public void testAppend() throws Exception {
319     String tableNameString = "testAppend";
320     byte[] tableName = Bytes.toBytes(tableNameString);
321     byte[] cf = Bytes.toBytes("d");
322     byte[] row = Bytes.toBytes("rk");
323     byte[] qualifier = Bytes.toBytes("qual");
324     byte[] val = Bytes.toBytes("One");
325 
326 
327     TEST_UTIL.createTable(tableName, cf);
328     HTable t = new HTable(conf, tableName);
329 
330     Put p = new Put(row);
331     p.add(cf, qualifier, val);
332     t.put(p);
333     t.flushCommits();
334 
335     for(int count = 0; count< 73; count++) {
336       Append append = new Append(row);
337       append.add(cf, qualifier, Bytes.toBytes(",Test"));
338       t.append(append);
339     }
340 
341     t.flushCommits();
342 
343     metricsRegionServer.getRegionServerWrapper().forceRecompute();
344     metricsHelper.assertCounter("appendNumOps", 73, serverSource);
345 
346     t.close();
347   }
348 
349   @Test
350   public void testScanNext() throws IOException {
351     String tableNameString = "testScanNext";
352     byte[] tableName = Bytes.toBytes(tableNameString);
353     byte[] cf = Bytes.toBytes("d");
354     byte[] qualifier = Bytes.toBytes("qual");
355     byte[] val = Bytes.toBytes("One");
356 
357 
358     TEST_UTIL.createTable(tableName, cf);
359     HTable t = new HTable(conf, tableName);
360     t.setAutoFlush(false, true);
361     for (int insertCount =0; insertCount < 100; insertCount++) {
362       Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
363       p.add(cf, qualifier, val);
364       t.put(p);
365     }
366     t.flushCommits();
367 
368     Scan s = new Scan();
369     s.setBatch(1);
370     s.setCaching(1);
371     ResultScanner resultScanners = t.getScanner(s);
372 
373     for (int nextCount = 0; nextCount < 30; nextCount++) {
374       Result result = resultScanners.next();
375       assertNotNull(result);
376       assertEquals(1, result.size());
377     }
378     for ( HRegionInfo i:t.getRegionLocations().keySet()) {
379       MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
380           .getMetrics()
381           .getSource()
382           .getAggregateSource();
383       String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
384           "_table_"+tableNameString +
385           "_region_" + i.getEncodedName()+
386           "_metric";
387       metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
388     }
389   }
390 }