1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.*;
22
23 import java.io.IOException;
24 import java.util.List;
25 import java.util.ArrayList;
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.client.HTable;
30 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
31 import org.apache.hadoop.hbase.regionserver.HRegionServer;
32 import org.apache.hadoop.hbase.regionserver.HRegion;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.apache.hadoop.hbase.util.JVMClusterUtil;
35 import org.apache.hadoop.hbase.util.Threads;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38
39
40
41
42
43 @Category(MediumTests.class)
44 public class TestGlobalMemStoreSize {
45 private final Log LOG = LogFactory.getLog(this.getClass().getName());
46 private static int regionServerNum = 4;
47 private static int regionNum = 16;
48
49 private static int totalRegionNum = regionNum+2;
50
51 private HBaseTestingUtility TEST_UTIL;
52 private MiniHBaseCluster cluster;
53
54
55
56
57
58
59 @Test
60 public void testGlobalMemStore() throws Exception {
61
62 LOG.info("Starting cluster");
63 Configuration conf = HBaseConfiguration.create();
64 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
65 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000);
66 TEST_UTIL = new HBaseTestingUtility(conf);
67 TEST_UTIL.startMiniCluster(1, regionServerNum);
68 cluster = TEST_UTIL.getHBaseCluster();
69 LOG.info("Waiting for active/ready master");
70 cluster.waitForActiveAndReadyMaster();
71
72
73 byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
74 byte [] family = Bytes.toBytes("family");
75 LOG.info("Creating table with " + regionNum + " regions");
76 HTable ht = TEST_UTIL.createTable(table, family);
77 int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family,
78 regionNum);
79 assertEquals(regionNum,numRegions);
80 waitForAllRegionsAssigned();
81
82 for (HRegionServer server : getOnlineRegionServers()) {
83 long globalMemStoreSize = 0;
84 for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server)) {
85 globalMemStoreSize +=
86 server.getFromOnlineRegions(regionInfo.getEncodedName()).
87 getMemstoreSize().get();
88 }
89 assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
90 globalMemStoreSize);
91 }
92
93
94 int i = 0;
95 for (HRegionServer server : getOnlineRegionServers()) {
96 LOG.info("Starting flushes on " + server.getServerName() +
97 ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
98
99 for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server)) {
100 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
101 flush(r, server);
102 }
103 LOG.info("Post flush on " + server.getServerName());
104 long now = System.currentTimeMillis();
105 long timeout = now + 1000;
106 while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
107 timeout < System.currentTimeMillis()) {
108 Threads.sleep(10);
109 }
110 long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
111 if (size > 0) {
112
113
114 for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server)) {
115 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
116 long l = r.getMemstoreSize().longValue();
117 if (l > 0) {
118
119
120 assertTrue(regionInfo.isMetaRegion());
121 LOG.info(r.toString() + " " + l + ", reflushing");
122 r.flushcache();
123 }
124 }
125 }
126 size = server.getRegionServerAccounting().getGlobalMemstoreSize();
127 assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
128 }
129
130 ht.close();
131 TEST_UTIL.shutdownMiniCluster();
132 }
133
134
135
136
137
138
139
140 private void flush(final HRegion r, final HRegionServer server)
141 throws IOException {
142 LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
143 ", " + r.flushcache() + ", size=" +
144 server.getRegionServerAccounting().getGlobalMemstoreSize());
145 }
146
147
148 private int getRegionCount() throws IOException {
149 int total = 0;
150 for (HRegionServer server : getOnlineRegionServers()) {
151 total += ProtobufUtil.getOnlineRegions(server).size();
152 }
153 return total;
154 }
155
156 private List<HRegionServer> getOnlineRegionServers() {
157 List<HRegionServer> list = new ArrayList<HRegionServer>();
158 for (JVMClusterUtil.RegionServerThread rst :
159 cluster.getRegionServerThreads()) {
160 if (rst.getRegionServer().isOnline()) {
161 list.add(rst.getRegionServer());
162 }
163 }
164 return list;
165 }
166
167
168
169
170 private void waitForAllRegionsAssigned() throws IOException {
171 while (getRegionCount() < totalRegionNum) {
172 LOG.debug("Waiting for there to be "+totalRegionNum+" regions, but there are " + getRegionCount() + " right now.");
173 try {
174 Thread.sleep(100);
175 } catch (InterruptedException e) {}
176 }
177 }
178
179 }