1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.catalog.CatalogTracker;
25 import org.apache.hadoop.hbase.executor.EventType;
26 import org.apache.hadoop.hbase.executor.ExecutorService;
27 import org.apache.hadoop.hbase.executor.ExecutorType;
28 import org.apache.hadoop.hbase.master.AssignmentManager;
29 import org.apache.hadoop.hbase.master.HMaster;
30 import org.apache.hadoop.hbase.master.LoadBalancer;
31 import org.apache.hadoop.hbase.master.RegionPlan;
32 import org.apache.hadoop.hbase.master.RegionState;
33 import org.apache.hadoop.hbase.master.ServerManager;
34 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
35 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
36 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
37 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38 import org.junit.AfterClass;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42 import org.mockito.Mockito;
43
44 import java.util.ArrayList;
45 import java.util.HashMap;
46 import java.util.HashSet;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Map.Entry;
50 import java.util.Set;
51
52 import static org.junit.Assert.assertFalse;
53 import static org.junit.Assert.assertNotEquals;
54 import static org.junit.Assert.assertTrue;
55
56
57
58
59
60 @Category(MediumTests.class)
61 public class TestDrainingServer {
62 private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
63 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
64 private Abortable abortable = new Abortable() {
65 @Override
66 public boolean isAborted() {
67 return false;
68 }
69
70 @Override
71 public void abort(String why, Throwable e) {
72 }
73 };
74
75 @AfterClass
76 public static void afterClass() throws Exception {
77 TEST_UTIL.shutdownMiniZKCluster();
78 }
79
80 @BeforeClass
81 public static void beforeClass() throws Exception {
82 TEST_UTIL.startMiniZKCluster();
83 }
84
85 @Test
86 public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
87 AssignmentManager am;
88 Configuration conf = TEST_UTIL.getConfiguration();
89 final HMaster master = Mockito.mock(HMaster.class);
90 final Server server = Mockito.mock(Server.class);
91 final ServerManager serverManager = Mockito.mock(ServerManager.class);
92 final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
93 final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
94 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
95 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
96 final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
97 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
98
99 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
100 "zkWatcher-Test", abortable, true);
101
102 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
103
104 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
105 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
106
107 Mockito.when(server.getConfiguration()).thenReturn(conf);
108 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
109 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
110
111 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
112 Mockito.when(serverManager.getOnlineServersList())
113 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
114
115 Mockito.when(serverManager.createDestinationServersList())
116 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
117 Mockito.when(serverManager.createDestinationServersList(null))
118 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
119
120 for (ServerName sn : onlineServers.keySet()) {
121 Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
122 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
123 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
124 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
125 .thenReturn(RegionOpeningState.OPENED);
126 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
127 .thenReturn(RegionOpeningState.OPENED);
128 Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
129 }
130
131 Mockito.when(master.getServerManager()).thenReturn(serverManager);
132
133 am = new AssignmentManager(server, serverManager, catalogTracker,
134 balancer, startupMasterExecutor("mockExecutorService"), null, null);
135
136 Mockito.when(master.getAssignmentManager()).thenReturn(am);
137 Mockito.when(master.getZooKeeperWatcher()).thenReturn(zkWatcher);
138 Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
139
140 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
141
142 zkWatcher.registerListenerFirst(am);
143
144 addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
145
146 am.assign(REGIONINFO, true);
147
148 setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
149 setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
150
151 am.waitForAssignment(REGIONINFO);
152
153 assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
154 assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
155 }
156
157 @Test
158 public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
159 Configuration conf = TEST_UTIL.getConfiguration();
160 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
161 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
162 AssignmentManager am;
163 final HMaster master = Mockito.mock(HMaster.class);
164 final Server server = Mockito.mock(Server.class);
165 final ServerManager serverManager = Mockito.mock(ServerManager.class);
166 final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
167 final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
168 final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
169 final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
170 final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
171 final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
172
173 Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
174
175 HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
176 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
177 HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
178 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
179 HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
180 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
181 HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
182 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
183 HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
184 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
185
186 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
187 List<ServerName> drainedServers = new ArrayList<ServerName>();
188
189 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
190 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
191 onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
192 onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
193 onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
194
195 bulk.put(REGIONINFO_A, SERVERNAME_A);
196 bulk.put(REGIONINFO_B, SERVERNAME_B);
197 bulk.put(REGIONINFO_C, SERVERNAME_C);
198 bulk.put(REGIONINFO_D, SERVERNAME_D);
199 bulk.put(REGIONINFO_E, SERVERNAME_E);
200
201 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
202 "zkWatcher-BulkAssignTest", abortable, true);
203
204 Mockito.when(server.getConfiguration()).thenReturn(conf);
205 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
206 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
207
208 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
209 Mockito.when(serverManager.getOnlineServersList()).thenReturn(
210 new ArrayList<ServerName>(onlineServers.keySet()));
211
212 Mockito.when(serverManager.createDestinationServersList()).thenReturn(
213 new ArrayList<ServerName>(onlineServers.keySet()));
214 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
215 new ArrayList<ServerName>(onlineServers.keySet()));
216
217 for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
218 Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
219 Mockito.when(serverManager.sendRegionClose(entry.getValue(),
220 entry.getKey(), -1)).thenReturn(true);
221 Mockito.when(serverManager.sendRegionOpen(entry.getValue(),
222 entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);
223 Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
224 }
225
226 Mockito.when(master.getServerManager()).thenReturn(serverManager);
227
228 drainedServers.add(SERVERNAME_A);
229 drainedServers.add(SERVERNAME_B);
230 drainedServers.add(SERVERNAME_C);
231 drainedServers.add(SERVERNAME_D);
232
233 am = new AssignmentManager(server, serverManager, catalogTracker,
234 balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
235
236 Mockito.when(master.getAssignmentManager()).thenReturn(am);
237
238 zkWatcher.registerListener(am);
239
240 for (ServerName drained : drainedServers) {
241 addServerToDrainedList(drained, onlineServers, serverManager);
242 }
243
244 am.assign(bulk);
245
246 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
247 for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
248 setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(),
249 entry.getValue().getRegion());
250 }
251
252 am.waitForAssignment(REGIONINFO_A);
253 am.waitForAssignment(REGIONINFO_B);
254 am.waitForAssignment(REGIONINFO_C);
255 am.waitForAssignment(REGIONINFO_D);
256 am.waitForAssignment(REGIONINFO_E);
257
258 Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
259 for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
260 LOG.info("Region Assignment: "
261 + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
262 bunchServersAssigned.add(entry.getValue());
263 }
264
265 for (ServerName sn : drainedServers) {
266 assertFalse(bunchServersAssigned.contains(sn));
267 }
268 }
269
270 private void addServerToDrainedList(ServerName serverName,
271 Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
272 onlineServers.remove(serverName);
273 List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
274 Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
275 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
276 }
277
278 private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
279 HRegionInfo hregionInfo) throws Exception {
280 int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
281 int versionTransition = ZKAssign.transitionNode(zkWatcher,
282 hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
283 EventType.RS_ZK_REGION_OPENING, version);
284 ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
285 }
286
287 private ExecutorService startupMasterExecutor(final String name) {
288 ExecutorService executor = new ExecutorService(name);
289 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
290 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
291 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
292 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
293 return executor;
294 }
295 }