View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.catalog.CatalogTracker;
25  import org.apache.hadoop.hbase.executor.EventType;
26  import org.apache.hadoop.hbase.executor.ExecutorService;
27  import org.apache.hadoop.hbase.executor.ExecutorType;
28  import org.apache.hadoop.hbase.master.AssignmentManager;
29  import org.apache.hadoop.hbase.master.HMaster;
30  import org.apache.hadoop.hbase.master.LoadBalancer;
31  import org.apache.hadoop.hbase.master.RegionPlan;
32  import org.apache.hadoop.hbase.master.RegionState;
33  import org.apache.hadoop.hbase.master.ServerManager;
34  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
35  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
36  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38  import org.junit.AfterClass;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  import org.mockito.Mockito;
43  
44  import java.util.ArrayList;
45  import java.util.HashMap;
46  import java.util.HashSet;
47  import java.util.List;
48  import java.util.Map;
49  import java.util.Map.Entry;
50  import java.util.Set;
51  
52  import static org.junit.Assert.assertFalse;
53  import static org.junit.Assert.assertNotEquals;
54  import static org.junit.Assert.assertTrue;
55  
56  
57  /**
58   * Test the draining servers feature.
59   */
60  @Category(MediumTests.class)
61  public class TestDrainingServer {
62    private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
63    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
64    private Abortable abortable = new Abortable() {
65      @Override
66      public boolean isAborted() {
67        return false;
68      }
69  
70      @Override
71      public void abort(String why, Throwable e) {
72      }
73    };
74  
75    @AfterClass
76    public static void afterClass() throws Exception {
77      TEST_UTIL.shutdownMiniZKCluster();
78    }
79    
80    @BeforeClass
81    public static void beforeClass() throws Exception {
82      TEST_UTIL.startMiniZKCluster();
83    }
84  
85    @Test
86    public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
87      AssignmentManager am;
88      Configuration conf = TEST_UTIL.getConfiguration();
89      final HMaster master = Mockito.mock(HMaster.class);
90      final Server server = Mockito.mock(Server.class);
91      final ServerManager serverManager = Mockito.mock(ServerManager.class);
92      final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
93      final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
94      LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
95      CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
96      final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
97          HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
98  
99      ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
100       "zkWatcher-Test", abortable, true);
101 
102     Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
103 
104     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
105     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
106 
107     Mockito.when(server.getConfiguration()).thenReturn(conf);
108     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
109     Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
110 
111     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
112     Mockito.when(serverManager.getOnlineServersList())
113     .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
114     
115     Mockito.when(serverManager.createDestinationServersList())
116         .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
117     Mockito.when(serverManager.createDestinationServersList(null))
118         .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
119     
120     for (ServerName sn : onlineServers.keySet()) {
121       Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
122       Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
123       Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
124       Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
125       .thenReturn(RegionOpeningState.OPENED);
126       Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
127       .thenReturn(RegionOpeningState.OPENED);
128       Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
129     }
130 
131     Mockito.when(master.getServerManager()).thenReturn(serverManager);
132 
133     am = new AssignmentManager(server, serverManager, catalogTracker,
134         balancer, startupMasterExecutor("mockExecutorService"), null, null);
135 
136     Mockito.when(master.getAssignmentManager()).thenReturn(am);
137     Mockito.when(master.getZooKeeperWatcher()).thenReturn(zkWatcher);
138     Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
139     
140     am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
141 
142     zkWatcher.registerListenerFirst(am);
143 
144     addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
145 
146     am.assign(REGIONINFO, true);
147 
148     setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
149     setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
150 
151     am.waitForAssignment(REGIONINFO);
152 
153     assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
154     assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
155   }
156 
157   @Test
158   public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
159     Configuration conf = TEST_UTIL.getConfiguration();
160     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
161     CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
162     AssignmentManager am;
163     final HMaster master = Mockito.mock(HMaster.class);
164     final Server server = Mockito.mock(Server.class);
165     final ServerManager serverManager = Mockito.mock(ServerManager.class);
166     final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
167     final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
168     final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
169     final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
170     final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
171     final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
172 
173     Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
174     
175     HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
176         HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
177     HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
178       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
179     HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
180       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
181     HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
182       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
183     HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
184       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
185 
186     Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
187     List<ServerName> drainedServers = new ArrayList<ServerName>();
188 
189     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
190     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
191     onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
192     onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
193     onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
194 
195     bulk.put(REGIONINFO_A, SERVERNAME_A);
196     bulk.put(REGIONINFO_B, SERVERNAME_B);
197     bulk.put(REGIONINFO_C, SERVERNAME_C);
198     bulk.put(REGIONINFO_D, SERVERNAME_D);
199     bulk.put(REGIONINFO_E, SERVERNAME_E);
200 
201     ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
202         "zkWatcher-BulkAssignTest", abortable, true);
203 
204     Mockito.when(server.getConfiguration()).thenReturn(conf);
205     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
206     Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
207 
208     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
209     Mockito.when(serverManager.getOnlineServersList()).thenReturn(
210       new ArrayList<ServerName>(onlineServers.keySet()));
211     
212     Mockito.when(serverManager.createDestinationServersList()).thenReturn(
213       new ArrayList<ServerName>(onlineServers.keySet()));
214     Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
215       new ArrayList<ServerName>(onlineServers.keySet()));
216     
217     for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
218       Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
219       Mockito.when(serverManager.sendRegionClose(entry.getValue(), 
220         entry.getKey(), -1)).thenReturn(true);
221       Mockito.when(serverManager.sendRegionOpen(entry.getValue(), 
222         entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);  
223       Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
224     }
225     
226     Mockito.when(master.getServerManager()).thenReturn(serverManager);
227 
228     drainedServers.add(SERVERNAME_A);
229     drainedServers.add(SERVERNAME_B);
230     drainedServers.add(SERVERNAME_C);
231     drainedServers.add(SERVERNAME_D);
232 
233     am = new AssignmentManager(server, serverManager, catalogTracker,
234       balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
235     
236     Mockito.when(master.getAssignmentManager()).thenReturn(am);
237 
238     zkWatcher.registerListener(am);
239     
240     for (ServerName drained : drainedServers) {
241       addServerToDrainedList(drained, onlineServers, serverManager);
242     }
243     
244     am.assign(bulk);
245 
246     Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
247     for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
248       setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(), 
249         entry.getValue().getRegion());
250     }
251     
252     am.waitForAssignment(REGIONINFO_A);
253     am.waitForAssignment(REGIONINFO_B);
254     am.waitForAssignment(REGIONINFO_C);
255     am.waitForAssignment(REGIONINFO_D);
256     am.waitForAssignment(REGIONINFO_E);
257     
258     Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
259     for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
260       LOG.info("Region Assignment: " 
261           + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
262       bunchServersAssigned.add(entry.getValue());
263     }
264     
265     for (ServerName sn : drainedServers) {
266       assertFalse(bunchServersAssigned.contains(sn));
267     }
268   }
269 
270   private void addServerToDrainedList(ServerName serverName, 
271       Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
272     onlineServers.remove(serverName);
273     List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
274     Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
275     Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
276   }
277 
278   private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
279                                    HRegionInfo hregionInfo) throws Exception {
280     int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
281     int versionTransition = ZKAssign.transitionNode(zkWatcher,
282         hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
283         EventType.RS_ZK_REGION_OPENING, version);
284     ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
285   }
286 
287   private ExecutorService startupMasterExecutor(final String name) {
288     ExecutorService executor = new ExecutorService(name);
289     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
290     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
291     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
292     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
293     return executor;
294   }
295 }