001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.ha;
020
021 import java.io.IOException;
022 import java.util.Arrays;
023 import java.util.List;
024 import java.util.concurrent.CountDownLatch;
025 import java.util.concurrent.TimeUnit;
026 import java.util.concurrent.locks.Lock;
027 import java.util.concurrent.locks.ReentrantLock;
028
029 import org.apache.commons.logging.Log;
030 import org.apache.commons.logging.LogFactory;
031 import org.apache.hadoop.HadoopIllegalArgumentException;
032 import org.apache.hadoop.classification.InterfaceAudience;
033 import org.apache.hadoop.classification.InterfaceStability;
034 import org.apache.hadoop.ha.HAZKUtil.ZKAuthInfo;
035 import org.apache.hadoop.util.StringUtils;
036 import org.apache.zookeeper.data.ACL;
037 import org.apache.zookeeper.KeeperException;
038 import org.apache.zookeeper.Watcher;
039 import org.apache.zookeeper.WatchedEvent;
040 import org.apache.zookeeper.Watcher.Event;
041 import org.apache.zookeeper.ZKUtil;
042 import org.apache.zookeeper.ZooKeeper;
043 import org.apache.zookeeper.CreateMode;
044 import org.apache.zookeeper.AsyncCallback.*;
045 import org.apache.zookeeper.data.Stat;
046 import org.apache.zookeeper.KeeperException.Code;
047
048 import com.google.common.annotations.VisibleForTesting;
049 import com.google.common.base.Preconditions;
050 import com.google.common.collect.Lists;
051
052 /**
053 *
054 * This class implements a simple library to perform leader election on top of
055 * Apache Zookeeper. Using Zookeeper as a coordination service, leader election
056 * can be performed by atomically creating an ephemeral lock file (znode) on
057 * Zookeeper. The service instance that successfully creates the znode becomes
058 * active and the rest become standbys. <br/>
059 * This election mechanism is only efficient for small number of election
060 * candidates (order of 10's) because contention on single znode by a large
061 * number of candidates can result in Zookeeper overload. <br/>
062 * The elector does not guarantee fencing (protection of shared resources) among
063 * service instances. After it has notified an instance about becoming a leader,
064 * then that instance must ensure that it meets the service consistency
065 * requirements. If it cannot do so, then it is recommended to quit the
066 * election. The application implements the {@link ActiveStandbyElectorCallback}
067 * to interact with the elector
068 */
069 @InterfaceAudience.Private
070 @InterfaceStability.Evolving
071 public class ActiveStandbyElector implements StatCallback, StringCallback {
072
073 /**
074 * Callback interface to interact with the ActiveStandbyElector object. <br/>
075 * The application will be notified with a callback only on state changes
076 * (i.e. there will never be successive calls to becomeActive without an
077 * intermediate call to enterNeutralMode). <br/>
078 * The callbacks will be running on Zookeeper client library threads. The
079 * application should return from these callbacks quickly so as not to impede
080 * Zookeeper client library performance and notifications. The app will
081 * typically remember the state change and return from the callback. It will
082 * then proceed with implementing actions around that state change. It is
083 * possible to be called back again while these actions are in flight and the
084 * app should handle this scenario.
085 */
086 public interface ActiveStandbyElectorCallback {
087 /**
088 * This method is called when the app becomes the active leader.
089 * If the service fails to become active, it should throw
090 * ServiceFailedException. This will cause the elector to
091 * sleep for a short period, then re-join the election.
092 *
093 * Callback implementations are expected to manage their own
094 * timeouts (e.g. when making an RPC to a remote node).
095 */
096 void becomeActive() throws ServiceFailedException;
097
098 /**
099 * This method is called when the app becomes a standby
100 */
101 void becomeStandby();
102
103 /**
104 * If the elector gets disconnected from Zookeeper and does not know about
105 * the lock state, then it will notify the service via the enterNeutralMode
106 * interface. The service may choose to ignore this or stop doing state
107 * changing operations. Upon reconnection, the elector verifies the leader
108 * status and calls back on the becomeActive and becomeStandby app
109 * interfaces. <br/>
110 * Zookeeper disconnects can happen due to network issues or loss of
111 * Zookeeper quorum. Thus enterNeutralMode can be used to guard against
112 * split-brain issues. In such situations it might be prudent to call
113 * becomeStandby too. However, such state change operations might be
114 * expensive and enterNeutralMode can help guard against doing that for
115 * transient issues.
116 */
117 void enterNeutralMode();
118
119 /**
120 * If there is any fatal error (e.g. wrong ACL's, unexpected Zookeeper
121 * errors or Zookeeper persistent unavailability) then notifyFatalError is
122 * called to notify the app about it.
123 */
124 void notifyFatalError(String errorMessage);
125
126 /**
127 * If an old active has failed, rather than exited gracefully, then
128 * the new active may need to take some fencing actions against it
129 * before proceeding with failover.
130 *
131 * @param oldActiveData the application data provided by the prior active
132 */
133 void fenceOldActive(byte[] oldActiveData);
134 }
135
136 /**
137 * Name of the lock znode used by the library. Protected for access in test
138 * classes
139 */
140 @VisibleForTesting
141 protected static final String LOCK_FILENAME = "ActiveStandbyElectorLock";
142 @VisibleForTesting
143 protected static final String BREADCRUMB_FILENAME = "ActiveBreadCrumb";
144
145 public static final Log LOG = LogFactory.getLog(ActiveStandbyElector.class);
146
147 static int NUM_RETRIES = 3;
148 private static final int SLEEP_AFTER_FAILURE_TO_BECOME_ACTIVE = 1000;
149
150 private static enum ConnectionState {
151 DISCONNECTED, CONNECTED, TERMINATED
152 };
153
154 static enum State {
155 INIT, ACTIVE, STANDBY, NEUTRAL
156 };
157
158 private State state = State.INIT;
159 private int createRetryCount = 0;
160 private int statRetryCount = 0;
161 private ZooKeeper zkClient;
162 private ConnectionState zkConnectionState = ConnectionState.TERMINATED;
163
164 private final ActiveStandbyElectorCallback appClient;
165 private final String zkHostPort;
166 private final int zkSessionTimeout;
167 private final List<ACL> zkAcl;
168 private final List<ZKAuthInfo> zkAuthInfo;
169 private byte[] appData;
170 private final String zkLockFilePath;
171 private final String zkBreadCrumbPath;
172 private final String znodeWorkingDir;
173
174 private Lock sessionReestablishLockForTests = new ReentrantLock();
175 private boolean wantToBeInElection;
176
177 /**
178 * Create a new ActiveStandbyElector object <br/>
179 * The elector is created by providing to it the Zookeeper configuration, the
180 * parent znode under which to create the znode and a reference to the
181 * callback interface. <br/>
182 * The parent znode name must be the same for all service instances and
183 * different across services. <br/>
184 * After the leader has been lost, a new leader will be elected after the
185 * session timeout expires. Hence, the app must set this parameter based on
186 * its needs for failure response time. The session timeout must be greater
187 * than the Zookeeper disconnect timeout and is recommended to be 3X that
188 * value to enable Zookeeper to retry transient disconnections. Setting a very
189 * short session timeout may result in frequent transitions between active and
190 * standby states during issues like network outages/GS pauses.
191 *
192 * @param zookeeperHostPorts
193 * ZooKeeper hostPort for all ZooKeeper servers
194 * @param zookeeperSessionTimeout
195 * ZooKeeper session timeout
196 * @param parentZnodeName
197 * znode under which to create the lock
198 * @param acl
199 * ZooKeeper ACL's
200 * @param authInfo a list of authentication credentials to add to the
201 * ZK connection
202 * @param app
203 * reference to callback interface object
204 * @throws IOException
205 * @throws HadoopIllegalArgumentException
206 */
207 public ActiveStandbyElector(String zookeeperHostPorts,
208 int zookeeperSessionTimeout, String parentZnodeName, List<ACL> acl,
209 List<ZKAuthInfo> authInfo,
210 ActiveStandbyElectorCallback app) throws IOException,
211 HadoopIllegalArgumentException, KeeperException {
212 if (app == null || acl == null || parentZnodeName == null
213 || zookeeperHostPorts == null || zookeeperSessionTimeout <= 0) {
214 throw new HadoopIllegalArgumentException("Invalid argument");
215 }
216 zkHostPort = zookeeperHostPorts;
217 zkSessionTimeout = zookeeperSessionTimeout;
218 zkAcl = acl;
219 zkAuthInfo = authInfo;
220 appClient = app;
221 znodeWorkingDir = parentZnodeName;
222 zkLockFilePath = znodeWorkingDir + "/" + LOCK_FILENAME;
223 zkBreadCrumbPath = znodeWorkingDir + "/" + BREADCRUMB_FILENAME;
224
225 // createConnection for future API calls
226 createConnection();
227 }
228
229 /**
230 * To participate in election, the app will call joinElection. The result will
231 * be notified by a callback on either the becomeActive or becomeStandby app
232 * interfaces. <br/>
233 * After this the elector will automatically monitor the leader status and
234 * perform re-election if necessary<br/>
235 * The app could potentially start off in standby mode and ignore the
236 * becomeStandby call.
237 *
238 * @param data
239 * to be set by the app. non-null data must be set.
240 * @throws HadoopIllegalArgumentException
241 * if valid data is not supplied
242 */
243 public synchronized void joinElection(byte[] data)
244 throws HadoopIllegalArgumentException {
245
246 if (data == null) {
247 throw new HadoopIllegalArgumentException("data cannot be null");
248 }
249
250 appData = new byte[data.length];
251 System.arraycopy(data, 0, appData, 0, data.length);
252
253 LOG.debug("Attempting active election for " + this);
254 joinElectionInternal();
255 }
256
257 /**
258 * @return true if the configured parent znode exists
259 */
260 public synchronized boolean parentZNodeExists()
261 throws IOException, InterruptedException {
262 Preconditions.checkState(zkClient != null);
263 try {
264 return zkClient.exists(znodeWorkingDir, false) != null;
265 } catch (KeeperException e) {
266 throw new IOException("Couldn't determine existence of znode '" +
267 znodeWorkingDir + "'", e);
268 }
269 }
270
271 /**
272 * Utility function to ensure that the configured base znode exists.
273 * This recursively creates the znode as well as all of its parents.
274 */
275 public synchronized void ensureParentZNode()
276 throws IOException, InterruptedException {
277 Preconditions.checkState(!wantToBeInElection,
278 "ensureParentZNode() may not be called while in the election");
279
280 String pathParts[] = znodeWorkingDir.split("/");
281 Preconditions.checkArgument(pathParts.length >= 1 &&
282 "".equals(pathParts[0]),
283 "Invalid path: %s", znodeWorkingDir);
284
285 StringBuilder sb = new StringBuilder();
286 for (int i = 1; i < pathParts.length; i++) {
287 sb.append("/").append(pathParts[i]);
288 String prefixPath = sb.toString();
289 LOG.debug("Ensuring existence of " + prefixPath);
290 try {
291 createWithRetries(prefixPath, new byte[]{}, zkAcl, CreateMode.PERSISTENT);
292 } catch (KeeperException e) {
293 if (isNodeExists(e.code())) {
294 // This is OK - just ensuring existence.
295 continue;
296 } else {
297 throw new IOException("Couldn't create " + prefixPath, e);
298 }
299 }
300 }
301
302 LOG.info("Successfully created " + znodeWorkingDir + " in ZK.");
303 }
304
305 /**
306 * Clear all of the state held within the parent ZNode.
307 * This recursively deletes everything within the znode as well as the
308 * parent znode itself. It should only be used when it's certain that
309 * no electors are currently participating in the election.
310 */
311 public synchronized void clearParentZNode()
312 throws IOException, InterruptedException {
313 Preconditions.checkState(!wantToBeInElection,
314 "clearParentZNode() may not be called while in the election");
315
316 try {
317 LOG.info("Recursively deleting " + znodeWorkingDir + " from ZK...");
318
319 zkDoWithRetries(new ZKAction<Void>() {
320 @Override
321 public Void run() throws KeeperException, InterruptedException {
322 ZKUtil.deleteRecursive(zkClient, znodeWorkingDir);
323 return null;
324 }
325 });
326 } catch (KeeperException e) {
327 throw new IOException("Couldn't clear parent znode " + znodeWorkingDir,
328 e);
329 }
330 LOG.info("Successfully deleted " + znodeWorkingDir + " from ZK.");
331 }
332
333
334 /**
335 * Any service instance can drop out of the election by calling quitElection.
336 * <br/>
337 * This will lose any leader status, if held, and stop monitoring of the lock
338 * node. <br/>
339 * If the instance wants to participate in election again, then it needs to
340 * call joinElection(). <br/>
341 * This allows service instances to take themselves out of rotation for known
342 * impending unavailable states (e.g. long GC pause or software upgrade).
343 *
344 * @param needFence true if the underlying daemon may need to be fenced
345 * if a failover occurs due to dropping out of the election.
346 */
347 public synchronized void quitElection(boolean needFence) {
348 LOG.info("Yielding from election");
349 if (!needFence && state == State.ACTIVE) {
350 // If active is gracefully going back to standby mode, remove
351 // our permanent znode so no one fences us.
352 tryDeleteOwnBreadCrumbNode();
353 }
354 reset();
355 wantToBeInElection = false;
356 }
357
358 /**
359 * Exception thrown when there is no active leader
360 */
361 public static class ActiveNotFoundException extends Exception {
362 private static final long serialVersionUID = 3505396722342846462L;
363 }
364
365 /**
366 * get data set by the active leader
367 *
368 * @return data set by the active instance
369 * @throws ActiveNotFoundException
370 * when there is no active leader
371 * @throws KeeperException
372 * other zookeeper operation errors
373 * @throws InterruptedException
374 * @throws IOException
375 * when ZooKeeper connection could not be established
376 */
377 public synchronized byte[] getActiveData() throws ActiveNotFoundException,
378 KeeperException, InterruptedException, IOException {
379 try {
380 if (zkClient == null) {
381 createConnection();
382 }
383 Stat stat = new Stat();
384 return getDataWithRetries(zkLockFilePath, false, stat);
385 } catch(KeeperException e) {
386 Code code = e.code();
387 if (isNodeDoesNotExist(code)) {
388 // handle the commonly expected cases that make sense for us
389 throw new ActiveNotFoundException();
390 } else {
391 throw e;
392 }
393 }
394 }
395
396 /**
397 * interface implementation of Zookeeper callback for create
398 */
399 @Override
400 public synchronized void processResult(int rc, String path, Object ctx,
401 String name) {
402 if (isStaleClient(ctx)) return;
403 LOG.debug("CreateNode result: " + rc + " for path: " + path
404 + " connectionState: " + zkConnectionState +
405 " for " + this);
406
407 Code code = Code.get(rc);
408 if (isSuccess(code)) {
409 // we successfully created the znode. we are the leader. start monitoring
410 if (becomeActive()) {
411 monitorActiveStatus();
412 } else {
413 reJoinElectionAfterFailureToBecomeActive();
414 }
415 return;
416 }
417
418 if (isNodeExists(code)) {
419 if (createRetryCount == 0) {
420 // znode exists and we did not retry the operation. so a different
421 // instance has created it. become standby and monitor lock.
422 becomeStandby();
423 }
424 // if we had retried then the znode could have been created by our first
425 // attempt to the server (that we lost) and this node exists response is
426 // for the second attempt. verify this case via ephemeral node owner. this
427 // will happen on the callback for monitoring the lock.
428 monitorActiveStatus();
429 return;
430 }
431
432 String errorMessage = "Received create error from Zookeeper. code:"
433 + code.toString() + " for path " + path;
434 LOG.debug(errorMessage);
435
436 if (shouldRetry(code)) {
437 if (createRetryCount < NUM_RETRIES) {
438 LOG.debug("Retrying createNode createRetryCount: " + createRetryCount);
439 ++createRetryCount;
440 createLockNodeAsync();
441 return;
442 }
443 errorMessage = errorMessage
444 + ". Not retrying further znode create connection errors.";
445 } else if (isSessionExpired(code)) {
446 // This isn't fatal - the client Watcher will re-join the election
447 LOG.warn("Lock acquisition failed because session was lost");
448 return;
449 }
450
451 fatalError(errorMessage);
452 }
453
454 /**
455 * interface implementation of Zookeeper callback for monitor (exists)
456 */
457 @Override
458 public synchronized void processResult(int rc, String path, Object ctx,
459 Stat stat) {
460 if (isStaleClient(ctx)) return;
461
462 assert wantToBeInElection :
463 "Got a StatNode result after quitting election";
464
465 LOG.debug("StatNode result: " + rc + " for path: " + path
466 + " connectionState: " + zkConnectionState + " for " + this);
467
468
469 Code code = Code.get(rc);
470 if (isSuccess(code)) {
471 // the following owner check completes verification in case the lock znode
472 // creation was retried
473 if (stat.getEphemeralOwner() == zkClient.getSessionId()) {
474 // we own the lock znode. so we are the leader
475 if (!becomeActive()) {
476 reJoinElectionAfterFailureToBecomeActive();
477 }
478 } else {
479 // we dont own the lock znode. so we are a standby.
480 becomeStandby();
481 }
482 // the watch set by us will notify about changes
483 return;
484 }
485
486 if (isNodeDoesNotExist(code)) {
487 // the lock znode disappeared before we started monitoring it
488 enterNeutralMode();
489 joinElectionInternal();
490 return;
491 }
492
493 String errorMessage = "Received stat error from Zookeeper. code:"
494 + code.toString();
495 LOG.debug(errorMessage);
496
497 if (shouldRetry(code)) {
498 if (statRetryCount < NUM_RETRIES) {
499 ++statRetryCount;
500 monitorLockNodeAsync();
501 return;
502 }
503 errorMessage = errorMessage
504 + ". Not retrying further znode monitoring connection errors.";
505 } else if (isSessionExpired(code)) {
506 // This isn't fatal - the client Watcher will re-join the election
507 LOG.warn("Lock monitoring failed because session was lost");
508 return;
509 }
510
511 fatalError(errorMessage);
512 }
513
514 /**
515 * We failed to become active. Re-join the election, but
516 * sleep for a few seconds after terminating our existing
517 * session, so that other nodes have a chance to become active.
518 * The failure to become active is already logged inside
519 * becomeActive().
520 */
521 private void reJoinElectionAfterFailureToBecomeActive() {
522 reJoinElection(SLEEP_AFTER_FAILURE_TO_BECOME_ACTIVE);
523 }
524
525 /**
526 * interface implementation of Zookeeper watch events (connection and node),
527 * proxied by {@link WatcherWithClientRef}.
528 */
529 synchronized void processWatchEvent(ZooKeeper zk, WatchedEvent event) {
530 Event.EventType eventType = event.getType();
531 if (isStaleClient(zk)) return;
532 LOG.debug("Watcher event type: " + eventType + " with state:"
533 + event.getState() + " for path:" + event.getPath()
534 + " connectionState: " + zkConnectionState
535 + " for " + this);
536
537 if (eventType == Event.EventType.None) {
538 // the connection state has changed
539 switch (event.getState()) {
540 case SyncConnected:
541 LOG.info("Session connected.");
542 // if the listener was asked to move to safe state then it needs to
543 // be undone
544 ConnectionState prevConnectionState = zkConnectionState;
545 zkConnectionState = ConnectionState.CONNECTED;
546 if (prevConnectionState == ConnectionState.DISCONNECTED &&
547 wantToBeInElection) {
548 monitorActiveStatus();
549 }
550 break;
551 case Disconnected:
552 LOG.info("Session disconnected. Entering neutral mode...");
553
554 // ask the app to move to safe state because zookeeper connection
555 // is not active and we dont know our state
556 zkConnectionState = ConnectionState.DISCONNECTED;
557 enterNeutralMode();
558 break;
559 case Expired:
560 // the connection got terminated because of session timeout
561 // call listener to reconnect
562 LOG.info("Session expired. Entering neutral mode and rejoining...");
563 enterNeutralMode();
564 reJoinElection(0);
565 break;
566 default:
567 fatalError("Unexpected Zookeeper watch event state: "
568 + event.getState());
569 break;
570 }
571
572 return;
573 }
574
575 // a watch on lock path in zookeeper has fired. so something has changed on
576 // the lock. ideally we should check that the path is the same as the lock
577 // path but trusting zookeeper for now
578 String path = event.getPath();
579 if (path != null) {
580 switch (eventType) {
581 case NodeDeleted:
582 if (state == State.ACTIVE) {
583 enterNeutralMode();
584 }
585 joinElectionInternal();
586 break;
587 case NodeDataChanged:
588 monitorActiveStatus();
589 break;
590 default:
591 LOG.debug("Unexpected node event: " + eventType + " for path: " + path);
592 monitorActiveStatus();
593 }
594
595 return;
596 }
597
598 // some unexpected error has occurred
599 fatalError("Unexpected watch error from Zookeeper");
600 }
601
602 /**
603 * Get a new zookeeper client instance. protected so that test class can
604 * inherit and pass in a mock object for zookeeper
605 *
606 * @return new zookeeper client instance
607 * @throws IOException
608 * @throws KeeperException zookeeper connectionloss exception
609 */
610 protected synchronized ZooKeeper getNewZooKeeper() throws IOException,
611 KeeperException {
612
613 // Unfortunately, the ZooKeeper constructor connects to ZooKeeper and
614 // may trigger the Connected event immediately. So, if we register the
615 // watcher after constructing ZooKeeper, we may miss that event. Instead,
616 // we construct the watcher first, and have it block any events it receives
617 // before we can set its ZooKeeper reference.
618 WatcherWithClientRef watcher = new WatcherWithClientRef();
619 ZooKeeper zk = new ZooKeeper(zkHostPort, zkSessionTimeout, watcher);
620 watcher.setZooKeeperRef(zk);
621
622 // Wait for the asynchronous success/failure. This may throw an exception
623 // if we don't connect within the session timeout.
624 watcher.waitForZKConnectionEvent(zkSessionTimeout);
625
626 for (ZKAuthInfo auth : zkAuthInfo) {
627 zk.addAuthInfo(auth.getScheme(), auth.getAuth());
628 }
629 return zk;
630 }
631
632 private void fatalError(String errorMessage) {
633 LOG.fatal(errorMessage);
634 reset();
635 appClient.notifyFatalError(errorMessage);
636 }
637
638 private void monitorActiveStatus() {
639 assert wantToBeInElection;
640 LOG.debug("Monitoring active leader for " + this);
641 statRetryCount = 0;
642 monitorLockNodeAsync();
643 }
644
645 private void joinElectionInternal() {
646 if (zkClient == null) {
647 if (!reEstablishSession()) {
648 fatalError("Failed to reEstablish connection with ZooKeeper");
649 return;
650 }
651 }
652
653 createRetryCount = 0;
654 wantToBeInElection = true;
655 createLockNodeAsync();
656 }
657
658 private void reJoinElection(int sleepTime) {
659 LOG.info("Trying to re-establish ZK session");
660
661 // Some of the test cases rely on expiring the ZK sessions and
662 // ensuring that the other node takes over. But, there's a race
663 // where the original lease holder could reconnect faster than the other
664 // thread manages to take the lock itself. This lock allows the
665 // tests to block the reconnection. It's a shame that this leaked
666 // into non-test code, but the lock is only acquired here so will never
667 // be contended.
668 sessionReestablishLockForTests.lock();
669 try {
670 terminateConnection();
671 sleepFor(sleepTime);
672
673 joinElectionInternal();
674 } finally {
675 sessionReestablishLockForTests.unlock();
676 }
677 }
678
679 /**
680 * Sleep for the given number of milliseconds.
681 * This is non-static, and separated out, so that unit tests
682 * can override the behavior not to sleep.
683 */
684 @VisibleForTesting
685 protected void sleepFor(int sleepMs) {
686 if (sleepMs > 0) {
687 try {
688 Thread.sleep(sleepMs);
689 } catch (InterruptedException e) {
690 Thread.currentThread().interrupt();
691 }
692 }
693 }
694
695 @VisibleForTesting
696 void preventSessionReestablishmentForTests() {
697 sessionReestablishLockForTests.lock();
698 }
699
700 @VisibleForTesting
701 void allowSessionReestablishmentForTests() {
702 sessionReestablishLockForTests.unlock();
703 }
704
705 @VisibleForTesting
706 synchronized long getZKSessionIdForTests() {
707 if (zkClient != null) {
708 return zkClient.getSessionId();
709 } else {
710 return -1;
711 }
712 }
713
714 @VisibleForTesting
715 synchronized State getStateForTests() {
716 return state;
717 }
718
719 private boolean reEstablishSession() {
720 int connectionRetryCount = 0;
721 boolean success = false;
722 while(!success && connectionRetryCount < NUM_RETRIES) {
723 LOG.debug("Establishing zookeeper connection for " + this);
724 try {
725 createConnection();
726 success = true;
727 } catch(IOException e) {
728 LOG.warn(e);
729 sleepFor(5000);
730 } catch(KeeperException e) {
731 LOG.warn(e);
732 sleepFor(5000);
733 }
734 ++connectionRetryCount;
735 }
736 return success;
737 }
738
739 private void createConnection() throws IOException, KeeperException {
740 if (zkClient != null) {
741 try {
742 zkClient.close();
743 } catch (InterruptedException e) {
744 throw new IOException("Interrupted while closing ZK",
745 e);
746 }
747 zkClient = null;
748 }
749 zkClient = getNewZooKeeper();
750 LOG.debug("Created new connection for " + this);
751 }
752
753 void terminateConnection() {
754 if (zkClient == null) {
755 return;
756 }
757 LOG.debug("Terminating ZK connection for " + this);
758 ZooKeeper tempZk = zkClient;
759 zkClient = null;
760 try {
761 tempZk.close();
762 } catch(InterruptedException e) {
763 LOG.warn(e);
764 }
765 zkConnectionState = ConnectionState.TERMINATED;
766 }
767
768 private void reset() {
769 state = State.INIT;
770 terminateConnection();
771 }
772
773 private boolean becomeActive() {
774 assert wantToBeInElection;
775 if (state == State.ACTIVE) {
776 // already active
777 return true;
778 }
779 try {
780 Stat oldBreadcrumbStat = fenceOldActive();
781 writeBreadCrumbNode(oldBreadcrumbStat);
782
783 LOG.debug("Becoming active for " + this);
784 appClient.becomeActive();
785 state = State.ACTIVE;
786 return true;
787 } catch (Exception e) {
788 LOG.warn("Exception handling the winning of election", e);
789 // Caller will handle quitting and rejoining the election.
790 return false;
791 }
792 }
793
794 /**
795 * Write the "ActiveBreadCrumb" node, indicating that this node may need
796 * to be fenced on failover.
797 * @param oldBreadcrumbStat
798 */
799 private void writeBreadCrumbNode(Stat oldBreadcrumbStat)
800 throws KeeperException, InterruptedException {
801 LOG.info("Writing znode " + zkBreadCrumbPath +
802 " to indicate that the local node is the most recent active...");
803 if (oldBreadcrumbStat == null) {
804 // No previous active, just create the node
805 createWithRetries(zkBreadCrumbPath, appData, zkAcl,
806 CreateMode.PERSISTENT);
807 } else {
808 // There was a previous active, update the node
809 setDataWithRetries(zkBreadCrumbPath, appData, oldBreadcrumbStat.getVersion());
810 }
811 }
812
813 /**
814 * Try to delete the "ActiveBreadCrumb" node when gracefully giving up
815 * active status.
816 * If this fails, it will simply warn, since the graceful release behavior
817 * is only an optimization.
818 */
819 private void tryDeleteOwnBreadCrumbNode() {
820 assert state == State.ACTIVE;
821 LOG.info("Deleting bread-crumb of active node...");
822
823 // Sanity check the data. This shouldn't be strictly necessary,
824 // but better to play it safe.
825 Stat stat = new Stat();
826 byte[] data = null;
827 try {
828 data = zkClient.getData(zkBreadCrumbPath, false, stat);
829
830 if (!Arrays.equals(data, appData)) {
831 throw new IllegalStateException(
832 "We thought we were active, but in fact " +
833 "the active znode had the wrong data: " +
834 StringUtils.byteToHexString(data) + " (stat=" + stat + ")");
835 }
836
837 deleteWithRetries(zkBreadCrumbPath, stat.getVersion());
838 } catch (Exception e) {
839 LOG.warn("Unable to delete our own bread-crumb of being active at " +
840 zkBreadCrumbPath + ": " + e.getLocalizedMessage() + ". " +
841 "Expecting to be fenced by the next active.");
842 }
843 }
844
845 /**
846 * If there is a breadcrumb node indicating that another node may need
847 * fencing, try to fence that node.
848 * @return the Stat of the breadcrumb node that was read, or null
849 * if no breadcrumb node existed
850 */
851 private Stat fenceOldActive() throws InterruptedException, KeeperException {
852 final Stat stat = new Stat();
853 byte[] data;
854 LOG.info("Checking for any old active which needs to be fenced...");
855 try {
856 data = zkDoWithRetries(new ZKAction<byte[]>() {
857 @Override
858 public byte[] run() throws KeeperException, InterruptedException {
859 return zkClient.getData(zkBreadCrumbPath, false, stat);
860 }
861 });
862 } catch (KeeperException ke) {
863 if (isNodeDoesNotExist(ke.code())) {
864 LOG.info("No old node to fence");
865 return null;
866 }
867
868 // If we failed to read for any other reason, then likely we lost
869 // our session, or we don't have permissions, etc. In any case,
870 // we probably shouldn't become active, and failing the whole
871 // thing is the best bet.
872 throw ke;
873 }
874
875 LOG.info("Old node exists: " + StringUtils.byteToHexString(data));
876 if (Arrays.equals(data, appData)) {
877 LOG.info("But old node has our own data, so don't need to fence it.");
878 } else {
879 appClient.fenceOldActive(data);
880 }
881 return stat;
882 }
883
884 private void becomeStandby() {
885 if (state != State.STANDBY) {
886 LOG.debug("Becoming standby for " + this);
887 state = State.STANDBY;
888 appClient.becomeStandby();
889 }
890 }
891
892 private void enterNeutralMode() {
893 if (state != State.NEUTRAL) {
894 LOG.debug("Entering neutral mode for " + this);
895 state = State.NEUTRAL;
896 appClient.enterNeutralMode();
897 }
898 }
899
900 private void createLockNodeAsync() {
901 zkClient.create(zkLockFilePath, appData, zkAcl, CreateMode.EPHEMERAL,
902 this, zkClient);
903 }
904
905 private void monitorLockNodeAsync() {
906 zkClient.exists(zkLockFilePath,
907 new WatcherWithClientRef(zkClient), this,
908 zkClient);
909 }
910
911 private String createWithRetries(final String path, final byte[] data,
912 final List<ACL> acl, final CreateMode mode)
913 throws InterruptedException, KeeperException {
914 return zkDoWithRetries(new ZKAction<String>() {
915 @Override
916 public String run() throws KeeperException, InterruptedException {
917 return zkClient.create(path, data, acl, mode);
918 }
919 });
920 }
921
922 private byte[] getDataWithRetries(final String path, final boolean watch,
923 final Stat stat) throws InterruptedException, KeeperException {
924 return zkDoWithRetries(new ZKAction<byte[]>() {
925 @Override
926 public byte[] run() throws KeeperException, InterruptedException {
927 return zkClient.getData(path, watch, stat);
928 }
929 });
930 }
931
932 private Stat setDataWithRetries(final String path, final byte[] data,
933 final int version) throws InterruptedException, KeeperException {
934 return zkDoWithRetries(new ZKAction<Stat>() {
935 @Override
936 public Stat run() throws KeeperException, InterruptedException {
937 return zkClient.setData(path, data, version);
938 }
939 });
940 }
941
942 private void deleteWithRetries(final String path, final int version)
943 throws KeeperException, InterruptedException {
944 zkDoWithRetries(new ZKAction<Void>() {
945 @Override
946 public Void run() throws KeeperException, InterruptedException {
947 zkClient.delete(path, version);
948 return null;
949 }
950 });
951 }
952
953 private static <T> T zkDoWithRetries(ZKAction<T> action)
954 throws KeeperException, InterruptedException {
955 int retry = 0;
956 while (true) {
957 try {
958 return action.run();
959 } catch (KeeperException ke) {
960 if (shouldRetry(ke.code()) && ++retry < NUM_RETRIES) {
961 continue;
962 }
963 throw ke;
964 }
965 }
966 }
967
968 private interface ZKAction<T> {
969 T run() throws KeeperException, InterruptedException;
970 }
971
972 /**
973 * The callbacks and watchers pass a reference to the ZK client
974 * which made the original call. We don't want to take action
975 * based on any callbacks from prior clients after we quit
976 * the election.
977 * @param ctx the ZK client passed into the watcher
978 * @return true if it matches the current client
979 */
980 private synchronized boolean isStaleClient(Object ctx) {
981 Preconditions.checkNotNull(ctx);
982 if (zkClient != (ZooKeeper)ctx) {
983 LOG.warn("Ignoring stale result from old client with sessionId " +
984 String.format("0x%08x", ((ZooKeeper)ctx).getSessionId()));
985 return true;
986 }
987 return false;
988 }
989
990 /**
991 * Watcher implementation which keeps a reference around to the
992 * original ZK connection, and passes it back along with any
993 * events.
994 */
995 private final class WatcherWithClientRef implements Watcher {
996 private ZooKeeper zk;
997
998 /**
999 * Latch fired whenever any event arrives. This is used in order
1000 * to wait for the Connected event when the client is first created.
1001 */
1002 private CountDownLatch hasReceivedEvent = new CountDownLatch(1);
1003
1004 /**
1005 * Latch used to wait until the reference to ZooKeeper is set.
1006 */
1007 private CountDownLatch hasSetZooKeeper = new CountDownLatch(1);
1008
1009 private WatcherWithClientRef() {
1010 }
1011
1012 private WatcherWithClientRef(ZooKeeper zk) {
1013 setZooKeeperRef(zk);
1014 }
1015
1016 /**
1017 * Waits for the next event from ZooKeeper to arrive.
1018 *
1019 * @param connectionTimeoutMs zookeeper connection timeout in milliseconds
1020 * @throws KeeperException if the connection attempt times out. This will
1021 * be a ZooKeeper ConnectionLoss exception code.
1022 * @throws IOException if interrupted while connecting to ZooKeeper
1023 */
1024 private void waitForZKConnectionEvent(int connectionTimeoutMs)
1025 throws KeeperException, IOException {
1026 try {
1027 if (!hasReceivedEvent.await(connectionTimeoutMs, TimeUnit.MILLISECONDS)) {
1028 LOG.error("Connection timed out: couldn't connect to ZooKeeper in "
1029 + connectionTimeoutMs + " milliseconds");
1030 zk.close();
1031 throw KeeperException.create(Code.CONNECTIONLOSS);
1032 }
1033 } catch (InterruptedException e) {
1034 Thread.currentThread().interrupt();
1035 throw new IOException(
1036 "Interrupted when connecting to zookeeper server", e);
1037 }
1038 }
1039
1040 private void setZooKeeperRef(ZooKeeper zk) {
1041 Preconditions.checkState(this.zk == null,
1042 "zk already set -- must be set exactly once");
1043 this.zk = zk;
1044 hasSetZooKeeper.countDown();
1045 }
1046
1047 @Override
1048 public void process(WatchedEvent event) {
1049 hasReceivedEvent.countDown();
1050 try {
1051 hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
1052 ActiveStandbyElector.this.processWatchEvent(
1053 zk, event);
1054 } catch (Throwable t) {
1055 fatalError(
1056 "Failed to process watcher event " + event + ": " +
1057 StringUtils.stringifyException(t));
1058 }
1059 }
1060 }
1061
1062 private static boolean isSuccess(Code code) {
1063 return (code == Code.OK);
1064 }
1065
1066 private static boolean isNodeExists(Code code) {
1067 return (code == Code.NODEEXISTS);
1068 }
1069
1070 private static boolean isNodeDoesNotExist(Code code) {
1071 return (code == Code.NONODE);
1072 }
1073
1074 private static boolean isSessionExpired(Code code) {
1075 return (code == Code.SESSIONEXPIRED);
1076 }
1077
1078 private static boolean shouldRetry(Code code) {
1079 switch (code) {
1080 case CONNECTIONLOSS:
1081 case OPERATIONTIMEOUT:
1082 return true;
1083 }
1084 return false;
1085 }
1086
1087 @Override
1088 public String toString() {
1089 return "elector id=" + System.identityHashCode(this) +
1090 " appData=" +
1091 ((appData == null) ? "null" : StringUtils.byteToHexString(appData)) +
1092 " cb=" + appClient;
1093 }
1094 }