001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.ha;
020    
021    import java.io.IOException;
022    import java.util.Arrays;
023    import java.util.List;
024    import java.util.concurrent.CountDownLatch;
025    import java.util.concurrent.TimeUnit;
026    import java.util.concurrent.locks.Lock;
027    import java.util.concurrent.locks.ReentrantLock;
028    
029    import org.apache.commons.logging.Log;
030    import org.apache.commons.logging.LogFactory;
031    import org.apache.hadoop.HadoopIllegalArgumentException;
032    import org.apache.hadoop.classification.InterfaceAudience;
033    import org.apache.hadoop.classification.InterfaceStability;
034    import org.apache.hadoop.ha.HAZKUtil.ZKAuthInfo;
035    import org.apache.hadoop.util.StringUtils;
036    import org.apache.zookeeper.data.ACL;
037    import org.apache.zookeeper.KeeperException;
038    import org.apache.zookeeper.Watcher;
039    import org.apache.zookeeper.WatchedEvent;
040    import org.apache.zookeeper.Watcher.Event;
041    import org.apache.zookeeper.ZKUtil;
042    import org.apache.zookeeper.ZooKeeper;
043    import org.apache.zookeeper.CreateMode;
044    import org.apache.zookeeper.AsyncCallback.*;
045    import org.apache.zookeeper.data.Stat;
046    import org.apache.zookeeper.KeeperException.Code;
047    
048    import com.google.common.annotations.VisibleForTesting;
049    import com.google.common.base.Preconditions;
050    import com.google.common.collect.Lists;
051    
052    /**
053     * 
054     * This class implements a simple library to perform leader election on top of
055     * Apache Zookeeper. Using Zookeeper as a coordination service, leader election
056     * can be performed by atomically creating an ephemeral lock file (znode) on
057     * Zookeeper. The service instance that successfully creates the znode becomes
058     * active and the rest become standbys. <br/>
059     * This election mechanism is only efficient for small number of election
060     * candidates (order of 10's) because contention on single znode by a large
061     * number of candidates can result in Zookeeper overload. <br/>
062     * The elector does not guarantee fencing (protection of shared resources) among
063     * service instances. After it has notified an instance about becoming a leader,
064     * then that instance must ensure that it meets the service consistency
065     * requirements. If it cannot do so, then it is recommended to quit the
066     * election. The application implements the {@link ActiveStandbyElectorCallback}
067     * to interact with the elector
068     */
069    @InterfaceAudience.Private
070    @InterfaceStability.Evolving
071    public class ActiveStandbyElector implements StatCallback, StringCallback {
072    
073      /**
074       * Callback interface to interact with the ActiveStandbyElector object. <br/>
075       * The application will be notified with a callback only on state changes
076       * (i.e. there will never be successive calls to becomeActive without an
077       * intermediate call to enterNeutralMode). <br/>
078       * The callbacks will be running on Zookeeper client library threads. The
079       * application should return from these callbacks quickly so as not to impede
080       * Zookeeper client library performance and notifications. The app will
081       * typically remember the state change and return from the callback. It will
082       * then proceed with implementing actions around that state change. It is
083       * possible to be called back again while these actions are in flight and the
084       * app should handle this scenario.
085       */
086      public interface ActiveStandbyElectorCallback {
087        /**
088         * This method is called when the app becomes the active leader.
089         * If the service fails to become active, it should throw
090         * ServiceFailedException. This will cause the elector to
091         * sleep for a short period, then re-join the election.
092         * 
093         * Callback implementations are expected to manage their own
094         * timeouts (e.g. when making an RPC to a remote node).
095         */
096        void becomeActive() throws ServiceFailedException;
097    
098        /**
099         * This method is called when the app becomes a standby
100         */
101        void becomeStandby();
102    
103        /**
104         * If the elector gets disconnected from Zookeeper and does not know about
105         * the lock state, then it will notify the service via the enterNeutralMode
106         * interface. The service may choose to ignore this or stop doing state
107         * changing operations. Upon reconnection, the elector verifies the leader
108         * status and calls back on the becomeActive and becomeStandby app
109         * interfaces. <br/>
110         * Zookeeper disconnects can happen due to network issues or loss of
111         * Zookeeper quorum. Thus enterNeutralMode can be used to guard against
112         * split-brain issues. In such situations it might be prudent to call
113         * becomeStandby too. However, such state change operations might be
114         * expensive and enterNeutralMode can help guard against doing that for
115         * transient issues.
116         */
117        void enterNeutralMode();
118    
119        /**
120         * If there is any fatal error (e.g. wrong ACL's, unexpected Zookeeper
121         * errors or Zookeeper persistent unavailability) then notifyFatalError is
122         * called to notify the app about it.
123         */
124        void notifyFatalError(String errorMessage);
125    
126        /**
127         * If an old active has failed, rather than exited gracefully, then
128         * the new active may need to take some fencing actions against it
129         * before proceeding with failover.
130         * 
131         * @param oldActiveData the application data provided by the prior active
132         */
133        void fenceOldActive(byte[] oldActiveData);
134      }
135    
136      /**
137       * Name of the lock znode used by the library. Protected for access in test
138       * classes
139       */
140      @VisibleForTesting
141      protected static final String LOCK_FILENAME = "ActiveStandbyElectorLock";
142      @VisibleForTesting
143      protected static final String BREADCRUMB_FILENAME = "ActiveBreadCrumb";
144    
145      public static final Log LOG = LogFactory.getLog(ActiveStandbyElector.class);
146    
147      static int NUM_RETRIES = 3;
148      private static final int SLEEP_AFTER_FAILURE_TO_BECOME_ACTIVE = 1000;
149    
150      private static enum ConnectionState {
151        DISCONNECTED, CONNECTED, TERMINATED
152      };
153    
154      static enum State {
155        INIT, ACTIVE, STANDBY, NEUTRAL
156      };
157    
158      private State state = State.INIT;
159      private int createRetryCount = 0;
160      private int statRetryCount = 0;
161      private ZooKeeper zkClient;
162      private ConnectionState zkConnectionState = ConnectionState.TERMINATED;
163    
164      private final ActiveStandbyElectorCallback appClient;
165      private final String zkHostPort;
166      private final int zkSessionTimeout;
167      private final List<ACL> zkAcl;
168      private final List<ZKAuthInfo> zkAuthInfo;
169      private byte[] appData;
170      private final String zkLockFilePath;
171      private final String zkBreadCrumbPath;
172      private final String znodeWorkingDir;
173    
174      private Lock sessionReestablishLockForTests = new ReentrantLock();
175      private boolean wantToBeInElection;
176      
177      /**
178       * Create a new ActiveStandbyElector object <br/>
179       * The elector is created by providing to it the Zookeeper configuration, the
180       * parent znode under which to create the znode and a reference to the
181       * callback interface. <br/>
182       * The parent znode name must be the same for all service instances and
183       * different across services. <br/>
184       * After the leader has been lost, a new leader will be elected after the
185       * session timeout expires. Hence, the app must set this parameter based on
186       * its needs for failure response time. The session timeout must be greater
187       * than the Zookeeper disconnect timeout and is recommended to be 3X that
188       * value to enable Zookeeper to retry transient disconnections. Setting a very
189       * short session timeout may result in frequent transitions between active and
190       * standby states during issues like network outages/GS pauses.
191       * 
192       * @param zookeeperHostPorts
193       *          ZooKeeper hostPort for all ZooKeeper servers
194       * @param zookeeperSessionTimeout
195       *          ZooKeeper session timeout
196       * @param parentZnodeName
197       *          znode under which to create the lock
198       * @param acl
199       *          ZooKeeper ACL's
200       * @param authInfo a list of authentication credentials to add to the
201       *                 ZK connection
202       * @param app
203       *          reference to callback interface object
204       * @throws IOException
205       * @throws HadoopIllegalArgumentException
206       */
207      public ActiveStandbyElector(String zookeeperHostPorts,
208          int zookeeperSessionTimeout, String parentZnodeName, List<ACL> acl,
209          List<ZKAuthInfo> authInfo,
210          ActiveStandbyElectorCallback app) throws IOException,
211          HadoopIllegalArgumentException, KeeperException {
212        if (app == null || acl == null || parentZnodeName == null
213            || zookeeperHostPorts == null || zookeeperSessionTimeout <= 0) {
214          throw new HadoopIllegalArgumentException("Invalid argument");
215        }
216        zkHostPort = zookeeperHostPorts;
217        zkSessionTimeout = zookeeperSessionTimeout;
218        zkAcl = acl;
219        zkAuthInfo = authInfo;
220        appClient = app;
221        znodeWorkingDir = parentZnodeName;
222        zkLockFilePath = znodeWorkingDir + "/" + LOCK_FILENAME;
223        zkBreadCrumbPath = znodeWorkingDir + "/" + BREADCRUMB_FILENAME;    
224    
225        // createConnection for future API calls
226        createConnection();
227      }
228    
229      /**
230       * To participate in election, the app will call joinElection. The result will
231       * be notified by a callback on either the becomeActive or becomeStandby app
232       * interfaces. <br/>
233       * After this the elector will automatically monitor the leader status and
234       * perform re-election if necessary<br/>
235       * The app could potentially start off in standby mode and ignore the
236       * becomeStandby call.
237       * 
238       * @param data
239       *          to be set by the app. non-null data must be set.
240       * @throws HadoopIllegalArgumentException
241       *           if valid data is not supplied
242       */
243      public synchronized void joinElection(byte[] data)
244          throws HadoopIllegalArgumentException {
245        
246        if (data == null) {
247          throw new HadoopIllegalArgumentException("data cannot be null");
248        }
249    
250        appData = new byte[data.length];
251        System.arraycopy(data, 0, appData, 0, data.length);
252    
253        LOG.debug("Attempting active election for " + this);
254        joinElectionInternal();
255      }
256      
257      /**
258       * @return true if the configured parent znode exists
259       */
260      public synchronized boolean parentZNodeExists()
261          throws IOException, InterruptedException {
262        Preconditions.checkState(zkClient != null);
263        try {
264          return zkClient.exists(znodeWorkingDir, false) != null;
265        } catch (KeeperException e) {
266          throw new IOException("Couldn't determine existence of znode '" +
267              znodeWorkingDir + "'", e);
268        }
269      }
270    
271      /**
272       * Utility function to ensure that the configured base znode exists.
273       * This recursively creates the znode as well as all of its parents.
274       */
275      public synchronized void ensureParentZNode()
276          throws IOException, InterruptedException {
277        Preconditions.checkState(!wantToBeInElection,
278            "ensureParentZNode() may not be called while in the election");
279    
280        String pathParts[] = znodeWorkingDir.split("/");
281        Preconditions.checkArgument(pathParts.length >= 1 &&
282            "".equals(pathParts[0]),
283            "Invalid path: %s", znodeWorkingDir);
284        
285        StringBuilder sb = new StringBuilder();
286        for (int i = 1; i < pathParts.length; i++) {
287          sb.append("/").append(pathParts[i]);
288          String prefixPath = sb.toString();
289          LOG.debug("Ensuring existence of " + prefixPath);
290          try {
291            createWithRetries(prefixPath, new byte[]{}, zkAcl, CreateMode.PERSISTENT);
292          } catch (KeeperException e) {
293            if (isNodeExists(e.code())) {
294              // This is OK - just ensuring existence.
295              continue;
296            } else {
297              throw new IOException("Couldn't create " + prefixPath, e);
298            }
299          }
300        }
301        
302        LOG.info("Successfully created " + znodeWorkingDir + " in ZK.");
303      }
304      
305      /**
306       * Clear all of the state held within the parent ZNode.
307       * This recursively deletes everything within the znode as well as the
308       * parent znode itself. It should only be used when it's certain that
309       * no electors are currently participating in the election.
310       */
311      public synchronized void clearParentZNode()
312          throws IOException, InterruptedException {
313        Preconditions.checkState(!wantToBeInElection,
314            "clearParentZNode() may not be called while in the election");
315    
316        try {
317          LOG.info("Recursively deleting " + znodeWorkingDir + " from ZK...");
318    
319          zkDoWithRetries(new ZKAction<Void>() {
320            @Override
321            public Void run() throws KeeperException, InterruptedException {
322              ZKUtil.deleteRecursive(zkClient, znodeWorkingDir);
323              return null;
324            }
325          });
326        } catch (KeeperException e) {
327          throw new IOException("Couldn't clear parent znode " + znodeWorkingDir,
328              e);
329        }
330        LOG.info("Successfully deleted " + znodeWorkingDir + " from ZK.");
331      }
332    
333    
334      /**
335       * Any service instance can drop out of the election by calling quitElection. 
336       * <br/>
337       * This will lose any leader status, if held, and stop monitoring of the lock
338       * node. <br/>
339       * If the instance wants to participate in election again, then it needs to
340       * call joinElection(). <br/>
341       * This allows service instances to take themselves out of rotation for known
342       * impending unavailable states (e.g. long GC pause or software upgrade).
343       * 
344       * @param needFence true if the underlying daemon may need to be fenced
345       * if a failover occurs due to dropping out of the election.
346       */
347      public synchronized void quitElection(boolean needFence) {
348        LOG.info("Yielding from election");
349        if (!needFence && state == State.ACTIVE) {
350          // If active is gracefully going back to standby mode, remove
351          // our permanent znode so no one fences us.
352          tryDeleteOwnBreadCrumbNode();
353        }
354        reset();
355        wantToBeInElection = false;
356      }
357    
358      /**
359       * Exception thrown when there is no active leader
360       */
361      public static class ActiveNotFoundException extends Exception {
362        private static final long serialVersionUID = 3505396722342846462L;
363      }
364    
365      /**
366       * get data set by the active leader
367       * 
368       * @return data set by the active instance
369       * @throws ActiveNotFoundException
370       *           when there is no active leader
371       * @throws KeeperException
372       *           other zookeeper operation errors
373       * @throws InterruptedException
374       * @throws IOException
375       *           when ZooKeeper connection could not be established
376       */
377      public synchronized byte[] getActiveData() throws ActiveNotFoundException,
378          KeeperException, InterruptedException, IOException {
379        try {
380          if (zkClient == null) {
381            createConnection();
382          }
383          Stat stat = new Stat();
384          return getDataWithRetries(zkLockFilePath, false, stat);
385        } catch(KeeperException e) {
386          Code code = e.code();
387          if (isNodeDoesNotExist(code)) {
388            // handle the commonly expected cases that make sense for us
389            throw new ActiveNotFoundException();
390          } else {
391            throw e;
392          }
393        }
394      }
395    
396      /**
397       * interface implementation of Zookeeper callback for create
398       */
399      @Override
400      public synchronized void processResult(int rc, String path, Object ctx,
401          String name) {
402        if (isStaleClient(ctx)) return;
403        LOG.debug("CreateNode result: " + rc + " for path: " + path
404            + " connectionState: " + zkConnectionState +
405            "  for " + this);
406    
407        Code code = Code.get(rc);
408        if (isSuccess(code)) {
409          // we successfully created the znode. we are the leader. start monitoring
410          if (becomeActive()) {
411            monitorActiveStatus();
412          } else {
413            reJoinElectionAfterFailureToBecomeActive();
414          }
415          return;
416        }
417    
418        if (isNodeExists(code)) {
419          if (createRetryCount == 0) {
420            // znode exists and we did not retry the operation. so a different
421            // instance has created it. become standby and monitor lock.
422            becomeStandby();
423          }
424          // if we had retried then the znode could have been created by our first
425          // attempt to the server (that we lost) and this node exists response is
426          // for the second attempt. verify this case via ephemeral node owner. this
427          // will happen on the callback for monitoring the lock.
428          monitorActiveStatus();
429          return;
430        }
431    
432        String errorMessage = "Received create error from Zookeeper. code:"
433            + code.toString() + " for path " + path;
434        LOG.debug(errorMessage);
435    
436        if (shouldRetry(code)) {
437          if (createRetryCount < NUM_RETRIES) {
438            LOG.debug("Retrying createNode createRetryCount: " + createRetryCount);
439            ++createRetryCount;
440            createLockNodeAsync();
441            return;
442          }
443          errorMessage = errorMessage
444              + ". Not retrying further znode create connection errors.";
445        } else if (isSessionExpired(code)) {
446          // This isn't fatal - the client Watcher will re-join the election
447          LOG.warn("Lock acquisition failed because session was lost");
448          return;
449        }
450    
451        fatalError(errorMessage);
452      }
453    
454      /**
455       * interface implementation of Zookeeper callback for monitor (exists)
456       */
457      @Override
458      public synchronized void processResult(int rc, String path, Object ctx,
459          Stat stat) {
460        if (isStaleClient(ctx)) return;
461        
462        assert wantToBeInElection :
463            "Got a StatNode result after quitting election";
464        
465        LOG.debug("StatNode result: " + rc + " for path: " + path
466            + " connectionState: " + zkConnectionState + " for " + this);
467            
468    
469        Code code = Code.get(rc);
470        if (isSuccess(code)) {
471          // the following owner check completes verification in case the lock znode
472          // creation was retried
473          if (stat.getEphemeralOwner() == zkClient.getSessionId()) {
474            // we own the lock znode. so we are the leader
475            if (!becomeActive()) {
476              reJoinElectionAfterFailureToBecomeActive();
477            }
478          } else {
479            // we dont own the lock znode. so we are a standby.
480            becomeStandby();
481          }
482          // the watch set by us will notify about changes
483          return;
484        }
485    
486        if (isNodeDoesNotExist(code)) {
487          // the lock znode disappeared before we started monitoring it
488          enterNeutralMode();
489          joinElectionInternal();
490          return;
491        }
492    
493        String errorMessage = "Received stat error from Zookeeper. code:"
494            + code.toString();
495        LOG.debug(errorMessage);
496    
497        if (shouldRetry(code)) {
498          if (statRetryCount < NUM_RETRIES) {
499            ++statRetryCount;
500            monitorLockNodeAsync();
501            return;
502          }
503          errorMessage = errorMessage
504              + ". Not retrying further znode monitoring connection errors.";
505        } else if (isSessionExpired(code)) {
506          // This isn't fatal - the client Watcher will re-join the election
507          LOG.warn("Lock monitoring failed because session was lost");
508          return;
509        }
510    
511        fatalError(errorMessage);
512      }
513    
514      /**
515       * We failed to become active. Re-join the election, but
516       * sleep for a few seconds after terminating our existing
517       * session, so that other nodes have a chance to become active.
518       * The failure to become active is already logged inside
519       * becomeActive().
520       */
521      private void reJoinElectionAfterFailureToBecomeActive() {
522        reJoinElection(SLEEP_AFTER_FAILURE_TO_BECOME_ACTIVE);
523      }
524    
525      /**
526       * interface implementation of Zookeeper watch events (connection and node),
527       * proxied by {@link WatcherWithClientRef}.
528       */
529      synchronized void processWatchEvent(ZooKeeper zk, WatchedEvent event) {
530        Event.EventType eventType = event.getType();
531        if (isStaleClient(zk)) return;
532        LOG.debug("Watcher event type: " + eventType + " with state:"
533            + event.getState() + " for path:" + event.getPath()
534            + " connectionState: " + zkConnectionState
535            + " for " + this);
536    
537        if (eventType == Event.EventType.None) {
538          // the connection state has changed
539          switch (event.getState()) {
540          case SyncConnected:
541            LOG.info("Session connected.");
542            // if the listener was asked to move to safe state then it needs to
543            // be undone
544            ConnectionState prevConnectionState = zkConnectionState;
545            zkConnectionState = ConnectionState.CONNECTED;
546            if (prevConnectionState == ConnectionState.DISCONNECTED &&
547                wantToBeInElection) {
548              monitorActiveStatus();
549            }
550            break;
551          case Disconnected:
552            LOG.info("Session disconnected. Entering neutral mode...");
553    
554            // ask the app to move to safe state because zookeeper connection
555            // is not active and we dont know our state
556            zkConnectionState = ConnectionState.DISCONNECTED;
557            enterNeutralMode();
558            break;
559          case Expired:
560            // the connection got terminated because of session timeout
561            // call listener to reconnect
562            LOG.info("Session expired. Entering neutral mode and rejoining...");
563            enterNeutralMode();
564            reJoinElection(0);
565            break;
566          default:
567            fatalError("Unexpected Zookeeper watch event state: "
568                + event.getState());
569            break;
570          }
571    
572          return;
573        }
574    
575        // a watch on lock path in zookeeper has fired. so something has changed on
576        // the lock. ideally we should check that the path is the same as the lock
577        // path but trusting zookeeper for now
578        String path = event.getPath();
579        if (path != null) {
580          switch (eventType) {
581          case NodeDeleted:
582            if (state == State.ACTIVE) {
583              enterNeutralMode();
584            }
585            joinElectionInternal();
586            break;
587          case NodeDataChanged:
588            monitorActiveStatus();
589            break;
590          default:
591            LOG.debug("Unexpected node event: " + eventType + " for path: " + path);
592            monitorActiveStatus();
593          }
594    
595          return;
596        }
597    
598        // some unexpected error has occurred
599        fatalError("Unexpected watch error from Zookeeper");
600      }
601    
602      /**
603       * Get a new zookeeper client instance. protected so that test class can
604       * inherit and pass in a mock object for zookeeper
605       * 
606       * @return new zookeeper client instance
607       * @throws IOException
608       * @throws KeeperException zookeeper connectionloss exception
609       */
610      protected synchronized ZooKeeper getNewZooKeeper() throws IOException,
611          KeeperException {
612        
613        // Unfortunately, the ZooKeeper constructor connects to ZooKeeper and
614        // may trigger the Connected event immediately. So, if we register the
615        // watcher after constructing ZooKeeper, we may miss that event. Instead,
616        // we construct the watcher first, and have it block any events it receives
617        // before we can set its ZooKeeper reference.
618        WatcherWithClientRef watcher = new WatcherWithClientRef();
619        ZooKeeper zk = new ZooKeeper(zkHostPort, zkSessionTimeout, watcher);
620        watcher.setZooKeeperRef(zk);
621    
622        // Wait for the asynchronous success/failure. This may throw an exception
623        // if we don't connect within the session timeout.
624        watcher.waitForZKConnectionEvent(zkSessionTimeout);
625        
626        for (ZKAuthInfo auth : zkAuthInfo) {
627          zk.addAuthInfo(auth.getScheme(), auth.getAuth());
628        }
629        return zk;
630      }
631    
632      private void fatalError(String errorMessage) {
633        LOG.fatal(errorMessage);
634        reset();
635        appClient.notifyFatalError(errorMessage);
636      }
637    
638      private void monitorActiveStatus() {
639        assert wantToBeInElection;
640        LOG.debug("Monitoring active leader for " + this);
641        statRetryCount = 0;
642        monitorLockNodeAsync();
643      }
644    
645      private void joinElectionInternal() {
646        if (zkClient == null) {
647          if (!reEstablishSession()) {
648            fatalError("Failed to reEstablish connection with ZooKeeper");
649            return;
650          }
651        }
652    
653        createRetryCount = 0;
654        wantToBeInElection = true;
655        createLockNodeAsync();
656      }
657    
658      private void reJoinElection(int sleepTime) {
659        LOG.info("Trying to re-establish ZK session");
660        
661        // Some of the test cases rely on expiring the ZK sessions and
662        // ensuring that the other node takes over. But, there's a race
663        // where the original lease holder could reconnect faster than the other
664        // thread manages to take the lock itself. This lock allows the
665        // tests to block the reconnection. It's a shame that this leaked
666        // into non-test code, but the lock is only acquired here so will never
667        // be contended.
668        sessionReestablishLockForTests.lock();
669        try {
670          terminateConnection();
671          sleepFor(sleepTime);
672          
673          joinElectionInternal();
674        } finally {
675          sessionReestablishLockForTests.unlock();
676        }
677      }
678    
679      /**
680       * Sleep for the given number of milliseconds.
681       * This is non-static, and separated out, so that unit tests
682       * can override the behavior not to sleep.
683       */
684      @VisibleForTesting
685      protected void sleepFor(int sleepMs) {
686        if (sleepMs > 0) {
687          try {
688            Thread.sleep(sleepMs);
689          } catch (InterruptedException e) {
690            Thread.currentThread().interrupt();
691          }
692        }
693      }
694    
695      @VisibleForTesting
696      void preventSessionReestablishmentForTests() {
697        sessionReestablishLockForTests.lock();
698      }
699      
700      @VisibleForTesting
701      void allowSessionReestablishmentForTests() {
702        sessionReestablishLockForTests.unlock();
703      }
704      
705      @VisibleForTesting
706      synchronized long getZKSessionIdForTests() {
707        if (zkClient != null) {
708          return zkClient.getSessionId();
709        } else {
710          return -1;
711        }
712      }
713      
714      @VisibleForTesting
715      synchronized State getStateForTests() {
716        return state;
717      }
718    
719      private boolean reEstablishSession() {
720        int connectionRetryCount = 0;
721        boolean success = false;
722        while(!success && connectionRetryCount < NUM_RETRIES) {
723          LOG.debug("Establishing zookeeper connection for " + this);
724          try {
725            createConnection();
726            success = true;
727          } catch(IOException e) {
728            LOG.warn(e);
729            sleepFor(5000);
730          } catch(KeeperException e) {
731            LOG.warn(e);
732            sleepFor(5000);
733          }
734          ++connectionRetryCount;
735        }
736        return success;
737      }
738    
739      private void createConnection() throws IOException, KeeperException {
740        if (zkClient != null) {
741          try {
742            zkClient.close();
743          } catch (InterruptedException e) {
744            throw new IOException("Interrupted while closing ZK",
745                e);
746          }
747          zkClient = null;
748        }
749        zkClient = getNewZooKeeper();
750        LOG.debug("Created new connection for " + this);
751      }
752      
753      void terminateConnection() {
754        if (zkClient == null) {
755          return;
756        }
757        LOG.debug("Terminating ZK connection for " + this);
758        ZooKeeper tempZk = zkClient;
759        zkClient = null;
760        try {
761          tempZk.close();
762        } catch(InterruptedException e) {
763          LOG.warn(e);
764        }
765        zkConnectionState = ConnectionState.TERMINATED;
766      }
767    
768      private void reset() {
769        state = State.INIT;
770        terminateConnection();
771      }
772    
773      private boolean becomeActive() {
774        assert wantToBeInElection;
775        if (state == State.ACTIVE) {
776          // already active
777          return true;
778        }
779        try {
780          Stat oldBreadcrumbStat = fenceOldActive();
781          writeBreadCrumbNode(oldBreadcrumbStat);
782          
783          LOG.debug("Becoming active for " + this);
784          appClient.becomeActive();
785          state = State.ACTIVE;
786          return true;
787        } catch (Exception e) {
788          LOG.warn("Exception handling the winning of election", e);
789          // Caller will handle quitting and rejoining the election.
790          return false;
791        }
792      }
793    
794      /**
795       * Write the "ActiveBreadCrumb" node, indicating that this node may need
796       * to be fenced on failover.
797       * @param oldBreadcrumbStat 
798       */
799      private void writeBreadCrumbNode(Stat oldBreadcrumbStat)
800          throws KeeperException, InterruptedException {
801        LOG.info("Writing znode " + zkBreadCrumbPath +
802            " to indicate that the local node is the most recent active...");
803        if (oldBreadcrumbStat == null) {
804          // No previous active, just create the node
805          createWithRetries(zkBreadCrumbPath, appData, zkAcl,
806            CreateMode.PERSISTENT);
807        } else {
808          // There was a previous active, update the node
809          setDataWithRetries(zkBreadCrumbPath, appData, oldBreadcrumbStat.getVersion());
810        }
811      }
812      
813      /**
814       * Try to delete the "ActiveBreadCrumb" node when gracefully giving up
815       * active status.
816       * If this fails, it will simply warn, since the graceful release behavior
817       * is only an optimization.
818       */
819      private void tryDeleteOwnBreadCrumbNode() {
820        assert state == State.ACTIVE;
821        LOG.info("Deleting bread-crumb of active node...");
822        
823        // Sanity check the data. This shouldn't be strictly necessary,
824        // but better to play it safe.
825        Stat stat = new Stat();
826        byte[] data = null;
827        try {
828          data = zkClient.getData(zkBreadCrumbPath, false, stat);
829    
830          if (!Arrays.equals(data, appData)) {
831            throw new IllegalStateException(
832                "We thought we were active, but in fact " +
833                "the active znode had the wrong data: " +
834                StringUtils.byteToHexString(data) + " (stat=" + stat + ")");
835          }
836          
837          deleteWithRetries(zkBreadCrumbPath, stat.getVersion());
838        } catch (Exception e) {
839          LOG.warn("Unable to delete our own bread-crumb of being active at " +
840              zkBreadCrumbPath + ": " + e.getLocalizedMessage() + ". " +
841              "Expecting to be fenced by the next active.");
842        }
843      }
844    
845      /**
846       * If there is a breadcrumb node indicating that another node may need
847       * fencing, try to fence that node.
848       * @return the Stat of the breadcrumb node that was read, or null
849       * if no breadcrumb node existed
850       */
851      private Stat fenceOldActive() throws InterruptedException, KeeperException {
852        final Stat stat = new Stat();
853        byte[] data;
854        LOG.info("Checking for any old active which needs to be fenced...");
855        try {
856          data = zkDoWithRetries(new ZKAction<byte[]>() {
857            @Override
858            public byte[] run() throws KeeperException, InterruptedException {
859              return zkClient.getData(zkBreadCrumbPath, false, stat);
860            }
861          });
862        } catch (KeeperException ke) {
863          if (isNodeDoesNotExist(ke.code())) {
864            LOG.info("No old node to fence");
865            return null;
866          }
867          
868          // If we failed to read for any other reason, then likely we lost
869          // our session, or we don't have permissions, etc. In any case,
870          // we probably shouldn't become active, and failing the whole
871          // thing is the best bet.
872          throw ke;
873        }
874    
875        LOG.info("Old node exists: " + StringUtils.byteToHexString(data));
876        if (Arrays.equals(data, appData)) {
877          LOG.info("But old node has our own data, so don't need to fence it.");
878        } else {
879          appClient.fenceOldActive(data);
880        }
881        return stat;
882      }
883    
884      private void becomeStandby() {
885        if (state != State.STANDBY) {
886          LOG.debug("Becoming standby for " + this);
887          state = State.STANDBY;
888          appClient.becomeStandby();
889        }
890      }
891    
892      private void enterNeutralMode() {
893        if (state != State.NEUTRAL) {
894          LOG.debug("Entering neutral mode for " + this);
895          state = State.NEUTRAL;
896          appClient.enterNeutralMode();
897        }
898      }
899    
900      private void createLockNodeAsync() {
901        zkClient.create(zkLockFilePath, appData, zkAcl, CreateMode.EPHEMERAL,
902            this, zkClient);
903      }
904    
905      private void monitorLockNodeAsync() {
906        zkClient.exists(zkLockFilePath, 
907            new WatcherWithClientRef(zkClient), this,
908            zkClient);
909      }
910    
911      private String createWithRetries(final String path, final byte[] data,
912          final List<ACL> acl, final CreateMode mode)
913          throws InterruptedException, KeeperException {
914        return zkDoWithRetries(new ZKAction<String>() {
915          @Override
916          public String run() throws KeeperException, InterruptedException {
917            return zkClient.create(path, data, acl, mode);
918          }
919        });
920      }
921    
922      private byte[] getDataWithRetries(final String path, final boolean watch,
923          final Stat stat) throws InterruptedException, KeeperException {
924        return zkDoWithRetries(new ZKAction<byte[]>() {
925          @Override
926          public byte[] run() throws KeeperException, InterruptedException {
927            return zkClient.getData(path, watch, stat);
928          }
929        });
930      }
931    
932      private Stat setDataWithRetries(final String path, final byte[] data,
933          final int version) throws InterruptedException, KeeperException {
934        return zkDoWithRetries(new ZKAction<Stat>() {
935          @Override
936          public Stat run() throws KeeperException, InterruptedException {
937            return zkClient.setData(path, data, version);
938          }
939        });
940      }
941      
942      private void deleteWithRetries(final String path, final int version)
943          throws KeeperException, InterruptedException {
944        zkDoWithRetries(new ZKAction<Void>() {
945          @Override
946          public Void run() throws KeeperException, InterruptedException {
947            zkClient.delete(path, version);
948            return null;
949          }
950        });
951      }
952    
953      private static <T> T zkDoWithRetries(ZKAction<T> action)
954          throws KeeperException, InterruptedException {
955        int retry = 0;
956        while (true) {
957          try {
958            return action.run();
959          } catch (KeeperException ke) {
960            if (shouldRetry(ke.code()) && ++retry < NUM_RETRIES) {
961              continue;
962            }
963            throw ke;
964          }
965        }
966      }
967    
968      private interface ZKAction<T> {
969        T run() throws KeeperException, InterruptedException; 
970      }
971      
972      /**
973       * The callbacks and watchers pass a reference to the ZK client
974       * which made the original call. We don't want to take action
975       * based on any callbacks from prior clients after we quit
976       * the election.
977       * @param ctx the ZK client passed into the watcher
978       * @return true if it matches the current client
979       */
980      private synchronized boolean isStaleClient(Object ctx) {
981        Preconditions.checkNotNull(ctx);
982        if (zkClient != (ZooKeeper)ctx) {
983          LOG.warn("Ignoring stale result from old client with sessionId " +
984              String.format("0x%08x", ((ZooKeeper)ctx).getSessionId()));
985          return true;
986        }
987        return false;
988      }
989    
990      /**
991       * Watcher implementation which keeps a reference around to the
992       * original ZK connection, and passes it back along with any
993       * events.
994       */
995      private final class WatcherWithClientRef implements Watcher {
996        private ZooKeeper zk;
997        
998        /**
999         * Latch fired whenever any event arrives. This is used in order
1000         * to wait for the Connected event when the client is first created.
1001         */
1002        private CountDownLatch hasReceivedEvent = new CountDownLatch(1);
1003    
1004        /**
1005         * Latch used to wait until the reference to ZooKeeper is set.
1006         */
1007        private CountDownLatch hasSetZooKeeper = new CountDownLatch(1);
1008        
1009        private WatcherWithClientRef() {
1010        }
1011    
1012        private WatcherWithClientRef(ZooKeeper zk) {
1013          setZooKeeperRef(zk);
1014        }
1015    
1016        /**
1017         * Waits for the next event from ZooKeeper to arrive.
1018         * 
1019         * @param connectionTimeoutMs zookeeper connection timeout in milliseconds
1020         * @throws KeeperException if the connection attempt times out. This will
1021         * be a ZooKeeper ConnectionLoss exception code.
1022         * @throws IOException if interrupted while connecting to ZooKeeper
1023         */
1024        private void waitForZKConnectionEvent(int connectionTimeoutMs)
1025            throws KeeperException, IOException {
1026          try {
1027            if (!hasReceivedEvent.await(connectionTimeoutMs, TimeUnit.MILLISECONDS)) {
1028              LOG.error("Connection timed out: couldn't connect to ZooKeeper in "
1029                  + connectionTimeoutMs + " milliseconds");
1030              zk.close();
1031              throw KeeperException.create(Code.CONNECTIONLOSS);
1032            }
1033          } catch (InterruptedException e) {
1034            Thread.currentThread().interrupt();
1035            throw new IOException(
1036                "Interrupted when connecting to zookeeper server", e);
1037          }
1038        }
1039    
1040        private void setZooKeeperRef(ZooKeeper zk) {
1041          Preconditions.checkState(this.zk == null,
1042              "zk already set -- must be set exactly once");
1043          this.zk = zk;
1044          hasSetZooKeeper.countDown();
1045        }
1046    
1047        @Override
1048        public void process(WatchedEvent event) {
1049          hasReceivedEvent.countDown();
1050          try {
1051            hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
1052            ActiveStandbyElector.this.processWatchEvent(
1053                zk, event);
1054          } catch (Throwable t) {
1055            fatalError(
1056                "Failed to process watcher event " + event + ": " +
1057                StringUtils.stringifyException(t));
1058          }
1059        }
1060      }
1061    
1062      private static boolean isSuccess(Code code) {
1063        return (code == Code.OK);
1064      }
1065    
1066      private static boolean isNodeExists(Code code) {
1067        return (code == Code.NODEEXISTS);
1068      }
1069    
1070      private static boolean isNodeDoesNotExist(Code code) {
1071        return (code == Code.NONODE);
1072      }
1073      
1074      private static boolean isSessionExpired(Code code) {
1075        return (code == Code.SESSIONEXPIRED);
1076      }
1077    
1078      private static boolean shouldRetry(Code code) {
1079        switch (code) {
1080        case CONNECTIONLOSS:
1081        case OPERATIONTIMEOUT:
1082          return true;
1083        }
1084        return false;
1085      }
1086      
1087      @Override
1088      public String toString() {
1089        return "elector id=" + System.identityHashCode(this) +
1090          " appData=" +
1091          ((appData == null) ? "null" : StringUtils.byteToHexString(appData)) + 
1092          " cb=" + appClient;
1093      }
1094    }