001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
025import static com.google.common.util.concurrent.Service.State.FAILED;
026import static com.google.common.util.concurrent.Service.State.NEW;
027import static com.google.common.util.concurrent.Service.State.RUNNING;
028import static com.google.common.util.concurrent.Service.State.STARTING;
029import static com.google.common.util.concurrent.Service.State.STOPPING;
030import static com.google.common.util.concurrent.Service.State.TERMINATED;
031import static java.util.concurrent.TimeUnit.MILLISECONDS;
032
033import com.google.common.annotations.Beta;
034import com.google.common.annotations.GwtIncompatible;
035import com.google.common.base.Function;
036import com.google.common.base.MoreObjects;
037import com.google.common.base.Stopwatch;
038import com.google.common.collect.Collections2;
039import com.google.common.collect.ImmutableCollection;
040import com.google.common.collect.ImmutableList;
041import com.google.common.collect.ImmutableMap;
042import com.google.common.collect.ImmutableMultimap;
043import com.google.common.collect.ImmutableSet;
044import com.google.common.collect.ImmutableSetMultimap;
045import com.google.common.collect.Lists;
046import com.google.common.collect.Maps;
047import com.google.common.collect.MultimapBuilder;
048import com.google.common.collect.Multimaps;
049import com.google.common.collect.Multiset;
050import com.google.common.collect.Ordering;
051import com.google.common.collect.SetMultimap;
052import com.google.common.util.concurrent.Service.State;
053import com.google.errorprone.annotations.CanIgnoreReturnValue;
054import com.google.errorprone.annotations.concurrent.GuardedBy;
055import com.google.j2objc.annotations.WeakOuter;
056import java.lang.ref.WeakReference;
057import java.util.Collections;
058import java.util.EnumSet;
059import java.util.List;
060import java.util.Map;
061import java.util.Map.Entry;
062import java.util.concurrent.Executor;
063import java.util.concurrent.TimeUnit;
064import java.util.concurrent.TimeoutException;
065import java.util.logging.Level;
066import java.util.logging.Logger;
067
068/**
069 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
070 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
071 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
072 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
073 * mechanism.
074 *
075 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
076 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
077 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
078 * when appropriate and {@link #awaitHealthy} will still work as expected.
079 *
080 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
081 *
082 * <pre>{@code
083 * class Server {
084 *   public static void main(String[] args) {
085 *     Set<Service> services = ...;
086 *     ServiceManager manager = new ServiceManager(services);
087 *     manager.addListener(new Listener() {
088 *         public void stopped() {}
089 *         public void healthy() {
090 *           // Services have been initialized and are healthy, start accepting requests...
091 *         }
092 *         public void failure(Service service) {
093 *           // Something failed, at this point we could log it, notify a load balancer, or take
094 *           // some other action.  For now we will just exit.
095 *           System.exit(1);
096 *         }
097 *       },
098 *       MoreExecutors.directExecutor());
099 *
100 *     Runtime.getRuntime().addShutdownHook(new Thread() {
101 *       public void run() {
102 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
103 *         // requests.
104 *         try {
105 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
106 *         } catch (TimeoutException timeout) {
107 *           // stopping timed out
108 *         }
109 *       }
110 *     });
111 *     manager.startAsync();  // start all the services asynchronously
112 *   }
113 * }
114 * }</pre>
115 *
116 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
117 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
118 *
119 * @author Luke Sandberg
120 * @since 14.0
121 */
122@Beta
123@GwtIncompatible
124public final class ServiceManager {
125  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
126  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
127      new ListenerCallQueue.Event<Listener>() {
128        @Override
129        public void call(Listener listener) {
130          listener.healthy();
131        }
132
133        @Override
134        public String toString() {
135          return "healthy()";
136        }
137      };
138  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
139      new ListenerCallQueue.Event<Listener>() {
140        @Override
141        public void call(Listener listener) {
142          listener.stopped();
143        }
144
145        @Override
146        public String toString() {
147          return "stopped()";
148        }
149      };
150
151  /**
152   * A listener for the aggregate state changes of the services that are under management. Users
153   * that need to listen to more fine-grained events (such as when each particular {@linkplain
154   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
155   * listeners} to each individual service.
156   *
157   * @author Luke Sandberg
158   * @since 15.0 (present as an interface in 14.0)
159   */
160  @Beta // Should come out of Beta when ServiceManager does
161  public abstract static class Listener {
162    /**
163     * Called when the service initially becomes healthy.
164     *
165     * <p>This will be called at most once after all the services have entered the {@linkplain
166     * State#RUNNING running} state. If any services fail during start up or {@linkplain
167     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
168     * started {@linkplain State#RUNNING running} then this method will not be called.
169     */
170    public void healthy() {}
171
172    /**
173     * Called when the all of the component services have reached a terminal state, either
174     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
175     */
176    public void stopped() {}
177
178    /**
179     * Called when a component service has {@linkplain State#FAILED failed}.
180     *
181     * @param service The service that failed.
182     */
183    public void failure(Service service) {}
184  }
185
186  /**
187   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
188   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
189   * be made {@code static} and its instances can be safely constructed and added in the {@link
190   * ServiceManager} constructor without having to close over the partially constructed {@link
191   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
192   */
193  private final ServiceManagerState state;
194
195  private final ImmutableList<Service> services;
196
197  /**
198   * Constructs a new instance for managing the given services.
199   *
200   * @param services The services to manage
201   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
202   *     are any duplicate services.
203   */
204  public ServiceManager(Iterable<? extends Service> services) {
205    ImmutableList<Service> copy = ImmutableList.copyOf(services);
206    if (copy.isEmpty()) {
207      // Having no services causes the manager to behave strangely. Notably, listeners are never
208      // fired. To avoid this we substitute a placeholder service.
209      logger.log(
210          Level.WARNING,
211          "ServiceManager configured with no services.  Is your application configured properly?",
212          new EmptyServiceManagerWarning());
213      copy = ImmutableList.<Service>of(new NoOpService());
214    }
215    this.state = new ServiceManagerState(copy);
216    this.services = copy;
217    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
218    for (Service service : copy) {
219      service.addListener(new ServiceListener(service, stateReference), directExecutor());
220      // We check the state after adding the listener as a way to ensure that our listener was added
221      // to a NEW service.
222      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
223    }
224    // We have installed all of our listeners and after this point any state transition should be
225    // correct.
226    this.state.markReady();
227  }
228
229  /**
230   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
231   * executor. The listener will not have previous state changes replayed, so it is suggested that
232   * listeners are added before any of the managed services are {@linkplain Service#startAsync
233   * started}.
234   *
235   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
236   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
237   * invoked in the same order as the underlying service enters those states. Additionally, at most
238   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
239   * may execute concurrently, and listeners may execute in an order different from the one in which
240   * they were registered.
241   *
242   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
243   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
244   * logged.
245   *
246   * <p>For fast, lightweight listeners that would be safe to execute in any thread, consider
247   * calling {@link #addListener(Listener)}.
248   *
249   * @param listener the listener to run when the manager changes state
250   * @param executor the executor in which the listeners callback methods will be run.
251   */
252  public void addListener(Listener listener, Executor executor) {
253    state.addListener(listener, executor);
254  }
255
256  /**
257   * Registers a {@link Listener} to be run when this {@link ServiceManager} changes state. The
258   * listener will not have previous state changes replayed, so it is suggested that listeners are
259   * added before any of the managed services are {@linkplain Service#startAsync started}.
260   *
261   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
262   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
263   * invoked in the same order as the underlying service enters those states. Additionally, at most
264   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
265   * may execute concurrently, and listeners may execute in an order different from the one in which
266   * they were registered.
267   *
268   * <p>RuntimeExceptions thrown by a listener will be caught and logged.
269   *
270   * @param listener the listener to run when the manager changes state
271   */
272  public void addListener(Listener listener) {
273    state.addListener(listener, directExecutor());
274  }
275
276  /**
277   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
278   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
279   *
280   * @return this
281   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
282   *     method is called.
283   */
284  @CanIgnoreReturnValue
285  public ServiceManager startAsync() {
286    for (Service service : services) {
287      State state = service.state();
288      checkState(state == NEW, "Service %s is %s, cannot start it.", service, state);
289    }
290    for (Service service : services) {
291      try {
292        state.tryStartTiming(service);
293        service.startAsync();
294      } catch (IllegalStateException e) {
295        // This can happen if the service has already been started or stopped (e.g. by another
296        // service or listener). Our contract says it is safe to call this method if
297        // all services were NEW when it was called, and this has already been verified above, so we
298        // don't propagate the exception.
299        logger.log(Level.WARNING, "Unable to start Service " + service, e);
300      }
301    }
302    return this;
303  }
304
305  /**
306   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
307   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
308   * running} state.
309   *
310   * @throws IllegalStateException if the service manager reaches a state from which it cannot
311   *     become {@linkplain #isHealthy() healthy}.
312   */
313  public void awaitHealthy() {
314    state.awaitHealthy();
315  }
316
317  /**
318   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
319   * than the given time. The manager will become healthy after all the component services have
320   * reached the {@linkplain State#RUNNING running} state.
321   *
322   * @param timeout the maximum time to wait
323   * @param unit the time unit of the timeout argument
324   * @throws TimeoutException if not all of the services have finished starting within the deadline
325   * @throws IllegalStateException if the service manager reaches a state from which it cannot
326   *     become {@linkplain #isHealthy() healthy}.
327   */
328  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
329  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
330    state.awaitHealthy(timeout, unit);
331  }
332
333  /**
334   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
335   * being managed.
336   *
337   * @return this
338   */
339  @CanIgnoreReturnValue
340  public ServiceManager stopAsync() {
341    for (Service service : services) {
342      service.stopAsync();
343    }
344    return this;
345  }
346
347  /**
348   * Waits for the all the services to reach a terminal state. After this method returns all
349   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
350   * Service.State#FAILED failed}.
351   */
352  public void awaitStopped() {
353    state.awaitStopped();
354  }
355
356  /**
357   * Waits for the all the services to reach a terminal state for no more than the given time. After
358   * this method returns all services will either be {@linkplain Service.State#TERMINATED
359   * terminated} or {@linkplain Service.State#FAILED failed}.
360   *
361   * @param timeout the maximum time to wait
362   * @param unit the time unit of the timeout argument
363   * @throws TimeoutException if not all of the services have stopped within the deadline
364   */
365  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
366  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
367    state.awaitStopped(timeout, unit);
368  }
369
370  /**
371   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
372   *
373   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
374   * get detailed information about which services are not running.
375   */
376  public boolean isHealthy() {
377    for (Service service : services) {
378      if (!service.isRunning()) {
379        return false;
380      }
381    }
382    return true;
383  }
384
385  /**
386   * Provides a snapshot of the current state of all the services under management.
387   *
388   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
389   * correspond to a point in time view of the services.
390   */
391  public ImmutableMultimap<State, Service> servicesByState() {
392    return state.servicesByState();
393  }
394
395  /**
396   * Returns the service load times. This value will only return startup times for services that
397   * have finished starting.
398   *
399   * @return Map of services and their corresponding startup time in millis, the map entries will be
400   *     ordered by startup time.
401   */
402  public ImmutableMap<Service, Long> startupTimes() {
403    return state.startupTimes();
404  }
405
406  @Override
407  public String toString() {
408    return MoreObjects.toStringHelper(ServiceManager.class)
409        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
410        .toString();
411  }
412
413  /**
414   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
415   * accessed by instances of {@link ServiceListener}.
416   */
417  private static final class ServiceManagerState {
418    final Monitor monitor = new Monitor();
419
420    @GuardedBy("monitor")
421    final SetMultimap<State, Service> servicesByState =
422        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
423
424    @GuardedBy("monitor")
425    final Multiset<State> states = servicesByState.keys();
426
427    @GuardedBy("monitor")
428    final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
429
430    /**
431     * These two booleans are used to mark the state as ready to start.
432     *
433     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
434     * correctly installed
435     *
436     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
437     * transition has been performed.
438     *
439     * <p>Together, they allow us to enforce that all services have their listeners installed prior
440     * to any service performing a transition, then we can fail in the ServiceManager constructor
441     * rather than in a Service.Listener callback.
442     */
443    @GuardedBy("monitor")
444    boolean ready;
445
446    @GuardedBy("monitor")
447    boolean transitioned;
448
449    final int numberOfServices;
450
451    /**
452     * Controls how long to wait for all the services to either become healthy or reach a state from
453     * which it is guaranteed that it can never become healthy.
454     */
455    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
456
457    @WeakOuter
458    final class AwaitHealthGuard extends Monitor.Guard {
459      AwaitHealthGuard() {
460        super(ServiceManagerState.this.monitor);
461      }
462
463      @Override
464      @GuardedBy("ServiceManagerState.this.monitor")
465      public boolean isSatisfied() {
466        // All services have started or some service has terminated/failed.
467        return states.count(RUNNING) == numberOfServices
468            || states.contains(STOPPING)
469            || states.contains(TERMINATED)
470            || states.contains(FAILED);
471      }
472    }
473
474    /** Controls how long to wait for all services to reach a terminal state. */
475    final Monitor.Guard stoppedGuard = new StoppedGuard();
476
477    @WeakOuter
478    final class StoppedGuard extends Monitor.Guard {
479      StoppedGuard() {
480        super(ServiceManagerState.this.monitor);
481      }
482
483      @Override
484      @GuardedBy("ServiceManagerState.this.monitor")
485      public boolean isSatisfied() {
486        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
487      }
488    }
489
490    /** The listeners to notify during a state transition. */
491    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
492
493    /**
494     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
495     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
496     * responsibility to only call {@link #markReady()} if all services were new at the time this
497     * method was called and when all the listeners were installed.
498     */
499    ServiceManagerState(ImmutableCollection<Service> services) {
500      this.numberOfServices = services.size();
501      servicesByState.putAll(NEW, services);
502    }
503
504    /**
505     * Attempts to start the timer immediately prior to the service being started via {@link
506     * Service#startAsync()}.
507     */
508    void tryStartTiming(Service service) {
509      monitor.enter();
510      try {
511        Stopwatch stopwatch = startupTimers.get(service);
512        if (stopwatch == null) {
513          startupTimers.put(service, Stopwatch.createStarted());
514        }
515      } finally {
516        monitor.leave();
517      }
518    }
519
520    /**
521     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
522     * been observed yet.
523     */
524    void markReady() {
525      monitor.enter();
526      try {
527        if (!transitioned) {
528          // nothing has transitioned since construction, good.
529          ready = true;
530        } else {
531          // This should be an extremely rare race condition.
532          List<Service> servicesInBadStates = Lists.newArrayList();
533          for (Service service : servicesByState().values()) {
534            if (service.state() != NEW) {
535              servicesInBadStates.add(service);
536            }
537          }
538          throw new IllegalArgumentException(
539              "Services started transitioning asynchronously before "
540                  + "the ServiceManager was constructed: "
541                  + servicesInBadStates);
542        }
543      } finally {
544        monitor.leave();
545      }
546    }
547
548    void addListener(Listener listener, Executor executor) {
549      listeners.addListener(listener, executor);
550    }
551
552    void awaitHealthy() {
553      monitor.enterWhenUninterruptibly(awaitHealthGuard);
554      try {
555        checkHealthy();
556      } finally {
557        monitor.leave();
558      }
559    }
560
561    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
562      monitor.enter();
563      try {
564        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
565          throw new TimeoutException(
566              "Timeout waiting for the services to become healthy. The "
567                  + "following services have not started: "
568                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
569        }
570        checkHealthy();
571      } finally {
572        monitor.leave();
573      }
574    }
575
576    void awaitStopped() {
577      monitor.enterWhenUninterruptibly(stoppedGuard);
578      monitor.leave();
579    }
580
581    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
582      monitor.enter();
583      try {
584        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
585          throw new TimeoutException(
586              "Timeout waiting for the services to stop. The following "
587                  + "services have not stopped: "
588                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
589        }
590      } finally {
591        monitor.leave();
592      }
593    }
594
595    ImmutableMultimap<State, Service> servicesByState() {
596      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
597      monitor.enter();
598      try {
599        for (Entry<State, Service> entry : servicesByState.entries()) {
600          if (!(entry.getValue() instanceof NoOpService)) {
601            builder.put(entry);
602          }
603        }
604      } finally {
605        monitor.leave();
606      }
607      return builder.build();
608    }
609
610    ImmutableMap<Service, Long> startupTimes() {
611      List<Entry<Service, Long>> loadTimes;
612      monitor.enter();
613      try {
614        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
615        // N.B. There will only be an entry in the map if the service has started
616        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
617          Service service = entry.getKey();
618          Stopwatch stopWatch = entry.getValue();
619          if (!stopWatch.isRunning() && !(service instanceof NoOpService)) {
620            loadTimes.add(Maps.immutableEntry(service, stopWatch.elapsed(MILLISECONDS)));
621          }
622        }
623      } finally {
624        monitor.leave();
625      }
626      Collections.sort(
627          loadTimes,
628          Ordering.natural()
629              .onResultOf(
630                  new Function<Entry<Service, Long>, Long>() {
631                    @Override
632                    public Long apply(Entry<Service, Long> input) {
633                      return input.getValue();
634                    }
635                  }));
636      return ImmutableMap.copyOf(loadTimes);
637    }
638
639    /**
640     * Updates the state with the given service transition.
641     *
642     * <p>This method performs the main logic of ServiceManager in the following steps.
643     *
644     * <ol>
645     *   <li>Update the {@link #servicesByState()}
646     *   <li>Update the {@link #startupTimers}
647     *   <li>Based on the new state queue listeners to run
648     *   <li>Run the listeners (outside of the lock)
649     * </ol>
650     */
651    void transitionService(final Service service, State from, State to) {
652      checkNotNull(service);
653      checkArgument(from != to);
654      monitor.enter();
655      try {
656        transitioned = true;
657        if (!ready) {
658          return;
659        }
660        // Update state.
661        checkState(
662            servicesByState.remove(from, service),
663            "Service %s not at the expected location in the state map %s",
664            service,
665            from);
666        checkState(
667            servicesByState.put(to, service),
668            "Service %s in the state map unexpectedly at %s",
669            service,
670            to);
671        // Update the timer
672        Stopwatch stopwatch = startupTimers.get(service);
673        if (stopwatch == null) {
674          // This means the service was started by some means other than ServiceManager.startAsync
675          stopwatch = Stopwatch.createStarted();
676          startupTimers.put(service, stopwatch);
677        }
678        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
679          // N.B. if we miss the STARTING event then we may never record a startup time.
680          stopwatch.stop();
681          if (!(service instanceof NoOpService)) {
682            logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
683          }
684        }
685        // Queue our listeners
686
687        // Did a service fail?
688        if (to == FAILED) {
689          enqueueFailedEvent(service);
690        }
691
692        if (states.count(RUNNING) == numberOfServices) {
693          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
694          // they are not guaranteed to get 'true', because any service could fail right now.
695          enqueueHealthyEvent();
696        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
697          enqueueStoppedEvent();
698        }
699      } finally {
700        monitor.leave();
701        // Run our executors outside of the lock
702        dispatchListenerEvents();
703      }
704    }
705
706    void enqueueStoppedEvent() {
707      listeners.enqueue(STOPPED_EVENT);
708    }
709
710    void enqueueHealthyEvent() {
711      listeners.enqueue(HEALTHY_EVENT);
712    }
713
714    void enqueueFailedEvent(final Service service) {
715      listeners.enqueue(
716          new ListenerCallQueue.Event<Listener>() {
717            @Override
718            public void call(Listener listener) {
719              listener.failure(service);
720            }
721
722            @Override
723            public String toString() {
724              return "failed({service=" + service + "})";
725            }
726          });
727    }
728
729    /** Attempts to execute all the listeners in {@link #listeners}. */
730    void dispatchListenerEvents() {
731      checkState(
732          !monitor.isOccupiedByCurrentThread(),
733          "It is incorrect to execute listeners with the monitor held.");
734      listeners.dispatch();
735    }
736
737    @GuardedBy("monitor")
738    void checkHealthy() {
739      if (states.count(RUNNING) != numberOfServices) {
740        IllegalStateException exception =
741            new IllegalStateException(
742                "Expected to be healthy after starting. The following services are not running: "
743                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
744        for (Service service : servicesByState.get(State.FAILED)) {
745          exception.addSuppressed(new FailedService(service));
746        }
747        throw exception;
748      }
749    }
750  }
751
752  /**
753   * A {@link Service} that wraps another service and times how long it takes for it to start and
754   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
755   * the state transitions.
756   */
757  private static final class ServiceListener extends Service.Listener {
758    final Service service;
759    // We store the state in a weak reference to ensure that if something went wrong while
760    // constructing the ServiceManager we don't pointlessly keep updating the state.
761    final WeakReference<ServiceManagerState> state;
762
763    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
764      this.service = service;
765      this.state = state;
766    }
767
768    @Override
769    public void starting() {
770      ServiceManagerState state = this.state.get();
771      if (state != null) {
772        state.transitionService(service, NEW, STARTING);
773        if (!(service instanceof NoOpService)) {
774          logger.log(Level.FINE, "Starting {0}.", service);
775        }
776      }
777    }
778
779    @Override
780    public void running() {
781      ServiceManagerState state = this.state.get();
782      if (state != null) {
783        state.transitionService(service, STARTING, RUNNING);
784      }
785    }
786
787    @Override
788    public void stopping(State from) {
789      ServiceManagerState state = this.state.get();
790      if (state != null) {
791        state.transitionService(service, from, STOPPING);
792      }
793    }
794
795    @Override
796    public void terminated(State from) {
797      ServiceManagerState state = this.state.get();
798      if (state != null) {
799        if (!(service instanceof NoOpService)) {
800          logger.log(
801              Level.FINE,
802              "Service {0} has terminated. Previous state was: {1}",
803              new Object[] {service, from});
804        }
805        state.transitionService(service, from, TERMINATED);
806      }
807    }
808
809    @Override
810    public void failed(State from, Throwable failure) {
811      ServiceManagerState state = this.state.get();
812      if (state != null) {
813        // Log before the transition, so that if the process exits in response to server failure,
814        // there is a higher likelihood that the cause will be in the logs.
815        boolean log = !(service instanceof NoOpService);
816        /*
817         * We have already exposed startup exceptions to the user in the form of suppressed
818         * exceptions. We don't need to log those exceptions again.
819         */
820        log &= from != State.STARTING;
821        if (log) {
822          logger.log(
823              Level.SEVERE,
824              "Service " + service + " has failed in the " + from + " state.",
825              failure);
826        }
827        state.transitionService(service, from, FAILED);
828      }
829    }
830  }
831
832  /**
833   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
834   * that the {@link ServiceManager} functions properly even when it is managing no services.
835   *
836   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
837   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
838   * logging statements.
839   */
840  private static final class NoOpService extends AbstractService {
841    @Override
842    protected void doStart() {
843      notifyStarted();
844    }
845
846    @Override
847    protected void doStop() {
848      notifyStopped();
849    }
850  }
851
852  /** This is never thrown but only used for logging. */
853  private static final class EmptyServiceManagerWarning extends Throwable {}
854
855  private static final class FailedService extends Throwable {
856    FailedService(Service service) {
857      super(
858          service.toString(),
859          service.failureCause(),
860          false /* don't enable suppression */,
861          false /* don't calculate a stack trace. */);
862    }
863  }
864}