001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.Internal.toNanosSaturated;
025import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
026import static com.google.common.util.concurrent.Service.State.FAILED;
027import static com.google.common.util.concurrent.Service.State.NEW;
028import static com.google.common.util.concurrent.Service.State.RUNNING;
029import static com.google.common.util.concurrent.Service.State.STARTING;
030import static com.google.common.util.concurrent.Service.State.STOPPING;
031import static com.google.common.util.concurrent.Service.State.TERMINATED;
032import static java.util.Collections.sort;
033import static java.util.concurrent.TimeUnit.MILLISECONDS;
034
035import com.google.common.annotations.GwtIncompatible;
036import com.google.common.annotations.J2ktIncompatible;
037import com.google.common.base.MoreObjects;
038import com.google.common.base.Stopwatch;
039import com.google.common.collect.Collections2;
040import com.google.common.collect.ImmutableCollection;
041import com.google.common.collect.ImmutableList;
042import com.google.common.collect.ImmutableMap;
043import com.google.common.collect.ImmutableSet;
044import com.google.common.collect.ImmutableSetMultimap;
045import com.google.common.collect.Lists;
046import com.google.common.collect.Maps;
047import com.google.common.collect.MultimapBuilder;
048import com.google.common.collect.Multimaps;
049import com.google.common.collect.Multiset;
050import com.google.common.collect.Ordering;
051import com.google.common.collect.SetMultimap;
052import com.google.common.util.concurrent.Service.State;
053import com.google.errorprone.annotations.CanIgnoreReturnValue;
054import com.google.errorprone.annotations.concurrent.GuardedBy;
055import com.google.j2objc.annotations.J2ObjCIncompatible;
056import com.google.j2objc.annotations.WeakOuter;
057import java.lang.ref.WeakReference;
058import java.time.Duration;
059import java.util.EnumSet;
060import java.util.IdentityHashMap;
061import java.util.List;
062import java.util.Map.Entry;
063import java.util.concurrent.Executor;
064import java.util.concurrent.TimeUnit;
065import java.util.concurrent.TimeoutException;
066import java.util.logging.Level;
067
068/**
069 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
070 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
071 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
072 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
073 * mechanism.
074 *
075 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
076 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
077 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
078 * when appropriate and {@link #awaitHealthy} will still work as expected.
079 *
080 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
081 *
082 * <pre>{@code
083 * class Server {
084 *   public static void main(String[] args) {
085 *     Set<Service> services = ...;
086 *     ServiceManager manager = new ServiceManager(services);
087 *     manager.addListener(new Listener() {
088 *         public void stopped() {}
089 *         public void healthy() {
090 *           // Services have been initialized and are healthy, start accepting requests...
091 *         }
092 *         public void failure(Service service) {
093 *           // Something failed, at this point we could log it, notify a load balancer, or take
094 *           // some other action.  For now we will just exit.
095 *           System.exit(1);
096 *         }
097 *       },
098 *       MoreExecutors.directExecutor());
099 *
100 *     Runtime.getRuntime().addShutdownHook(new Thread() {
101 *       public void run() {
102 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
103 *         // requests.
104 *         try {
105 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
106 *         } catch (TimeoutException timeout) {
107 *           // stopping timed out
108 *         }
109 *       }
110 *     });
111 *     manager.startAsync();  // start all the services asynchronously
112 *   }
113 * }
114 * }</pre>
115 *
116 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
117 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
118 *
119 * @author Luke Sandberg
120 * @since 14.0
121 */
122@J2ktIncompatible
123@GwtIncompatible
124public final class ServiceManager implements ServiceManagerBridge {
125  private static final LazyLogger logger = new LazyLogger(ServiceManager.class);
126  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
127      new ListenerCallQueue.Event<Listener>() {
128        @Override
129        public void call(Listener listener) {
130          listener.healthy();
131        }
132
133        @Override
134        public String toString() {
135          return "healthy()";
136        }
137      };
138  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
139      new ListenerCallQueue.Event<Listener>() {
140        @Override
141        public void call(Listener listener) {
142          listener.stopped();
143        }
144
145        @Override
146        public String toString() {
147          return "stopped()";
148        }
149      };
150
151  /**
152   * A listener for the aggregate state changes of the services that are under management. Users
153   * that need to listen to more fine-grained events (such as when each particular {@linkplain
154   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
155   * listeners} to each individual service.
156   *
157   * @author Luke Sandberg
158   * @since 15.0 (present as an interface in 14.0)
159   */
160  public abstract static class Listener {
161    /** Constructor for use by subclasses. */
162    public Listener() {}
163
164    /**
165     * Called when the service initially becomes healthy.
166     *
167     * <p>This will be called at most once after all the services have entered the {@linkplain
168     * State#RUNNING running} state. If any services fail during start up or {@linkplain
169     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
170     * started {@linkplain State#RUNNING running} then this method will not be called.
171     */
172    public void healthy() {}
173
174    /**
175     * Called when the all of the component services have reached a terminal state, either
176     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
177     */
178    public void stopped() {}
179
180    /**
181     * Called when a component service has {@linkplain State#FAILED failed}.
182     *
183     * @param service The service that failed.
184     */
185    public void failure(Service service) {}
186  }
187
188  /**
189   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
190   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
191   * be made {@code static} and its instances can be safely constructed and added in the {@link
192   * ServiceManager} constructor without having to close over the partially constructed {@link
193   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
194   */
195  private final ServiceManagerState state;
196
197  private final ImmutableList<Service> services;
198
199  /**
200   * Constructs a new instance for managing the given services.
201   *
202   * @param services The services to manage
203   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
204   *     are any duplicate services.
205   */
206  public ServiceManager(Iterable<? extends Service> services) {
207    ImmutableList<Service> copy = ImmutableList.copyOf(services);
208    if (copy.isEmpty()) {
209      // Having no services causes the manager to behave strangely. Notably, listeners are never
210      // fired. To avoid this we substitute a placeholder service.
211      logger
212          .get()
213          .log(
214              Level.WARNING,
215              "ServiceManager configured with no services.  Is your application configured"
216                  + " properly?",
217              new EmptyServiceManagerWarning());
218      copy = ImmutableList.<Service>of(new NoOpService());
219    }
220    this.state = new ServiceManagerState(copy);
221    this.services = copy;
222    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
223    for (Service service : copy) {
224      service.addListener(new ServiceListener(service, stateReference), directExecutor());
225      // We check the state after adding the listener as a way to ensure that our listener was added
226      // to a NEW service.
227      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
228    }
229    // We have installed all of our listeners and after this point any state transition should be
230    // correct.
231    this.state.markReady();
232  }
233
234  /**
235   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
236   * executor. The listener will not have previous state changes replayed, so it is suggested that
237   * listeners are added before any of the managed services are {@linkplain Service#startAsync
238   * started}.
239   *
240   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
241   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
242   * invoked in the same order as the underlying service enters those states. Additionally, at most
243   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
244   * may execute concurrently, and listeners may execute in an order different from the one in which
245   * they were registered.
246   *
247   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
248   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
249   * logged.
250   *
251   * <p>When selecting an executor, note that {@code directExecutor} is dangerous in some cases. See
252   * the discussion in the {@link ListenableFuture#addListener ListenableFuture.addListener}
253   * documentation.
254   *
255   * @param listener the listener to run when the manager changes state
256   * @param executor the executor in which the listeners callback methods will be run.
257   */
258  public void addListener(Listener listener, Executor executor) {
259    state.addListener(listener, executor);
260  }
261
262  /**
263   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
264   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
265   *
266   * @return this
267   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
268   *     method is called.
269   */
270  @CanIgnoreReturnValue
271  public ServiceManager startAsync() {
272    for (Service service : services) {
273      checkState(service.state() == NEW, "Not all services are NEW, cannot start %s", this);
274    }
275    for (Service service : services) {
276      try {
277        state.tryStartTiming(service);
278        service.startAsync();
279      } catch (IllegalStateException e) {
280        // This can happen if the service has already been started or stopped (e.g. by another
281        // service or listener). Our contract says it is safe to call this method if
282        // all services were NEW when it was called, and this has already been verified above, so we
283        // don't propagate the exception.
284        logger.get().log(Level.WARNING, "Unable to start Service " + service, e);
285      }
286    }
287    return this;
288  }
289
290  /**
291   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
292   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
293   * running} state.
294   *
295   * @throws IllegalStateException if the service manager reaches a state from which it cannot
296   *     become {@linkplain #isHealthy() healthy}.
297   */
298  public void awaitHealthy() {
299    state.awaitHealthy();
300  }
301
302  /**
303   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
304   * than the given time. The manager will become healthy after all the component services have
305   * reached the {@linkplain State#RUNNING running} state.
306   *
307   * @param timeout the maximum time to wait
308   * @throws TimeoutException if not all of the services have finished starting within the deadline
309   * @throws IllegalStateException if the service manager reaches a state from which it cannot
310   *     become {@linkplain #isHealthy() healthy}.
311   * @since 28.0 (but only since 33.4.0 in the Android flavor)
312   */
313  public void awaitHealthy(Duration timeout) throws TimeoutException {
314    awaitHealthy(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
315  }
316
317  /**
318   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
319   * than the given time. The manager will become healthy after all the component services have
320   * reached the {@linkplain State#RUNNING running} state.
321   *
322   * @param timeout the maximum time to wait
323   * @param unit the time unit of the timeout argument
324   * @throws TimeoutException if not all of the services have finished starting within the deadline
325   * @throws IllegalStateException if the service manager reaches a state from which it cannot
326   *     become {@linkplain #isHealthy() healthy}.
327   */
328  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
329  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
330    state.awaitHealthy(timeout, unit);
331  }
332
333  /**
334   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
335   * being managed.
336   *
337   * @return this
338   */
339  @CanIgnoreReturnValue
340  public ServiceManager stopAsync() {
341    for (Service service : services) {
342      service.stopAsync();
343    }
344    return this;
345  }
346
347  /**
348   * Waits for the all the services to reach a terminal state. After this method returns all
349   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
350   * Service.State#FAILED failed}.
351   */
352  public void awaitStopped() {
353    state.awaitStopped();
354  }
355
356  /**
357   * Waits for the all the services to reach a terminal state for no more than the given time. After
358   * this method returns all services will either be {@linkplain Service.State#TERMINATED
359   * terminated} or {@linkplain Service.State#FAILED failed}.
360   *
361   * @param timeout the maximum time to wait
362   * @throws TimeoutException if not all of the services have stopped within the deadline
363   * @since 28.0 (but only since 33.4.0 in the Android flavor)
364   */
365  public void awaitStopped(Duration timeout) throws TimeoutException {
366    awaitStopped(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
367  }
368
369  /**
370   * Waits for the all the services to reach a terminal state for no more than the given time. After
371   * this method returns all services will either be {@linkplain Service.State#TERMINATED
372   * terminated} or {@linkplain Service.State#FAILED failed}.
373   *
374   * @param timeout the maximum time to wait
375   * @param unit the time unit of the timeout argument
376   * @throws TimeoutException if not all of the services have stopped within the deadline
377   */
378  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
379  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
380    state.awaitStopped(timeout, unit);
381  }
382
383  /**
384   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
385   *
386   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
387   * get detailed information about which services are not running.
388   */
389  public boolean isHealthy() {
390    for (Service service : services) {
391      if (!service.isRunning()) {
392        return false;
393      }
394    }
395    return true;
396  }
397
398  /**
399   * Provides a snapshot of the current state of all the services under management.
400   *
401   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
402   * correspond to a point in time view of the services.
403   *
404   * @since 29.0 (present with return type {@code ImmutableMultimap} since 14.0)
405   */
406  @Override
407  public ImmutableSetMultimap<State, Service> servicesByState() {
408    return state.servicesByState();
409  }
410
411  /**
412   * Returns the service load times. This value will only return startup times for services that
413   * have finished starting.
414   *
415   * @return Map of services and their corresponding startup time in millis, the map entries will be
416   *     ordered by startup time.
417   */
418  public ImmutableMap<Service, Long> startupTimes() {
419    return state.startupTimes();
420  }
421
422  /**
423   * Returns the service load times. This value will only return startup times for services that
424   * have finished starting.
425   *
426   * @return Map of services and their corresponding startup time, the map entries will be ordered
427   *     by startup time.
428   * @since 31.0 (but only since 33.4.0 in the Android flavor)
429   */
430  @J2ObjCIncompatible
431  public ImmutableMap<Service, Duration> startupDurations() {
432    return ImmutableMap.copyOf(
433        Maps.<Service, Long, Duration>transformValues(startupTimes(), Duration::ofMillis));
434  }
435
436  @Override
437  public String toString() {
438    return MoreObjects.toStringHelper(ServiceManager.class)
439        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
440        .toString();
441  }
442
443  /**
444   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
445   * accessed by instances of {@link ServiceListener}.
446   */
447  private static final class ServiceManagerState {
448    final Monitor monitor = new Monitor();
449
450    @GuardedBy("monitor")
451    final SetMultimap<State, Service> servicesByState =
452        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
453
454    @GuardedBy("monitor")
455    final Multiset<State> states = servicesByState.keys();
456
457    @GuardedBy("monitor")
458    final IdentityHashMap<Service, Stopwatch> startupTimers = new IdentityHashMap<>();
459
460    /**
461     * These two booleans are used to mark the state as ready to start.
462     *
463     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
464     * correctly installed
465     *
466     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
467     * transition has been performed.
468     *
469     * <p>Together, they allow us to enforce that all services have their listeners installed prior
470     * to any service performing a transition, then we can fail in the ServiceManager constructor
471     * rather than in a Service.Listener callback.
472     */
473    @GuardedBy("monitor")
474    boolean ready;
475
476    @GuardedBy("monitor")
477    boolean transitioned;
478
479    final int numberOfServices;
480
481    /**
482     * Controls how long to wait for all the services to either become healthy or reach a state from
483     * which it is guaranteed that it can never become healthy.
484     */
485    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
486
487    @WeakOuter
488    final class AwaitHealthGuard extends Monitor.Guard {
489      AwaitHealthGuard() {
490        super(ServiceManagerState.this.monitor);
491      }
492
493      @Override
494      @GuardedBy("ServiceManagerState.this.monitor")
495      public boolean isSatisfied() {
496        // All services have started or some service has terminated/failed.
497        return states.count(RUNNING) == numberOfServices
498            || states.contains(STOPPING)
499            || states.contains(TERMINATED)
500            || states.contains(FAILED);
501      }
502    }
503
504    /** Controls how long to wait for all services to reach a terminal state. */
505    final Monitor.Guard stoppedGuard = new StoppedGuard();
506
507    @WeakOuter
508    final class StoppedGuard extends Monitor.Guard {
509      StoppedGuard() {
510        super(ServiceManagerState.this.monitor);
511      }
512
513      @Override
514      @GuardedBy("ServiceManagerState.this.monitor")
515      public boolean isSatisfied() {
516        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
517      }
518    }
519
520    /** The listeners to notify during a state transition. */
521    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
522
523    /**
524     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
525     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
526     * responsibility to only call {@link #markReady()} if all services were new at the time this
527     * method was called and when all the listeners were installed.
528     */
529    ServiceManagerState(ImmutableCollection<Service> services) {
530      this.numberOfServices = services.size();
531      servicesByState.putAll(NEW, services);
532    }
533
534    /**
535     * Attempts to start the timer immediately prior to the service being started via {@link
536     * Service#startAsync()}.
537     */
538    void tryStartTiming(Service service) {
539      monitor.enter();
540      try {
541        Stopwatch stopwatch = startupTimers.get(service);
542        if (stopwatch == null) {
543          startupTimers.put(service, Stopwatch.createStarted());
544        }
545      } finally {
546        monitor.leave();
547      }
548    }
549
550    /**
551     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
552     * been observed yet.
553     */
554    void markReady() {
555      monitor.enter();
556      try {
557        if (!transitioned) {
558          // nothing has transitioned since construction, good.
559          ready = true;
560        } else {
561          // This should be an extremely rare race condition.
562          List<Service> servicesInBadStates = Lists.newArrayList();
563          for (Service service : servicesByState().values()) {
564            if (service.state() != NEW) {
565              servicesInBadStates.add(service);
566            }
567          }
568          throw new IllegalArgumentException(
569              "Services started transitioning asynchronously before "
570                  + "the ServiceManager was constructed: "
571                  + servicesInBadStates);
572        }
573      } finally {
574        monitor.leave();
575      }
576    }
577
578    void addListener(Listener listener, Executor executor) {
579      listeners.addListener(listener, executor);
580    }
581
582    void awaitHealthy() {
583      monitor.enterWhenUninterruptibly(awaitHealthGuard);
584      try {
585        checkHealthy();
586      } finally {
587        monitor.leave();
588      }
589    }
590
591    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
592      monitor.enter();
593      try {
594        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
595          throw new TimeoutException(
596              "Timeout waiting for the services to become healthy. The "
597                  + "following services have not started: "
598                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
599        }
600        checkHealthy();
601      } finally {
602        monitor.leave();
603      }
604    }
605
606    void awaitStopped() {
607      monitor.enterWhenUninterruptibly(stoppedGuard);
608      monitor.leave();
609    }
610
611    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
612      monitor.enter();
613      try {
614        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
615          throw new TimeoutException(
616              "Timeout waiting for the services to stop. The following "
617                  + "services have not stopped: "
618                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
619        }
620      } finally {
621        monitor.leave();
622      }
623    }
624
625    ImmutableSetMultimap<State, Service> servicesByState() {
626      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
627      monitor.enter();
628      try {
629        for (Entry<State, Service> entry : servicesByState.entries()) {
630          if (!(entry.getValue() instanceof NoOpService)) {
631            builder.put(entry);
632          }
633        }
634      } finally {
635        monitor.leave();
636      }
637      return builder.build();
638    }
639
640    ImmutableMap<Service, Long> startupTimes() {
641      List<Entry<Service, Long>> loadTimes;
642      monitor.enter();
643      try {
644        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
645        // N.B. There will only be an entry in the map if the service has started
646        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
647          Service service = entry.getKey();
648          Stopwatch stopwatch = entry.getValue();
649          if (!stopwatch.isRunning() && !(service instanceof NoOpService)) {
650            loadTimes.add(Maps.immutableEntry(service, stopwatch.elapsed(MILLISECONDS)));
651          }
652        }
653      } finally {
654        monitor.leave();
655      }
656      sort(loadTimes, Ordering.natural().onResultOf(Entry::getValue));
657      return ImmutableMap.copyOf(loadTimes);
658    }
659
660    /**
661     * Updates the state with the given service transition.
662     *
663     * <p>This method performs the main logic of ServiceManager in the following steps.
664     *
665     * <ol>
666     *   <li>Update the {@link #servicesByState()}
667     *   <li>Update the {@link #startupTimers}
668     *   <li>Based on the new state queue listeners to run
669     *   <li>Run the listeners (outside of the lock)
670     * </ol>
671     */
672    void transitionService(Service service, State from, State to) {
673      checkNotNull(service);
674      checkArgument(from != to);
675      monitor.enter();
676      try {
677        transitioned = true;
678        if (!ready) {
679          return;
680        }
681        // Update state.
682        checkState(
683            servicesByState.remove(from, service),
684            "Service %s not at the expected location in the state map %s",
685            service,
686            from);
687        checkState(
688            servicesByState.put(to, service),
689            "Service %s in the state map unexpectedly at %s",
690            service,
691            to);
692        // Update the timer
693        Stopwatch stopwatch = startupTimers.get(service);
694        if (stopwatch == null) {
695          // This means the service was started by some means other than ServiceManager.startAsync
696          stopwatch = Stopwatch.createStarted();
697          startupTimers.put(service, stopwatch);
698        }
699        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
700          // N.B. if we miss the STARTING event then we may never record a startup time.
701          stopwatch.stop();
702          if (!(service instanceof NoOpService)) {
703            logger.get().log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
704          }
705        }
706        // Queue our listeners
707
708        // Did a service fail?
709        if (to == FAILED) {
710          enqueueFailedEvent(service);
711        }
712
713        if (states.count(RUNNING) == numberOfServices) {
714          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
715          // they are not guaranteed to get 'true', because any service could fail right now.
716          enqueueHealthyEvent();
717        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
718          enqueueStoppedEvent();
719        }
720      } finally {
721        monitor.leave();
722        // Run our executors outside of the lock
723        dispatchListenerEvents();
724      }
725    }
726
727    void enqueueStoppedEvent() {
728      listeners.enqueue(STOPPED_EVENT);
729    }
730
731    void enqueueHealthyEvent() {
732      listeners.enqueue(HEALTHY_EVENT);
733    }
734
735    void enqueueFailedEvent(Service service) {
736      listeners.enqueue(
737          new ListenerCallQueue.Event<Listener>() {
738            @Override
739            public void call(Listener listener) {
740              listener.failure(service);
741            }
742
743            @Override
744            public String toString() {
745              return "failed({service=" + service + "})";
746            }
747          });
748    }
749
750    /** Attempts to execute all the listeners in {@link #listeners}. */
751    void dispatchListenerEvents() {
752      checkState(
753          !monitor.isOccupiedByCurrentThread(),
754          "It is incorrect to execute listeners with the monitor held.");
755      listeners.dispatch();
756    }
757
758    @GuardedBy("monitor")
759    void checkHealthy() {
760      if (states.count(RUNNING) != numberOfServices) {
761        IllegalStateException exception =
762            new IllegalStateException(
763                "Expected to be healthy after starting. The following services are not running: "
764                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
765        for (Service service : servicesByState.get(State.FAILED)) {
766          exception.addSuppressed(new FailedService(service));
767        }
768        throw exception;
769      }
770    }
771  }
772
773  /**
774   * A {@link Service} that wraps another service and times how long it takes for it to start and
775   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
776   * the state transitions.
777   */
778  private static final class ServiceListener extends Service.Listener {
779    final Service service;
780    // We store the state in a weak reference to ensure that if something went wrong while
781    // constructing the ServiceManager we don't pointlessly keep updating the state.
782    final WeakReference<ServiceManagerState> state;
783
784    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
785      this.service = service;
786      this.state = state;
787    }
788
789    @Override
790    public void starting() {
791      ServiceManagerState state = this.state.get();
792      if (state != null) {
793        state.transitionService(service, NEW, STARTING);
794        if (!(service instanceof NoOpService)) {
795          logger.get().log(Level.FINE, "Starting {0}.", service);
796        }
797      }
798    }
799
800    @Override
801    public void running() {
802      ServiceManagerState state = this.state.get();
803      if (state != null) {
804        state.transitionService(service, STARTING, RUNNING);
805      }
806    }
807
808    @Override
809    public void stopping(State from) {
810      ServiceManagerState state = this.state.get();
811      if (state != null) {
812        state.transitionService(service, from, STOPPING);
813      }
814    }
815
816    @Override
817    public void terminated(State from) {
818      ServiceManagerState state = this.state.get();
819      if (state != null) {
820        if (!(service instanceof NoOpService)) {
821          logger
822              .get()
823              .log(
824                  Level.FINE,
825                  "Service {0} has terminated. Previous state was: {1}",
826                  new Object[] {service, from});
827        }
828        state.transitionService(service, from, TERMINATED);
829      }
830    }
831
832    @Override
833    public void failed(State from, Throwable failure) {
834      ServiceManagerState state = this.state.get();
835      if (state != null) {
836        // Log before the transition, so that if the process exits in response to server failure,
837        // there is a higher likelihood that the cause will be in the logs.
838        boolean log = !(service instanceof NoOpService);
839        /*
840         * We have already exposed startup exceptions to the user in the form of suppressed
841         * exceptions. We don't need to log those exceptions again.
842         */
843        log &= from != State.STARTING;
844        if (log) {
845          logger
846              .get()
847              .log(
848                  Level.SEVERE,
849                  "Service " + service + " has failed in the " + from + " state.",
850                  failure);
851        }
852        state.transitionService(service, from, FAILED);
853      }
854    }
855  }
856
857  /**
858   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
859   * that the {@link ServiceManager} functions properly even when it is managing no services.
860   *
861   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
862   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
863   * logging statements.
864   */
865  private static final class NoOpService extends AbstractService {
866    @Override
867    protected void doStart() {
868      notifyStarted();
869    }
870
871    @Override
872    protected void doStop() {
873      notifyStopped();
874    }
875  }
876
877  /** This is never thrown but only used for logging. */
878  private static final class EmptyServiceManagerWarning extends Throwable {}
879
880  private static final class FailedService extends Throwable {
881    FailedService(Service service) {
882      super(
883          service.toString(),
884          service.failureCause(),
885          false /* don't enable suppression */,
886          false /* don't calculate a stack trace. */);
887    }
888  }
889}