001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.Internal.toNanosSaturated;
025import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
026import static com.google.common.util.concurrent.Service.State.FAILED;
027import static com.google.common.util.concurrent.Service.State.NEW;
028import static com.google.common.util.concurrent.Service.State.RUNNING;
029import static com.google.common.util.concurrent.Service.State.STARTING;
030import static com.google.common.util.concurrent.Service.State.STOPPING;
031import static com.google.common.util.concurrent.Service.State.TERMINATED;
032import static java.util.concurrent.TimeUnit.MILLISECONDS;
033
034import com.google.common.annotations.GwtIncompatible;
035import com.google.common.annotations.J2ktIncompatible;
036import com.google.common.base.Function;
037import com.google.common.base.MoreObjects;
038import com.google.common.base.Stopwatch;
039import com.google.common.collect.Collections2;
040import com.google.common.collect.ImmutableCollection;
041import com.google.common.collect.ImmutableList;
042import com.google.common.collect.ImmutableMap;
043import com.google.common.collect.ImmutableSet;
044import com.google.common.collect.ImmutableSetMultimap;
045import com.google.common.collect.Lists;
046import com.google.common.collect.Maps;
047import com.google.common.collect.MultimapBuilder;
048import com.google.common.collect.Multimaps;
049import com.google.common.collect.Multiset;
050import com.google.common.collect.Ordering;
051import com.google.common.collect.SetMultimap;
052import com.google.common.util.concurrent.Service.State;
053import com.google.errorprone.annotations.CanIgnoreReturnValue;
054import com.google.errorprone.annotations.concurrent.GuardedBy;
055import com.google.j2objc.annotations.J2ObjCIncompatible;
056import com.google.j2objc.annotations.WeakOuter;
057import java.lang.ref.WeakReference;
058import java.time.Duration;
059import java.util.Collections;
060import java.util.EnumSet;
061import java.util.IdentityHashMap;
062import java.util.List;
063import java.util.Map.Entry;
064import java.util.concurrent.Executor;
065import java.util.concurrent.TimeUnit;
066import java.util.concurrent.TimeoutException;
067import java.util.logging.Level;
068
069/**
070 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
071 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
072 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
073 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
074 * mechanism.
075 *
076 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
077 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
078 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
079 * when appropriate and {@link #awaitHealthy} will still work as expected.
080 *
081 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
082 *
083 * <pre>{@code
084 * class Server {
085 *   public static void main(String[] args) {
086 *     Set<Service> services = ...;
087 *     ServiceManager manager = new ServiceManager(services);
088 *     manager.addListener(new Listener() {
089 *         public void stopped() {}
090 *         public void healthy() {
091 *           // Services have been initialized and are healthy, start accepting requests...
092 *         }
093 *         public void failure(Service service) {
094 *           // Something failed, at this point we could log it, notify a load balancer, or take
095 *           // some other action.  For now we will just exit.
096 *           System.exit(1);
097 *         }
098 *       },
099 *       MoreExecutors.directExecutor());
100 *
101 *     Runtime.getRuntime().addShutdownHook(new Thread() {
102 *       public void run() {
103 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
104 *         // requests.
105 *         try {
106 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
107 *         } catch (TimeoutException timeout) {
108 *           // stopping timed out
109 *         }
110 *       }
111 *     });
112 *     manager.startAsync();  // start all the services asynchronously
113 *   }
114 * }
115 * }</pre>
116 *
117 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
118 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
119 *
120 * @author Luke Sandberg
121 * @since 14.0
122 */
123@J2ktIncompatible
124@GwtIncompatible
125@ElementTypesAreNonnullByDefault
126public final class ServiceManager implements ServiceManagerBridge {
127  private static final LazyLogger logger = new LazyLogger(ServiceManager.class);
128  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
129      new ListenerCallQueue.Event<Listener>() {
130        @Override
131        public void call(Listener listener) {
132          listener.healthy();
133        }
134
135        @Override
136        public String toString() {
137          return "healthy()";
138        }
139      };
140  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
141      new ListenerCallQueue.Event<Listener>() {
142        @Override
143        public void call(Listener listener) {
144          listener.stopped();
145        }
146
147        @Override
148        public String toString() {
149          return "stopped()";
150        }
151      };
152
153  /**
154   * A listener for the aggregate state changes of the services that are under management. Users
155   * that need to listen to more fine-grained events (such as when each particular {@linkplain
156   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
157   * listeners} to each individual service.
158   *
159   * @author Luke Sandberg
160   * @since 15.0 (present as an interface in 14.0)
161   */
162  public abstract static class Listener {
163    /** Constructor for use by subclasses. */
164    public Listener() {}
165
166    /**
167     * Called when the service initially becomes healthy.
168     *
169     * <p>This will be called at most once after all the services have entered the {@linkplain
170     * State#RUNNING running} state. If any services fail during start up or {@linkplain
171     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
172     * started {@linkplain State#RUNNING running} then this method will not be called.
173     */
174    public void healthy() {}
175
176    /**
177     * Called when the all of the component services have reached a terminal state, either
178     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
179     */
180    public void stopped() {}
181
182    /**
183     * Called when a component service has {@linkplain State#FAILED failed}.
184     *
185     * @param service The service that failed.
186     */
187    public void failure(Service service) {}
188  }
189
190  /**
191   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
192   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
193   * be made {@code static} and its instances can be safely constructed and added in the {@link
194   * ServiceManager} constructor without having to close over the partially constructed {@link
195   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
196   */
197  private final ServiceManagerState state;
198
199  private final ImmutableList<Service> services;
200
201  /**
202   * Constructs a new instance for managing the given services.
203   *
204   * @param services The services to manage
205   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
206   *     are any duplicate services.
207   */
208  public ServiceManager(Iterable<? extends Service> services) {
209    ImmutableList<Service> copy = ImmutableList.copyOf(services);
210    if (copy.isEmpty()) {
211      // Having no services causes the manager to behave strangely. Notably, listeners are never
212      // fired. To avoid this we substitute a placeholder service.
213      logger
214          .get()
215          .log(
216              Level.WARNING,
217              "ServiceManager configured with no services.  Is your application configured"
218                  + " properly?",
219              new EmptyServiceManagerWarning());
220      copy = ImmutableList.<Service>of(new NoOpService());
221    }
222    this.state = new ServiceManagerState(copy);
223    this.services = copy;
224    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
225    for (Service service : copy) {
226      service.addListener(new ServiceListener(service, stateReference), directExecutor());
227      // We check the state after adding the listener as a way to ensure that our listener was added
228      // to a NEW service.
229      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
230    }
231    // We have installed all of our listeners and after this point any state transition should be
232    // correct.
233    this.state.markReady();
234  }
235
236  /**
237   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
238   * executor. The listener will not have previous state changes replayed, so it is suggested that
239   * listeners are added before any of the managed services are {@linkplain Service#startAsync
240   * started}.
241   *
242   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
243   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
244   * invoked in the same order as the underlying service enters those states. Additionally, at most
245   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
246   * may execute concurrently, and listeners may execute in an order different from the one in which
247   * they were registered.
248   *
249   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
250   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
251   * logged.
252   *
253   * <p>When selecting an executor, note that {@code directExecutor} is dangerous in some cases. See
254   * the discussion in the {@link ListenableFuture#addListener ListenableFuture.addListener}
255   * documentation.
256   *
257   * @param listener the listener to run when the manager changes state
258   * @param executor the executor in which the listeners callback methods will be run.
259   */
260  public void addListener(Listener listener, Executor executor) {
261    state.addListener(listener, executor);
262  }
263
264  /**
265   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
266   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
267   *
268   * @return this
269   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
270   *     method is called.
271   */
272  @CanIgnoreReturnValue
273  public ServiceManager startAsync() {
274    for (Service service : services) {
275      checkState(service.state() == NEW, "Not all services are NEW, cannot start %s", this);
276    }
277    for (Service service : services) {
278      try {
279        state.tryStartTiming(service);
280        service.startAsync();
281      } catch (IllegalStateException e) {
282        // This can happen if the service has already been started or stopped (e.g. by another
283        // service or listener). Our contract says it is safe to call this method if
284        // all services were NEW when it was called, and this has already been verified above, so we
285        // don't propagate the exception.
286        logger.get().log(Level.WARNING, "Unable to start Service " + service, e);
287      }
288    }
289    return this;
290  }
291
292  /**
293   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
294   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
295   * running} state.
296   *
297   * @throws IllegalStateException if the service manager reaches a state from which it cannot
298   *     become {@linkplain #isHealthy() healthy}.
299   */
300  public void awaitHealthy() {
301    state.awaitHealthy();
302  }
303
304  /**
305   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
306   * than the given time. The manager will become healthy after all the component services have
307   * reached the {@linkplain State#RUNNING running} state.
308   *
309   * @param timeout the maximum time to wait
310   * @throws TimeoutException if not all of the services have finished starting within the deadline
311   * @throws IllegalStateException if the service manager reaches a state from which it cannot
312   *     become {@linkplain #isHealthy() healthy}.
313   * @since 33.4.0 (but since 28.0 in the JRE flavor)
314   */
315  @SuppressWarnings("Java7ApiChecker")
316  @IgnoreJRERequirement // Users will use this only if they're already using Duration.
317  public void awaitHealthy(Duration timeout) throws TimeoutException {
318    awaitHealthy(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
319  }
320
321  /**
322   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
323   * than the given time. The manager will become healthy after all the component services have
324   * reached the {@linkplain State#RUNNING running} state.
325   *
326   * @param timeout the maximum time to wait
327   * @param unit the time unit of the timeout argument
328   * @throws TimeoutException if not all of the services have finished starting within the deadline
329   * @throws IllegalStateException if the service manager reaches a state from which it cannot
330   *     become {@linkplain #isHealthy() healthy}.
331   */
332  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
333  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
334    state.awaitHealthy(timeout, unit);
335  }
336
337  /**
338   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
339   * being managed.
340   *
341   * @return this
342   */
343  @CanIgnoreReturnValue
344  public ServiceManager stopAsync() {
345    for (Service service : services) {
346      service.stopAsync();
347    }
348    return this;
349  }
350
351  /**
352   * Waits for the all the services to reach a terminal state. After this method returns all
353   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
354   * Service.State#FAILED failed}.
355   */
356  public void awaitStopped() {
357    state.awaitStopped();
358  }
359
360  /**
361   * Waits for the all the services to reach a terminal state for no more than the given time. After
362   * this method returns all services will either be {@linkplain Service.State#TERMINATED
363   * terminated} or {@linkplain Service.State#FAILED failed}.
364   *
365   * @param timeout the maximum time to wait
366   * @throws TimeoutException if not all of the services have stopped within the deadline
367   * @since 33.4.0 (but since 28.0 in the JRE flavor)
368   */
369  @SuppressWarnings("Java7ApiChecker")
370  @IgnoreJRERequirement // Users will use this only if they're already using Duration.
371  public void awaitStopped(Duration timeout) throws TimeoutException {
372    awaitStopped(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
373  }
374
375  /**
376   * Waits for the all the services to reach a terminal state for no more than the given time. After
377   * this method returns all services will either be {@linkplain Service.State#TERMINATED
378   * terminated} or {@linkplain Service.State#FAILED failed}.
379   *
380   * @param timeout the maximum time to wait
381   * @param unit the time unit of the timeout argument
382   * @throws TimeoutException if not all of the services have stopped within the deadline
383   */
384  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
385  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
386    state.awaitStopped(timeout, unit);
387  }
388
389  /**
390   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
391   *
392   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
393   * get detailed information about which services are not running.
394   */
395  public boolean isHealthy() {
396    for (Service service : services) {
397      if (!service.isRunning()) {
398        return false;
399      }
400    }
401    return true;
402  }
403
404  /**
405   * Provides a snapshot of the current state of all the services under management.
406   *
407   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
408   * correspond to a point in time view of the services.
409   *
410   * @since 29.0 (present with return type {@code ImmutableMultimap} since 14.0)
411   */
412  @Override
413  public ImmutableSetMultimap<State, Service> servicesByState() {
414    return state.servicesByState();
415  }
416
417  /**
418   * Returns the service load times. This value will only return startup times for services that
419   * have finished starting.
420   *
421   * @return Map of services and their corresponding startup time in millis, the map entries will be
422   *     ordered by startup time.
423   */
424  public ImmutableMap<Service, Long> startupTimes() {
425    return state.startupTimes();
426  }
427
428  /**
429   * Returns the service load times. This value will only return startup times for services that
430   * have finished starting.
431   *
432   * @return Map of services and their corresponding startup time, the map entries will be ordered
433   *     by startup time.
434   * @since 33.4.0 (but since 31.0 in the JRE flavor)
435   */
436  @J2ObjCIncompatible
437  @SuppressWarnings("Java7ApiChecker")
438  // If users use this when they shouldn't, we hope that NewApi will catch subsequent Duration calls
439  @IgnoreJRERequirement
440  public ImmutableMap<Service, Duration> startupDurations() {
441    return ImmutableMap.copyOf(
442        Maps.<Service, Long, Duration>transformValues(startupTimes(), Duration::ofMillis));
443  }
444
445  @Override
446  public String toString() {
447    return MoreObjects.toStringHelper(ServiceManager.class)
448        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
449        .toString();
450  }
451
452  /**
453   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
454   * accessed by instances of {@link ServiceListener}.
455   */
456  private static final class ServiceManagerState {
457    final Monitor monitor = new Monitor();
458
459    @GuardedBy("monitor")
460    final SetMultimap<State, Service> servicesByState =
461        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
462
463    @GuardedBy("monitor")
464    final Multiset<State> states = servicesByState.keys();
465
466    @GuardedBy("monitor")
467    final IdentityHashMap<Service, Stopwatch> startupTimers = new IdentityHashMap<>();
468
469    /**
470     * These two booleans are used to mark the state as ready to start.
471     *
472     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
473     * correctly installed
474     *
475     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
476     * transition has been performed.
477     *
478     * <p>Together, they allow us to enforce that all services have their listeners installed prior
479     * to any service performing a transition, then we can fail in the ServiceManager constructor
480     * rather than in a Service.Listener callback.
481     */
482    @GuardedBy("monitor")
483    boolean ready;
484
485    @GuardedBy("monitor")
486    boolean transitioned;
487
488    final int numberOfServices;
489
490    /**
491     * Controls how long to wait for all the services to either become healthy or reach a state from
492     * which it is guaranteed that it can never become healthy.
493     */
494    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
495
496    @WeakOuter
497    final class AwaitHealthGuard extends Monitor.Guard {
498      AwaitHealthGuard() {
499        super(ServiceManagerState.this.monitor);
500      }
501
502      @Override
503      @GuardedBy("ServiceManagerState.this.monitor")
504      public boolean isSatisfied() {
505        // All services have started or some service has terminated/failed.
506        return states.count(RUNNING) == numberOfServices
507            || states.contains(STOPPING)
508            || states.contains(TERMINATED)
509            || states.contains(FAILED);
510      }
511    }
512
513    /** Controls how long to wait for all services to reach a terminal state. */
514    final Monitor.Guard stoppedGuard = new StoppedGuard();
515
516    @WeakOuter
517    final class StoppedGuard extends Monitor.Guard {
518      StoppedGuard() {
519        super(ServiceManagerState.this.monitor);
520      }
521
522      @Override
523      @GuardedBy("ServiceManagerState.this.monitor")
524      public boolean isSatisfied() {
525        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
526      }
527    }
528
529    /** The listeners to notify during a state transition. */
530    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
531
532    /**
533     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
534     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
535     * responsibility to only call {@link #markReady()} if all services were new at the time this
536     * method was called and when all the listeners were installed.
537     */
538    ServiceManagerState(ImmutableCollection<Service> services) {
539      this.numberOfServices = services.size();
540      servicesByState.putAll(NEW, services);
541    }
542
543    /**
544     * Attempts to start the timer immediately prior to the service being started via {@link
545     * Service#startAsync()}.
546     */
547    void tryStartTiming(Service service) {
548      monitor.enter();
549      try {
550        Stopwatch stopwatch = startupTimers.get(service);
551        if (stopwatch == null) {
552          startupTimers.put(service, Stopwatch.createStarted());
553        }
554      } finally {
555        monitor.leave();
556      }
557    }
558
559    /**
560     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
561     * been observed yet.
562     */
563    void markReady() {
564      monitor.enter();
565      try {
566        if (!transitioned) {
567          // nothing has transitioned since construction, good.
568          ready = true;
569        } else {
570          // This should be an extremely rare race condition.
571          List<Service> servicesInBadStates = Lists.newArrayList();
572          for (Service service : servicesByState().values()) {
573            if (service.state() != NEW) {
574              servicesInBadStates.add(service);
575            }
576          }
577          throw new IllegalArgumentException(
578              "Services started transitioning asynchronously before "
579                  + "the ServiceManager was constructed: "
580                  + servicesInBadStates);
581        }
582      } finally {
583        monitor.leave();
584      }
585    }
586
587    void addListener(Listener listener, Executor executor) {
588      listeners.addListener(listener, executor);
589    }
590
591    void awaitHealthy() {
592      monitor.enterWhenUninterruptibly(awaitHealthGuard);
593      try {
594        checkHealthy();
595      } finally {
596        monitor.leave();
597      }
598    }
599
600    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
601      monitor.enter();
602      try {
603        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
604          throw new TimeoutException(
605              "Timeout waiting for the services to become healthy. The "
606                  + "following services have not started: "
607                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
608        }
609        checkHealthy();
610      } finally {
611        monitor.leave();
612      }
613    }
614
615    void awaitStopped() {
616      monitor.enterWhenUninterruptibly(stoppedGuard);
617      monitor.leave();
618    }
619
620    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
621      monitor.enter();
622      try {
623        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
624          throw new TimeoutException(
625              "Timeout waiting for the services to stop. The following "
626                  + "services have not stopped: "
627                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
628        }
629      } finally {
630        monitor.leave();
631      }
632    }
633
634    ImmutableSetMultimap<State, Service> servicesByState() {
635      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
636      monitor.enter();
637      try {
638        for (Entry<State, Service> entry : servicesByState.entries()) {
639          if (!(entry.getValue() instanceof NoOpService)) {
640            builder.put(entry);
641          }
642        }
643      } finally {
644        monitor.leave();
645      }
646      return builder.build();
647    }
648
649    ImmutableMap<Service, Long> startupTimes() {
650      List<Entry<Service, Long>> loadTimes;
651      monitor.enter();
652      try {
653        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
654        // N.B. There will only be an entry in the map if the service has started
655        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
656          Service service = entry.getKey();
657          Stopwatch stopwatch = entry.getValue();
658          if (!stopwatch.isRunning() && !(service instanceof NoOpService)) {
659            loadTimes.add(Maps.immutableEntry(service, stopwatch.elapsed(MILLISECONDS)));
660          }
661        }
662      } finally {
663        monitor.leave();
664      }
665      Collections.sort(
666          loadTimes,
667          Ordering.natural()
668              .onResultOf(
669                  new Function<Entry<Service, Long>, Long>() {
670                    @Override
671                    public Long apply(Entry<Service, Long> input) {
672                      return input.getValue();
673                    }
674                  }));
675      return ImmutableMap.copyOf(loadTimes);
676    }
677
678    /**
679     * Updates the state with the given service transition.
680     *
681     * <p>This method performs the main logic of ServiceManager in the following steps.
682     *
683     * <ol>
684     *   <li>Update the {@link #servicesByState()}
685     *   <li>Update the {@link #startupTimers}
686     *   <li>Based on the new state queue listeners to run
687     *   <li>Run the listeners (outside of the lock)
688     * </ol>
689     */
690    void transitionService(final Service service, State from, State to) {
691      checkNotNull(service);
692      checkArgument(from != to);
693      monitor.enter();
694      try {
695        transitioned = true;
696        if (!ready) {
697          return;
698        }
699        // Update state.
700        checkState(
701            servicesByState.remove(from, service),
702            "Service %s not at the expected location in the state map %s",
703            service,
704            from);
705        checkState(
706            servicesByState.put(to, service),
707            "Service %s in the state map unexpectedly at %s",
708            service,
709            to);
710        // Update the timer
711        Stopwatch stopwatch = startupTimers.get(service);
712        if (stopwatch == null) {
713          // This means the service was started by some means other than ServiceManager.startAsync
714          stopwatch = Stopwatch.createStarted();
715          startupTimers.put(service, stopwatch);
716        }
717        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
718          // N.B. if we miss the STARTING event then we may never record a startup time.
719          stopwatch.stop();
720          if (!(service instanceof NoOpService)) {
721            logger.get().log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
722          }
723        }
724        // Queue our listeners
725
726        // Did a service fail?
727        if (to == FAILED) {
728          enqueueFailedEvent(service);
729        }
730
731        if (states.count(RUNNING) == numberOfServices) {
732          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
733          // they are not guaranteed to get 'true', because any service could fail right now.
734          enqueueHealthyEvent();
735        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
736          enqueueStoppedEvent();
737        }
738      } finally {
739        monitor.leave();
740        // Run our executors outside of the lock
741        dispatchListenerEvents();
742      }
743    }
744
745    void enqueueStoppedEvent() {
746      listeners.enqueue(STOPPED_EVENT);
747    }
748
749    void enqueueHealthyEvent() {
750      listeners.enqueue(HEALTHY_EVENT);
751    }
752
753    void enqueueFailedEvent(final Service service) {
754      listeners.enqueue(
755          new ListenerCallQueue.Event<Listener>() {
756            @Override
757            public void call(Listener listener) {
758              listener.failure(service);
759            }
760
761            @Override
762            public String toString() {
763              return "failed({service=" + service + "})";
764            }
765          });
766    }
767
768    /** Attempts to execute all the listeners in {@link #listeners}. */
769    void dispatchListenerEvents() {
770      checkState(
771          !monitor.isOccupiedByCurrentThread(),
772          "It is incorrect to execute listeners with the monitor held.");
773      listeners.dispatch();
774    }
775
776    @GuardedBy("monitor")
777    void checkHealthy() {
778      if (states.count(RUNNING) != numberOfServices) {
779        IllegalStateException exception =
780            new IllegalStateException(
781                "Expected to be healthy after starting. The following services are not running: "
782                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
783        for (Service service : servicesByState.get(State.FAILED)) {
784          exception.addSuppressed(new FailedService(service));
785        }
786        throw exception;
787      }
788    }
789  }
790
791  /**
792   * A {@link Service} that wraps another service and times how long it takes for it to start and
793   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
794   * the state transitions.
795   */
796  private static final class ServiceListener extends Service.Listener {
797    final Service service;
798    // We store the state in a weak reference to ensure that if something went wrong while
799    // constructing the ServiceManager we don't pointlessly keep updating the state.
800    final WeakReference<ServiceManagerState> state;
801
802    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
803      this.service = service;
804      this.state = state;
805    }
806
807    @Override
808    public void starting() {
809      ServiceManagerState state = this.state.get();
810      if (state != null) {
811        state.transitionService(service, NEW, STARTING);
812        if (!(service instanceof NoOpService)) {
813          logger.get().log(Level.FINE, "Starting {0}.", service);
814        }
815      }
816    }
817
818    @Override
819    public void running() {
820      ServiceManagerState state = this.state.get();
821      if (state != null) {
822        state.transitionService(service, STARTING, RUNNING);
823      }
824    }
825
826    @Override
827    public void stopping(State from) {
828      ServiceManagerState state = this.state.get();
829      if (state != null) {
830        state.transitionService(service, from, STOPPING);
831      }
832    }
833
834    @Override
835    public void terminated(State from) {
836      ServiceManagerState state = this.state.get();
837      if (state != null) {
838        if (!(service instanceof NoOpService)) {
839          logger
840              .get()
841              .log(
842                  Level.FINE,
843                  "Service {0} has terminated. Previous state was: {1}",
844                  new Object[] {service, from});
845        }
846        state.transitionService(service, from, TERMINATED);
847      }
848    }
849
850    @Override
851    public void failed(State from, Throwable failure) {
852      ServiceManagerState state = this.state.get();
853      if (state != null) {
854        // Log before the transition, so that if the process exits in response to server failure,
855        // there is a higher likelihood that the cause will be in the logs.
856        boolean log = !(service instanceof NoOpService);
857        /*
858         * We have already exposed startup exceptions to the user in the form of suppressed
859         * exceptions. We don't need to log those exceptions again.
860         */
861        log &= from != State.STARTING;
862        if (log) {
863          logger
864              .get()
865              .log(
866                  Level.SEVERE,
867                  "Service " + service + " has failed in the " + from + " state.",
868                  failure);
869        }
870        state.transitionService(service, from, FAILED);
871      }
872    }
873  }
874
875  /**
876   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
877   * that the {@link ServiceManager} functions properly even when it is managing no services.
878   *
879   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
880   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
881   * logging statements.
882   */
883  private static final class NoOpService extends AbstractService {
884    @Override
885    protected void doStart() {
886      notifyStarted();
887    }
888
889    @Override
890    protected void doStop() {
891      notifyStopped();
892    }
893  }
894
895  /** This is never thrown but only used for logging. */
896  private static final class EmptyServiceManagerWarning extends Throwable {}
897
898  private static final class FailedService extends Throwable {
899    FailedService(Service service) {
900      super(
901          service.toString(),
902          service.failureCause(),
903          false /* don't enable suppression */,
904          false /* don't calculate a stack trace. */);
905    }
906  }
907}