001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.Internal.toNanosSaturated;
025import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
026import static com.google.common.util.concurrent.Service.State.FAILED;
027import static com.google.common.util.concurrent.Service.State.NEW;
028import static com.google.common.util.concurrent.Service.State.RUNNING;
029import static com.google.common.util.concurrent.Service.State.STARTING;
030import static com.google.common.util.concurrent.Service.State.STOPPING;
031import static com.google.common.util.concurrent.Service.State.TERMINATED;
032import static java.util.concurrent.TimeUnit.MILLISECONDS;
033
034import com.google.common.annotations.GwtIncompatible;
035import com.google.common.annotations.J2ktIncompatible;
036import com.google.common.base.Function;
037import com.google.common.base.MoreObjects;
038import com.google.common.base.Stopwatch;
039import com.google.common.collect.Collections2;
040import com.google.common.collect.ImmutableCollection;
041import com.google.common.collect.ImmutableList;
042import com.google.common.collect.ImmutableMap;
043import com.google.common.collect.ImmutableSet;
044import com.google.common.collect.ImmutableSetMultimap;
045import com.google.common.collect.Lists;
046import com.google.common.collect.Maps;
047import com.google.common.collect.MultimapBuilder;
048import com.google.common.collect.Multimaps;
049import com.google.common.collect.Multiset;
050import com.google.common.collect.Ordering;
051import com.google.common.collect.SetMultimap;
052import com.google.common.util.concurrent.Service.State;
053import com.google.errorprone.annotations.CanIgnoreReturnValue;
054import com.google.errorprone.annotations.concurrent.GuardedBy;
055import com.google.j2objc.annotations.J2ObjCIncompatible;
056import com.google.j2objc.annotations.WeakOuter;
057import java.lang.ref.WeakReference;
058import java.time.Duration;
059import java.util.Collections;
060import java.util.EnumSet;
061import java.util.IdentityHashMap;
062import java.util.List;
063import java.util.Map.Entry;
064import java.util.concurrent.Executor;
065import java.util.concurrent.TimeUnit;
066import java.util.concurrent.TimeoutException;
067import java.util.logging.Level;
068
069/**
070 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
071 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
072 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
073 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
074 * mechanism.
075 *
076 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
077 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
078 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
079 * when appropriate and {@link #awaitHealthy} will still work as expected.
080 *
081 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
082 *
083 * <pre>{@code
084 * class Server {
085 *   public static void main(String[] args) {
086 *     Set<Service> services = ...;
087 *     ServiceManager manager = new ServiceManager(services);
088 *     manager.addListener(new Listener() {
089 *         public void stopped() {}
090 *         public void healthy() {
091 *           // Services have been initialized and are healthy, start accepting requests...
092 *         }
093 *         public void failure(Service service) {
094 *           // Something failed, at this point we could log it, notify a load balancer, or take
095 *           // some other action.  For now we will just exit.
096 *           System.exit(1);
097 *         }
098 *       },
099 *       MoreExecutors.directExecutor());
100 *
101 *     Runtime.getRuntime().addShutdownHook(new Thread() {
102 *       public void run() {
103 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
104 *         // requests.
105 *         try {
106 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
107 *         } catch (TimeoutException timeout) {
108 *           // stopping timed out
109 *         }
110 *       }
111 *     });
112 *     manager.startAsync();  // start all the services asynchronously
113 *   }
114 * }
115 * }</pre>
116 *
117 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
118 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
119 *
120 * @author Luke Sandberg
121 * @since 14.0
122 */
123@J2ktIncompatible
124@GwtIncompatible
125public final class ServiceManager implements ServiceManagerBridge {
126  private static final LazyLogger logger = new LazyLogger(ServiceManager.class);
127  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
128      new ListenerCallQueue.Event<Listener>() {
129        @Override
130        public void call(Listener listener) {
131          listener.healthy();
132        }
133
134        @Override
135        public String toString() {
136          return "healthy()";
137        }
138      };
139  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
140      new ListenerCallQueue.Event<Listener>() {
141        @Override
142        public void call(Listener listener) {
143          listener.stopped();
144        }
145
146        @Override
147        public String toString() {
148          return "stopped()";
149        }
150      };
151
152  /**
153   * A listener for the aggregate state changes of the services that are under management. Users
154   * that need to listen to more fine-grained events (such as when each particular {@linkplain
155   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
156   * listeners} to each individual service.
157   *
158   * @author Luke Sandberg
159   * @since 15.0 (present as an interface in 14.0)
160   */
161  public abstract static class Listener {
162    /** Constructor for use by subclasses. */
163    public Listener() {}
164
165    /**
166     * Called when the service initially becomes healthy.
167     *
168     * <p>This will be called at most once after all the services have entered the {@linkplain
169     * State#RUNNING running} state. If any services fail during start up or {@linkplain
170     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
171     * started {@linkplain State#RUNNING running} then this method will not be called.
172     */
173    public void healthy() {}
174
175    /**
176     * Called when the all of the component services have reached a terminal state, either
177     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
178     */
179    public void stopped() {}
180
181    /**
182     * Called when a component service has {@linkplain State#FAILED failed}.
183     *
184     * @param service The service that failed.
185     */
186    public void failure(Service service) {}
187  }
188
189  /**
190   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
191   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
192   * be made {@code static} and its instances can be safely constructed and added in the {@link
193   * ServiceManager} constructor without having to close over the partially constructed {@link
194   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
195   */
196  private final ServiceManagerState state;
197
198  private final ImmutableList<Service> services;
199
200  /**
201   * Constructs a new instance for managing the given services.
202   *
203   * @param services The services to manage
204   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
205   *     are any duplicate services.
206   */
207  public ServiceManager(Iterable<? extends Service> services) {
208    ImmutableList<Service> copy = ImmutableList.copyOf(services);
209    if (copy.isEmpty()) {
210      // Having no services causes the manager to behave strangely. Notably, listeners are never
211      // fired. To avoid this we substitute a placeholder service.
212      logger
213          .get()
214          .log(
215              Level.WARNING,
216              "ServiceManager configured with no services.  Is your application configured"
217                  + " properly?",
218              new EmptyServiceManagerWarning());
219      copy = ImmutableList.<Service>of(new NoOpService());
220    }
221    this.state = new ServiceManagerState(copy);
222    this.services = copy;
223    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
224    for (Service service : copy) {
225      service.addListener(new ServiceListener(service, stateReference), directExecutor());
226      // We check the state after adding the listener as a way to ensure that our listener was added
227      // to a NEW service.
228      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
229    }
230    // We have installed all of our listeners and after this point any state transition should be
231    // correct.
232    this.state.markReady();
233  }
234
235  /**
236   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
237   * executor. The listener will not have previous state changes replayed, so it is suggested that
238   * listeners are added before any of the managed services are {@linkplain Service#startAsync
239   * started}.
240   *
241   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
242   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
243   * invoked in the same order as the underlying service enters those states. Additionally, at most
244   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
245   * may execute concurrently, and listeners may execute in an order different from the one in which
246   * they were registered.
247   *
248   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
249   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
250   * logged.
251   *
252   * <p>When selecting an executor, note that {@code directExecutor} is dangerous in some cases. See
253   * the discussion in the {@link ListenableFuture#addListener ListenableFuture.addListener}
254   * documentation.
255   *
256   * @param listener the listener to run when the manager changes state
257   * @param executor the executor in which the listeners callback methods will be run.
258   */
259  public void addListener(Listener listener, Executor executor) {
260    state.addListener(listener, executor);
261  }
262
263  /**
264   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
265   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
266   *
267   * @return this
268   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
269   *     method is called.
270   */
271  @CanIgnoreReturnValue
272  public ServiceManager startAsync() {
273    for (Service service : services) {
274      checkState(service.state() == NEW, "Not all services are NEW, cannot start %s", this);
275    }
276    for (Service service : services) {
277      try {
278        state.tryStartTiming(service);
279        service.startAsync();
280      } catch (IllegalStateException e) {
281        // This can happen if the service has already been started or stopped (e.g. by another
282        // service or listener). Our contract says it is safe to call this method if
283        // all services were NEW when it was called, and this has already been verified above, so we
284        // don't propagate the exception.
285        logger.get().log(Level.WARNING, "Unable to start Service " + service, e);
286      }
287    }
288    return this;
289  }
290
291  /**
292   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
293   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
294   * running} state.
295   *
296   * @throws IllegalStateException if the service manager reaches a state from which it cannot
297   *     become {@linkplain #isHealthy() healthy}.
298   */
299  public void awaitHealthy() {
300    state.awaitHealthy();
301  }
302
303  /**
304   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
305   * than the given time. The manager will become healthy after all the component services have
306   * reached the {@linkplain State#RUNNING running} state.
307   *
308   * @param timeout the maximum time to wait
309   * @throws TimeoutException if not all of the services have finished starting within the deadline
310   * @throws IllegalStateException if the service manager reaches a state from which it cannot
311   *     become {@linkplain #isHealthy() healthy}.
312   * @since 33.4.0 (but since 28.0 in the JRE flavor)
313   */
314  @SuppressWarnings("Java7ApiChecker")
315  @IgnoreJRERequirement // Users will use this only if they're already using Duration.
316  public void awaitHealthy(Duration timeout) throws TimeoutException {
317    awaitHealthy(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
318  }
319
320  /**
321   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
322   * than the given time. The manager will become healthy after all the component services have
323   * reached the {@linkplain State#RUNNING running} state.
324   *
325   * @param timeout the maximum time to wait
326   * @param unit the time unit of the timeout argument
327   * @throws TimeoutException if not all of the services have finished starting within the deadline
328   * @throws IllegalStateException if the service manager reaches a state from which it cannot
329   *     become {@linkplain #isHealthy() healthy}.
330   */
331  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
332  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
333    state.awaitHealthy(timeout, unit);
334  }
335
336  /**
337   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
338   * being managed.
339   *
340   * @return this
341   */
342  @CanIgnoreReturnValue
343  public ServiceManager stopAsync() {
344    for (Service service : services) {
345      service.stopAsync();
346    }
347    return this;
348  }
349
350  /**
351   * Waits for the all the services to reach a terminal state. After this method returns all
352   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
353   * Service.State#FAILED failed}.
354   */
355  public void awaitStopped() {
356    state.awaitStopped();
357  }
358
359  /**
360   * Waits for the all the services to reach a terminal state for no more than the given time. After
361   * this method returns all services will either be {@linkplain Service.State#TERMINATED
362   * terminated} or {@linkplain Service.State#FAILED failed}.
363   *
364   * @param timeout the maximum time to wait
365   * @throws TimeoutException if not all of the services have stopped within the deadline
366   * @since 33.4.0 (but since 28.0 in the JRE flavor)
367   */
368  @SuppressWarnings("Java7ApiChecker")
369  @IgnoreJRERequirement // Users will use this only if they're already using Duration.
370  public void awaitStopped(Duration timeout) throws TimeoutException {
371    awaitStopped(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
372  }
373
374  /**
375   * Waits for the all the services to reach a terminal state for no more than the given time. After
376   * this method returns all services will either be {@linkplain Service.State#TERMINATED
377   * terminated} or {@linkplain Service.State#FAILED failed}.
378   *
379   * @param timeout the maximum time to wait
380   * @param unit the time unit of the timeout argument
381   * @throws TimeoutException if not all of the services have stopped within the deadline
382   */
383  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
384  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
385    state.awaitStopped(timeout, unit);
386  }
387
388  /**
389   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
390   *
391   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
392   * get detailed information about which services are not running.
393   */
394  public boolean isHealthy() {
395    for (Service service : services) {
396      if (!service.isRunning()) {
397        return false;
398      }
399    }
400    return true;
401  }
402
403  /**
404   * Provides a snapshot of the current state of all the services under management.
405   *
406   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
407   * correspond to a point in time view of the services.
408   *
409   * @since 29.0 (present with return type {@code ImmutableMultimap} since 14.0)
410   */
411  @Override
412  public ImmutableSetMultimap<State, Service> servicesByState() {
413    return state.servicesByState();
414  }
415
416  /**
417   * Returns the service load times. This value will only return startup times for services that
418   * have finished starting.
419   *
420   * @return Map of services and their corresponding startup time in millis, the map entries will be
421   *     ordered by startup time.
422   */
423  public ImmutableMap<Service, Long> startupTimes() {
424    return state.startupTimes();
425  }
426
427  /**
428   * Returns the service load times. This value will only return startup times for services that
429   * have finished starting.
430   *
431   * @return Map of services and their corresponding startup time, the map entries will be ordered
432   *     by startup time.
433   * @since 33.4.0 (but since 31.0 in the JRE flavor)
434   */
435  @J2ObjCIncompatible
436  @SuppressWarnings("Java7ApiChecker")
437  // If users use this when they shouldn't, we hope that NewApi will catch subsequent Duration calls
438  @IgnoreJRERequirement
439  public ImmutableMap<Service, Duration> startupDurations() {
440    return ImmutableMap.copyOf(
441        Maps.<Service, Long, Duration>transformValues(startupTimes(), Duration::ofMillis));
442  }
443
444  @Override
445  public String toString() {
446    return MoreObjects.toStringHelper(ServiceManager.class)
447        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
448        .toString();
449  }
450
451  /**
452   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
453   * accessed by instances of {@link ServiceListener}.
454   */
455  private static final class ServiceManagerState {
456    final Monitor monitor = new Monitor();
457
458    @GuardedBy("monitor")
459    final SetMultimap<State, Service> servicesByState =
460        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
461
462    @GuardedBy("monitor")
463    final Multiset<State> states = servicesByState.keys();
464
465    @GuardedBy("monitor")
466    final IdentityHashMap<Service, Stopwatch> startupTimers = new IdentityHashMap<>();
467
468    /**
469     * These two booleans are used to mark the state as ready to start.
470     *
471     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
472     * correctly installed
473     *
474     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
475     * transition has been performed.
476     *
477     * <p>Together, they allow us to enforce that all services have their listeners installed prior
478     * to any service performing a transition, then we can fail in the ServiceManager constructor
479     * rather than in a Service.Listener callback.
480     */
481    @GuardedBy("monitor")
482    boolean ready;
483
484    @GuardedBy("monitor")
485    boolean transitioned;
486
487    final int numberOfServices;
488
489    /**
490     * Controls how long to wait for all the services to either become healthy or reach a state from
491     * which it is guaranteed that it can never become healthy.
492     */
493    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
494
495    @WeakOuter
496    final class AwaitHealthGuard extends Monitor.Guard {
497      AwaitHealthGuard() {
498        super(ServiceManagerState.this.monitor);
499      }
500
501      @Override
502      @GuardedBy("ServiceManagerState.this.monitor")
503      public boolean isSatisfied() {
504        // All services have started or some service has terminated/failed.
505        return states.count(RUNNING) == numberOfServices
506            || states.contains(STOPPING)
507            || states.contains(TERMINATED)
508            || states.contains(FAILED);
509      }
510    }
511
512    /** Controls how long to wait for all services to reach a terminal state. */
513    final Monitor.Guard stoppedGuard = new StoppedGuard();
514
515    @WeakOuter
516    final class StoppedGuard extends Monitor.Guard {
517      StoppedGuard() {
518        super(ServiceManagerState.this.monitor);
519      }
520
521      @Override
522      @GuardedBy("ServiceManagerState.this.monitor")
523      public boolean isSatisfied() {
524        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
525      }
526    }
527
528    /** The listeners to notify during a state transition. */
529    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
530
531    /**
532     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
533     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
534     * responsibility to only call {@link #markReady()} if all services were new at the time this
535     * method was called and when all the listeners were installed.
536     */
537    ServiceManagerState(ImmutableCollection<Service> services) {
538      this.numberOfServices = services.size();
539      servicesByState.putAll(NEW, services);
540    }
541
542    /**
543     * Attempts to start the timer immediately prior to the service being started via {@link
544     * Service#startAsync()}.
545     */
546    void tryStartTiming(Service service) {
547      monitor.enter();
548      try {
549        Stopwatch stopwatch = startupTimers.get(service);
550        if (stopwatch == null) {
551          startupTimers.put(service, Stopwatch.createStarted());
552        }
553      } finally {
554        monitor.leave();
555      }
556    }
557
558    /**
559     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
560     * been observed yet.
561     */
562    void markReady() {
563      monitor.enter();
564      try {
565        if (!transitioned) {
566          // nothing has transitioned since construction, good.
567          ready = true;
568        } else {
569          // This should be an extremely rare race condition.
570          List<Service> servicesInBadStates = Lists.newArrayList();
571          for (Service service : servicesByState().values()) {
572            if (service.state() != NEW) {
573              servicesInBadStates.add(service);
574            }
575          }
576          throw new IllegalArgumentException(
577              "Services started transitioning asynchronously before "
578                  + "the ServiceManager was constructed: "
579                  + servicesInBadStates);
580        }
581      } finally {
582        monitor.leave();
583      }
584    }
585
586    void addListener(Listener listener, Executor executor) {
587      listeners.addListener(listener, executor);
588    }
589
590    void awaitHealthy() {
591      monitor.enterWhenUninterruptibly(awaitHealthGuard);
592      try {
593        checkHealthy();
594      } finally {
595        monitor.leave();
596      }
597    }
598
599    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
600      monitor.enter();
601      try {
602        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
603          throw new TimeoutException(
604              "Timeout waiting for the services to become healthy. The "
605                  + "following services have not started: "
606                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
607        }
608        checkHealthy();
609      } finally {
610        monitor.leave();
611      }
612    }
613
614    void awaitStopped() {
615      monitor.enterWhenUninterruptibly(stoppedGuard);
616      monitor.leave();
617    }
618
619    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
620      monitor.enter();
621      try {
622        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
623          throw new TimeoutException(
624              "Timeout waiting for the services to stop. The following "
625                  + "services have not stopped: "
626                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
627        }
628      } finally {
629        monitor.leave();
630      }
631    }
632
633    ImmutableSetMultimap<State, Service> servicesByState() {
634      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
635      monitor.enter();
636      try {
637        for (Entry<State, Service> entry : servicesByState.entries()) {
638          if (!(entry.getValue() instanceof NoOpService)) {
639            builder.put(entry);
640          }
641        }
642      } finally {
643        monitor.leave();
644      }
645      return builder.build();
646    }
647
648    ImmutableMap<Service, Long> startupTimes() {
649      List<Entry<Service, Long>> loadTimes;
650      monitor.enter();
651      try {
652        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
653        // N.B. There will only be an entry in the map if the service has started
654        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
655          Service service = entry.getKey();
656          Stopwatch stopwatch = entry.getValue();
657          if (!stopwatch.isRunning() && !(service instanceof NoOpService)) {
658            loadTimes.add(Maps.immutableEntry(service, stopwatch.elapsed(MILLISECONDS)));
659          }
660        }
661      } finally {
662        monitor.leave();
663      }
664      Collections.sort(
665          loadTimes,
666          Ordering.natural()
667              .onResultOf(
668                  new Function<Entry<Service, Long>, Long>() {
669                    @Override
670                    public Long apply(Entry<Service, Long> input) {
671                      return input.getValue();
672                    }
673                  }));
674      return ImmutableMap.copyOf(loadTimes);
675    }
676
677    /**
678     * Updates the state with the given service transition.
679     *
680     * <p>This method performs the main logic of ServiceManager in the following steps.
681     *
682     * <ol>
683     *   <li>Update the {@link #servicesByState()}
684     *   <li>Update the {@link #startupTimers}
685     *   <li>Based on the new state queue listeners to run
686     *   <li>Run the listeners (outside of the lock)
687     * </ol>
688     */
689    void transitionService(final Service service, State from, State to) {
690      checkNotNull(service);
691      checkArgument(from != to);
692      monitor.enter();
693      try {
694        transitioned = true;
695        if (!ready) {
696          return;
697        }
698        // Update state.
699        checkState(
700            servicesByState.remove(from, service),
701            "Service %s not at the expected location in the state map %s",
702            service,
703            from);
704        checkState(
705            servicesByState.put(to, service),
706            "Service %s in the state map unexpectedly at %s",
707            service,
708            to);
709        // Update the timer
710        Stopwatch stopwatch = startupTimers.get(service);
711        if (stopwatch == null) {
712          // This means the service was started by some means other than ServiceManager.startAsync
713          stopwatch = Stopwatch.createStarted();
714          startupTimers.put(service, stopwatch);
715        }
716        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
717          // N.B. if we miss the STARTING event then we may never record a startup time.
718          stopwatch.stop();
719          if (!(service instanceof NoOpService)) {
720            logger.get().log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
721          }
722        }
723        // Queue our listeners
724
725        // Did a service fail?
726        if (to == FAILED) {
727          enqueueFailedEvent(service);
728        }
729
730        if (states.count(RUNNING) == numberOfServices) {
731          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
732          // they are not guaranteed to get 'true', because any service could fail right now.
733          enqueueHealthyEvent();
734        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
735          enqueueStoppedEvent();
736        }
737      } finally {
738        monitor.leave();
739        // Run our executors outside of the lock
740        dispatchListenerEvents();
741      }
742    }
743
744    void enqueueStoppedEvent() {
745      listeners.enqueue(STOPPED_EVENT);
746    }
747
748    void enqueueHealthyEvent() {
749      listeners.enqueue(HEALTHY_EVENT);
750    }
751
752    void enqueueFailedEvent(final Service service) {
753      listeners.enqueue(
754          new ListenerCallQueue.Event<Listener>() {
755            @Override
756            public void call(Listener listener) {
757              listener.failure(service);
758            }
759
760            @Override
761            public String toString() {
762              return "failed({service=" + service + "})";
763            }
764          });
765    }
766
767    /** Attempts to execute all the listeners in {@link #listeners}. */
768    void dispatchListenerEvents() {
769      checkState(
770          !monitor.isOccupiedByCurrentThread(),
771          "It is incorrect to execute listeners with the monitor held.");
772      listeners.dispatch();
773    }
774
775    @GuardedBy("monitor")
776    void checkHealthy() {
777      if (states.count(RUNNING) != numberOfServices) {
778        IllegalStateException exception =
779            new IllegalStateException(
780                "Expected to be healthy after starting. The following services are not running: "
781                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
782        for (Service service : servicesByState.get(State.FAILED)) {
783          exception.addSuppressed(new FailedService(service));
784        }
785        throw exception;
786      }
787    }
788  }
789
790  /**
791   * A {@link Service} that wraps another service and times how long it takes for it to start and
792   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
793   * the state transitions.
794   */
795  private static final class ServiceListener extends Service.Listener {
796    final Service service;
797    // We store the state in a weak reference to ensure that if something went wrong while
798    // constructing the ServiceManager we don't pointlessly keep updating the state.
799    final WeakReference<ServiceManagerState> state;
800
801    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
802      this.service = service;
803      this.state = state;
804    }
805
806    @Override
807    public void starting() {
808      ServiceManagerState state = this.state.get();
809      if (state != null) {
810        state.transitionService(service, NEW, STARTING);
811        if (!(service instanceof NoOpService)) {
812          logger.get().log(Level.FINE, "Starting {0}.", service);
813        }
814      }
815    }
816
817    @Override
818    public void running() {
819      ServiceManagerState state = this.state.get();
820      if (state != null) {
821        state.transitionService(service, STARTING, RUNNING);
822      }
823    }
824
825    @Override
826    public void stopping(State from) {
827      ServiceManagerState state = this.state.get();
828      if (state != null) {
829        state.transitionService(service, from, STOPPING);
830      }
831    }
832
833    @Override
834    public void terminated(State from) {
835      ServiceManagerState state = this.state.get();
836      if (state != null) {
837        if (!(service instanceof NoOpService)) {
838          logger
839              .get()
840              .log(
841                  Level.FINE,
842                  "Service {0} has terminated. Previous state was: {1}",
843                  new Object[] {service, from});
844        }
845        state.transitionService(service, from, TERMINATED);
846      }
847    }
848
849    @Override
850    public void failed(State from, Throwable failure) {
851      ServiceManagerState state = this.state.get();
852      if (state != null) {
853        // Log before the transition, so that if the process exits in response to server failure,
854        // there is a higher likelihood that the cause will be in the logs.
855        boolean log = !(service instanceof NoOpService);
856        /*
857         * We have already exposed startup exceptions to the user in the form of suppressed
858         * exceptions. We don't need to log those exceptions again.
859         */
860        log &= from != State.STARTING;
861        if (log) {
862          logger
863              .get()
864              .log(
865                  Level.SEVERE,
866                  "Service " + service + " has failed in the " + from + " state.",
867                  failure);
868        }
869        state.transitionService(service, from, FAILED);
870      }
871    }
872  }
873
874  /**
875   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
876   * that the {@link ServiceManager} functions properly even when it is managing no services.
877   *
878   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
879   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
880   * logging statements.
881   */
882  private static final class NoOpService extends AbstractService {
883    @Override
884    protected void doStart() {
885      notifyStarted();
886    }
887
888    @Override
889    protected void doStop() {
890      notifyStopped();
891    }
892  }
893
894  /** This is never thrown but only used for logging. */
895  private static final class EmptyServiceManagerWarning extends Throwable {}
896
897  private static final class FailedService extends Throwable {
898    FailedService(Service service) {
899      super(
900          service.toString(),
901          service.failureCause(),
902          false /* don't enable suppression */,
903          false /* don't calculate a stack trace. */);
904    }
905  }
906}