001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.Internal.toNanosSaturated;
025import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
026import static com.google.common.util.concurrent.Service.State.FAILED;
027import static com.google.common.util.concurrent.Service.State.NEW;
028import static com.google.common.util.concurrent.Service.State.RUNNING;
029import static com.google.common.util.concurrent.Service.State.STARTING;
030import static com.google.common.util.concurrent.Service.State.STOPPING;
031import static com.google.common.util.concurrent.Service.State.TERMINATED;
032import static java.util.concurrent.TimeUnit.MILLISECONDS;
033
034import com.google.common.annotations.GwtIncompatible;
035import com.google.common.annotations.J2ktIncompatible;
036import com.google.common.base.Function;
037import com.google.common.base.MoreObjects;
038import com.google.common.base.Stopwatch;
039import com.google.common.collect.Collections2;
040import com.google.common.collect.ImmutableCollection;
041import com.google.common.collect.ImmutableList;
042import com.google.common.collect.ImmutableMap;
043import com.google.common.collect.ImmutableSet;
044import com.google.common.collect.ImmutableSetMultimap;
045import com.google.common.collect.Lists;
046import com.google.common.collect.Maps;
047import com.google.common.collect.MultimapBuilder;
048import com.google.common.collect.Multimaps;
049import com.google.common.collect.Multiset;
050import com.google.common.collect.Ordering;
051import com.google.common.collect.SetMultimap;
052import com.google.common.util.concurrent.Service.State;
053import com.google.errorprone.annotations.CanIgnoreReturnValue;
054import com.google.errorprone.annotations.concurrent.GuardedBy;
055import com.google.j2objc.annotations.J2ObjCIncompatible;
056import com.google.j2objc.annotations.WeakOuter;
057import java.lang.ref.WeakReference;
058import java.time.Duration;
059import java.util.Collections;
060import java.util.EnumSet;
061import java.util.List;
062import java.util.Map;
063import java.util.Map.Entry;
064import java.util.concurrent.Executor;
065import java.util.concurrent.TimeUnit;
066import java.util.concurrent.TimeoutException;
067import java.util.logging.Level;
068import java.util.logging.Logger;
069
070/**
071 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
072 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
073 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
074 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
075 * mechanism.
076 *
077 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
078 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
079 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
080 * when appropriate and {@link #awaitHealthy} will still work as expected.
081 *
082 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
083 *
084 * <pre>{@code
085 * class Server {
086 *   public static void main(String[] args) {
087 *     Set<Service> services = ...;
088 *     ServiceManager manager = new ServiceManager(services);
089 *     manager.addListener(new Listener() {
090 *         public void stopped() {}
091 *         public void healthy() {
092 *           // Services have been initialized and are healthy, start accepting requests...
093 *         }
094 *         public void failure(Service service) {
095 *           // Something failed, at this point we could log it, notify a load balancer, or take
096 *           // some other action.  For now we will just exit.
097 *           System.exit(1);
098 *         }
099 *       },
100 *       MoreExecutors.directExecutor());
101 *
102 *     Runtime.getRuntime().addShutdownHook(new Thread() {
103 *       public void run() {
104 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
105 *         // requests.
106 *         try {
107 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
108 *         } catch (TimeoutException timeout) {
109 *           // stopping timed out
110 *         }
111 *       }
112 *     });
113 *     manager.startAsync();  // start all the services asynchronously
114 *   }
115 * }
116 * }</pre>
117 *
118 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
119 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
120 *
121 * @author Luke Sandberg
122 * @since 14.0
123 */
124@J2ktIncompatible
125@GwtIncompatible
126@ElementTypesAreNonnullByDefault
127public final class ServiceManager implements ServiceManagerBridge {
128  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
129  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
130      new ListenerCallQueue.Event<Listener>() {
131        @Override
132        public void call(Listener listener) {
133          listener.healthy();
134        }
135
136        @Override
137        public String toString() {
138          return "healthy()";
139        }
140      };
141  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
142      new ListenerCallQueue.Event<Listener>() {
143        @Override
144        public void call(Listener listener) {
145          listener.stopped();
146        }
147
148        @Override
149        public String toString() {
150          return "stopped()";
151        }
152      };
153
154  /**
155   * A listener for the aggregate state changes of the services that are under management. Users
156   * that need to listen to more fine-grained events (such as when each particular {@linkplain
157   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
158   * listeners} to each individual service.
159   *
160   * @author Luke Sandberg
161   * @since 15.0 (present as an interface in 14.0)
162   */
163  public abstract static class Listener {
164    /**
165     * Called when the service initially becomes healthy.
166     *
167     * <p>This will be called at most once after all the services have entered the {@linkplain
168     * State#RUNNING running} state. If any services fail during start up or {@linkplain
169     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
170     * started {@linkplain State#RUNNING running} then this method will not be called.
171     */
172    public void healthy() {}
173
174    /**
175     * Called when the all of the component services have reached a terminal state, either
176     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
177     */
178    public void stopped() {}
179
180    /**
181     * Called when a component service has {@linkplain State#FAILED failed}.
182     *
183     * @param service The service that failed.
184     */
185    public void failure(Service service) {}
186  }
187
188  /**
189   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
190   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
191   * be made {@code static} and its instances can be safely constructed and added in the {@link
192   * ServiceManager} constructor without having to close over the partially constructed {@link
193   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
194   */
195  private final ServiceManagerState state;
196
197  private final ImmutableList<Service> services;
198
199  /**
200   * Constructs a new instance for managing the given services.
201   *
202   * @param services The services to manage
203   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
204   *     are any duplicate services.
205   */
206  public ServiceManager(Iterable<? extends Service> services) {
207    ImmutableList<Service> copy = ImmutableList.copyOf(services);
208    if (copy.isEmpty()) {
209      // Having no services causes the manager to behave strangely. Notably, listeners are never
210      // fired. To avoid this we substitute a placeholder service.
211      logger.log(
212          Level.WARNING,
213          "ServiceManager configured with no services.  Is your application configured properly?",
214          new EmptyServiceManagerWarning());
215      copy = ImmutableList.<Service>of(new NoOpService());
216    }
217    this.state = new ServiceManagerState(copy);
218    this.services = copy;
219    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
220    for (Service service : copy) {
221      service.addListener(new ServiceListener(service, stateReference), directExecutor());
222      // We check the state after adding the listener as a way to ensure that our listener was added
223      // to a NEW service.
224      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
225    }
226    // We have installed all of our listeners and after this point any state transition should be
227    // correct.
228    this.state.markReady();
229  }
230
231  /**
232   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
233   * executor. The listener will not have previous state changes replayed, so it is suggested that
234   * listeners are added before any of the managed services are {@linkplain Service#startAsync
235   * started}.
236   *
237   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
238   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
239   * invoked in the same order as the underlying service enters those states. Additionally, at most
240   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
241   * may execute concurrently, and listeners may execute in an order different from the one in which
242   * they were registered.
243   *
244   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
245   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
246   * logged.
247   *
248   * <p>When selecting an executor, note that {@code directExecutor} is dangerous in some cases. See
249   * the discussion in the {@link ListenableFuture#addListener ListenableFuture.addListener}
250   * documentation.
251   *
252   * @param listener the listener to run when the manager changes state
253   * @param executor the executor in which the listeners callback methods will be run.
254   */
255  public void addListener(Listener listener, Executor executor) {
256    state.addListener(listener, executor);
257  }
258
259  /**
260   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
261   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
262   *
263   * @return this
264   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
265   *     method is called.
266   */
267  @CanIgnoreReturnValue
268  public ServiceManager startAsync() {
269    for (Service service : services) {
270      checkState(service.state() == NEW, "Not all services are NEW, cannot start %s", this);
271    }
272    for (Service service : services) {
273      try {
274        state.tryStartTiming(service);
275        service.startAsync();
276      } catch (IllegalStateException e) {
277        // This can happen if the service has already been started or stopped (e.g. by another
278        // service or listener). Our contract says it is safe to call this method if
279        // all services were NEW when it was called, and this has already been verified above, so we
280        // don't propagate the exception.
281        logger.log(Level.WARNING, "Unable to start Service " + service, e);
282      }
283    }
284    return this;
285  }
286
287  /**
288   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
289   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
290   * running} state.
291   *
292   * @throws IllegalStateException if the service manager reaches a state from which it cannot
293   *     become {@linkplain #isHealthy() healthy}.
294   */
295  public void awaitHealthy() {
296    state.awaitHealthy();
297  }
298
299  /**
300   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
301   * than the given time. The manager will become healthy after all the component services have
302   * reached the {@linkplain State#RUNNING running} state.
303   *
304   * @param timeout the maximum time to wait
305   * @throws TimeoutException if not all of the services have finished starting within the deadline
306   * @throws IllegalStateException if the service manager reaches a state from which it cannot
307   *     become {@linkplain #isHealthy() healthy}.
308   * @since 28.0
309   */
310  public void awaitHealthy(Duration timeout) throws TimeoutException {
311    awaitHealthy(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
312  }
313
314  /**
315   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
316   * than the given time. The manager will become healthy after all the component services have
317   * reached the {@linkplain State#RUNNING running} state.
318   *
319   * @param timeout the maximum time to wait
320   * @param unit the time unit of the timeout argument
321   * @throws TimeoutException if not all of the services have finished starting within the deadline
322   * @throws IllegalStateException if the service manager reaches a state from which it cannot
323   *     become {@linkplain #isHealthy() healthy}.
324   */
325  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
326  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
327    state.awaitHealthy(timeout, unit);
328  }
329
330  /**
331   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
332   * being managed.
333   *
334   * @return this
335   */
336  @CanIgnoreReturnValue
337  public ServiceManager stopAsync() {
338    for (Service service : services) {
339      service.stopAsync();
340    }
341    return this;
342  }
343
344  /**
345   * Waits for the all the services to reach a terminal state. After this method returns all
346   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
347   * Service.State#FAILED failed}.
348   */
349  public void awaitStopped() {
350    state.awaitStopped();
351  }
352
353  /**
354   * Waits for the all the services to reach a terminal state for no more than the given time. After
355   * this method returns all services will either be {@linkplain Service.State#TERMINATED
356   * terminated} or {@linkplain Service.State#FAILED failed}.
357   *
358   * @param timeout the maximum time to wait
359   * @throws TimeoutException if not all of the services have stopped within the deadline
360   * @since 28.0
361   */
362  public void awaitStopped(Duration timeout) throws TimeoutException {
363    awaitStopped(toNanosSaturated(timeout), TimeUnit.NANOSECONDS);
364  }
365
366  /**
367   * Waits for the all the services to reach a terminal state for no more than the given time. After
368   * this method returns all services will either be {@linkplain Service.State#TERMINATED
369   * terminated} or {@linkplain Service.State#FAILED failed}.
370   *
371   * @param timeout the maximum time to wait
372   * @param unit the time unit of the timeout argument
373   * @throws TimeoutException if not all of the services have stopped within the deadline
374   */
375  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
376  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
377    state.awaitStopped(timeout, unit);
378  }
379
380  /**
381   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
382   *
383   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
384   * get detailed information about which services are not running.
385   */
386  public boolean isHealthy() {
387    for (Service service : services) {
388      if (!service.isRunning()) {
389        return false;
390      }
391    }
392    return true;
393  }
394
395  /**
396   * Provides a snapshot of the current state of all the services under management.
397   *
398   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
399   * correspond to a point in time view of the services.
400   *
401   * @since 29.0 (present with return type {@code ImmutableMultimap} since 14.0)
402   */
403  @Override
404  public ImmutableSetMultimap<State, Service> servicesByState() {
405    return state.servicesByState();
406  }
407
408  /**
409   * Returns the service load times. This value will only return startup times for services that
410   * have finished starting.
411   *
412   * @return Map of services and their corresponding startup time in millis, the map entries will be
413   *     ordered by startup time.
414   */
415  public ImmutableMap<Service, Long> startupTimes() {
416    return state.startupTimes();
417  }
418
419  /**
420   * Returns the service load times. This value will only return startup times for services that
421   * have finished starting.
422   *
423   * @return Map of services and their corresponding startup time, the map entries will be ordered
424   *     by startup time.
425   * @since 31.0
426   */
427  @J2ObjCIncompatible
428  public ImmutableMap<Service, Duration> startupDurations() {
429    return ImmutableMap.copyOf(
430        Maps.<Service, Long, Duration>transformValues(startupTimes(), Duration::ofMillis));
431  }
432
433  @Override
434  public String toString() {
435    return MoreObjects.toStringHelper(ServiceManager.class)
436        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
437        .toString();
438  }
439
440  /**
441   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
442   * accessed by instances of {@link ServiceListener}.
443   */
444  private static final class ServiceManagerState {
445    final Monitor monitor = new Monitor();
446
447    @GuardedBy("monitor")
448    final SetMultimap<State, Service> servicesByState =
449        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
450
451    @GuardedBy("monitor")
452    final Multiset<State> states = servicesByState.keys();
453
454    @GuardedBy("monitor")
455    final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
456
457    /**
458     * These two booleans are used to mark the state as ready to start.
459     *
460     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
461     * correctly installed
462     *
463     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
464     * transition has been performed.
465     *
466     * <p>Together, they allow us to enforce that all services have their listeners installed prior
467     * to any service performing a transition, then we can fail in the ServiceManager constructor
468     * rather than in a Service.Listener callback.
469     */
470    @GuardedBy("monitor")
471    boolean ready;
472
473    @GuardedBy("monitor")
474    boolean transitioned;
475
476    final int numberOfServices;
477
478    /**
479     * Controls how long to wait for all the services to either become healthy or reach a state from
480     * which it is guaranteed that it can never become healthy.
481     */
482    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
483
484    @WeakOuter
485    final class AwaitHealthGuard extends Monitor.Guard {
486      AwaitHealthGuard() {
487        super(ServiceManagerState.this.monitor);
488      }
489
490      @Override
491      @GuardedBy("ServiceManagerState.this.monitor")
492      public boolean isSatisfied() {
493        // All services have started or some service has terminated/failed.
494        return states.count(RUNNING) == numberOfServices
495            || states.contains(STOPPING)
496            || states.contains(TERMINATED)
497            || states.contains(FAILED);
498      }
499    }
500
501    /** Controls how long to wait for all services to reach a terminal state. */
502    final Monitor.Guard stoppedGuard = new StoppedGuard();
503
504    @WeakOuter
505    final class StoppedGuard extends Monitor.Guard {
506      StoppedGuard() {
507        super(ServiceManagerState.this.monitor);
508      }
509
510      @Override
511      @GuardedBy("ServiceManagerState.this.monitor")
512      public boolean isSatisfied() {
513        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
514      }
515    }
516
517    /** The listeners to notify during a state transition. */
518    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
519
520    /**
521     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
522     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
523     * responsibility to only call {@link #markReady()} if all services were new at the time this
524     * method was called and when all the listeners were installed.
525     */
526    ServiceManagerState(ImmutableCollection<Service> services) {
527      this.numberOfServices = services.size();
528      servicesByState.putAll(NEW, services);
529    }
530
531    /**
532     * Attempts to start the timer immediately prior to the service being started via {@link
533     * Service#startAsync()}.
534     */
535    void tryStartTiming(Service service) {
536      monitor.enter();
537      try {
538        Stopwatch stopwatch = startupTimers.get(service);
539        if (stopwatch == null) {
540          startupTimers.put(service, Stopwatch.createStarted());
541        }
542      } finally {
543        monitor.leave();
544      }
545    }
546
547    /**
548     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
549     * been observed yet.
550     */
551    void markReady() {
552      monitor.enter();
553      try {
554        if (!transitioned) {
555          // nothing has transitioned since construction, good.
556          ready = true;
557        } else {
558          // This should be an extremely rare race condition.
559          List<Service> servicesInBadStates = Lists.newArrayList();
560          for (Service service : servicesByState().values()) {
561            if (service.state() != NEW) {
562              servicesInBadStates.add(service);
563            }
564          }
565          throw new IllegalArgumentException(
566              "Services started transitioning asynchronously before "
567                  + "the ServiceManager was constructed: "
568                  + servicesInBadStates);
569        }
570      } finally {
571        monitor.leave();
572      }
573    }
574
575    void addListener(Listener listener, Executor executor) {
576      listeners.addListener(listener, executor);
577    }
578
579    void awaitHealthy() {
580      monitor.enterWhenUninterruptibly(awaitHealthGuard);
581      try {
582        checkHealthy();
583      } finally {
584        monitor.leave();
585      }
586    }
587
588    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
589      monitor.enter();
590      try {
591        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
592          throw new TimeoutException(
593              "Timeout waiting for the services to become healthy. The "
594                  + "following services have not started: "
595                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
596        }
597        checkHealthy();
598      } finally {
599        monitor.leave();
600      }
601    }
602
603    void awaitStopped() {
604      monitor.enterWhenUninterruptibly(stoppedGuard);
605      monitor.leave();
606    }
607
608    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
609      monitor.enter();
610      try {
611        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
612          throw new TimeoutException(
613              "Timeout waiting for the services to stop. The following "
614                  + "services have not stopped: "
615                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
616        }
617      } finally {
618        monitor.leave();
619      }
620    }
621
622    ImmutableSetMultimap<State, Service> servicesByState() {
623      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
624      monitor.enter();
625      try {
626        for (Entry<State, Service> entry : servicesByState.entries()) {
627          if (!(entry.getValue() instanceof NoOpService)) {
628            builder.put(entry);
629          }
630        }
631      } finally {
632        monitor.leave();
633      }
634      return builder.build();
635    }
636
637    ImmutableMap<Service, Long> startupTimes() {
638      List<Entry<Service, Long>> loadTimes;
639      monitor.enter();
640      try {
641        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
642        // N.B. There will only be an entry in the map if the service has started
643        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
644          Service service = entry.getKey();
645          Stopwatch stopwatch = entry.getValue();
646          if (!stopwatch.isRunning() && !(service instanceof NoOpService)) {
647            loadTimes.add(Maps.immutableEntry(service, stopwatch.elapsed(MILLISECONDS)));
648          }
649        }
650      } finally {
651        monitor.leave();
652      }
653      Collections.sort(
654          loadTimes,
655          Ordering.natural()
656              .onResultOf(
657                  new Function<Entry<Service, Long>, Long>() {
658                    @Override
659                    public Long apply(Entry<Service, Long> input) {
660                      return input.getValue();
661                    }
662                  }));
663      return ImmutableMap.copyOf(loadTimes);
664    }
665
666    /**
667     * Updates the state with the given service transition.
668     *
669     * <p>This method performs the main logic of ServiceManager in the following steps.
670     *
671     * <ol>
672     *   <li>Update the {@link #servicesByState()}
673     *   <li>Update the {@link #startupTimers}
674     *   <li>Based on the new state queue listeners to run
675     *   <li>Run the listeners (outside of the lock)
676     * </ol>
677     */
678    void transitionService(final Service service, State from, State to) {
679      checkNotNull(service);
680      checkArgument(from != to);
681      monitor.enter();
682      try {
683        transitioned = true;
684        if (!ready) {
685          return;
686        }
687        // Update state.
688        checkState(
689            servicesByState.remove(from, service),
690            "Service %s not at the expected location in the state map %s",
691            service,
692            from);
693        checkState(
694            servicesByState.put(to, service),
695            "Service %s in the state map unexpectedly at %s",
696            service,
697            to);
698        // Update the timer
699        Stopwatch stopwatch = startupTimers.get(service);
700        if (stopwatch == null) {
701          // This means the service was started by some means other than ServiceManager.startAsync
702          stopwatch = Stopwatch.createStarted();
703          startupTimers.put(service, stopwatch);
704        }
705        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
706          // N.B. if we miss the STARTING event then we may never record a startup time.
707          stopwatch.stop();
708          if (!(service instanceof NoOpService)) {
709            logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
710          }
711        }
712        // Queue our listeners
713
714        // Did a service fail?
715        if (to == FAILED) {
716          enqueueFailedEvent(service);
717        }
718
719        if (states.count(RUNNING) == numberOfServices) {
720          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
721          // they are not guaranteed to get 'true', because any service could fail right now.
722          enqueueHealthyEvent();
723        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
724          enqueueStoppedEvent();
725        }
726      } finally {
727        monitor.leave();
728        // Run our executors outside of the lock
729        dispatchListenerEvents();
730      }
731    }
732
733    void enqueueStoppedEvent() {
734      listeners.enqueue(STOPPED_EVENT);
735    }
736
737    void enqueueHealthyEvent() {
738      listeners.enqueue(HEALTHY_EVENT);
739    }
740
741    void enqueueFailedEvent(final Service service) {
742      listeners.enqueue(
743          new ListenerCallQueue.Event<Listener>() {
744            @Override
745            public void call(Listener listener) {
746              listener.failure(service);
747            }
748
749            @Override
750            public String toString() {
751              return "failed({service=" + service + "})";
752            }
753          });
754    }
755
756    /** Attempts to execute all the listeners in {@link #listeners}. */
757    void dispatchListenerEvents() {
758      checkState(
759          !monitor.isOccupiedByCurrentThread(),
760          "It is incorrect to execute listeners with the monitor held.");
761      listeners.dispatch();
762    }
763
764    @GuardedBy("monitor")
765    void checkHealthy() {
766      if (states.count(RUNNING) != numberOfServices) {
767        IllegalStateException exception =
768            new IllegalStateException(
769                "Expected to be healthy after starting. The following services are not running: "
770                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
771        for (Service service : servicesByState.get(State.FAILED)) {
772          exception.addSuppressed(new FailedService(service));
773        }
774        throw exception;
775      }
776    }
777  }
778
779  /**
780   * A {@link Service} that wraps another service and times how long it takes for it to start and
781   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
782   * the state transitions.
783   */
784  private static final class ServiceListener extends Service.Listener {
785    final Service service;
786    // We store the state in a weak reference to ensure that if something went wrong while
787    // constructing the ServiceManager we don't pointlessly keep updating the state.
788    final WeakReference<ServiceManagerState> state;
789
790    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
791      this.service = service;
792      this.state = state;
793    }
794
795    @Override
796    public void starting() {
797      ServiceManagerState state = this.state.get();
798      if (state != null) {
799        state.transitionService(service, NEW, STARTING);
800        if (!(service instanceof NoOpService)) {
801          logger.log(Level.FINE, "Starting {0}.", service);
802        }
803      }
804    }
805
806    @Override
807    public void running() {
808      ServiceManagerState state = this.state.get();
809      if (state != null) {
810        state.transitionService(service, STARTING, RUNNING);
811      }
812    }
813
814    @Override
815    public void stopping(State from) {
816      ServiceManagerState state = this.state.get();
817      if (state != null) {
818        state.transitionService(service, from, STOPPING);
819      }
820    }
821
822    @Override
823    public void terminated(State from) {
824      ServiceManagerState state = this.state.get();
825      if (state != null) {
826        if (!(service instanceof NoOpService)) {
827          logger.log(
828              Level.FINE,
829              "Service {0} has terminated. Previous state was: {1}",
830              new Object[] {service, from});
831        }
832        state.transitionService(service, from, TERMINATED);
833      }
834    }
835
836    @Override
837    public void failed(State from, Throwable failure) {
838      ServiceManagerState state = this.state.get();
839      if (state != null) {
840        // Log before the transition, so that if the process exits in response to server failure,
841        // there is a higher likelihood that the cause will be in the logs.
842        boolean log = !(service instanceof NoOpService);
843        /*
844         * We have already exposed startup exceptions to the user in the form of suppressed
845         * exceptions. We don't need to log those exceptions again.
846         */
847        log &= from != State.STARTING;
848        if (log) {
849          logger.log(
850              Level.SEVERE,
851              "Service " + service + " has failed in the " + from + " state.",
852              failure);
853        }
854        state.transitionService(service, from, FAILED);
855      }
856    }
857  }
858
859  /**
860   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
861   * that the {@link ServiceManager} functions properly even when it is managing no services.
862   *
863   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
864   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
865   * logging statements.
866   */
867  private static final class NoOpService extends AbstractService {
868    @Override
869    protected void doStart() {
870      notifyStarted();
871    }
872
873    @Override
874    protected void doStop() {
875      notifyStopped();
876    }
877  }
878
879  /** This is never thrown but only used for logging. */
880  private static final class EmptyServiceManagerWarning extends Throwable {}
881
882  private static final class FailedService extends Throwable {
883    FailedService(Service service) {
884      super(
885          service.toString(),
886          service.failureCause(),
887          false /* don't enable suppression */,
888          false /* don't calculate a stack trace. */);
889    }
890  }
891}