001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
025import static com.google.common.util.concurrent.Service.State.FAILED;
026import static com.google.common.util.concurrent.Service.State.NEW;
027import static com.google.common.util.concurrent.Service.State.RUNNING;
028import static com.google.common.util.concurrent.Service.State.STARTING;
029import static com.google.common.util.concurrent.Service.State.STOPPING;
030import static com.google.common.util.concurrent.Service.State.TERMINATED;
031import static java.util.concurrent.TimeUnit.MILLISECONDS;
032
033import com.google.common.annotations.GwtIncompatible;
034import com.google.common.base.Function;
035import com.google.common.base.MoreObjects;
036import com.google.common.base.Stopwatch;
037import com.google.common.collect.Collections2;
038import com.google.common.collect.ImmutableCollection;
039import com.google.common.collect.ImmutableList;
040import com.google.common.collect.ImmutableMap;
041import com.google.common.collect.ImmutableSet;
042import com.google.common.collect.ImmutableSetMultimap;
043import com.google.common.collect.Lists;
044import com.google.common.collect.Maps;
045import com.google.common.collect.MultimapBuilder;
046import com.google.common.collect.Multimaps;
047import com.google.common.collect.Multiset;
048import com.google.common.collect.Ordering;
049import com.google.common.collect.SetMultimap;
050import com.google.common.util.concurrent.Service.State;
051import com.google.errorprone.annotations.CanIgnoreReturnValue;
052import com.google.errorprone.annotations.concurrent.GuardedBy;
053import com.google.j2objc.annotations.WeakOuter;
054import java.lang.ref.WeakReference;
055import java.util.Collections;
056import java.util.EnumSet;
057import java.util.List;
058import java.util.Map;
059import java.util.Map.Entry;
060import java.util.concurrent.Executor;
061import java.util.concurrent.TimeUnit;
062import java.util.concurrent.TimeoutException;
063import java.util.logging.Level;
064import java.util.logging.Logger;
065
066/**
067 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
068 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
069 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
070 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
071 * mechanism.
072 *
073 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
074 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
075 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
076 * when appropriate and {@link #awaitHealthy} will still work as expected.
077 *
078 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
079 *
080 * <pre>{@code
081 * class Server {
082 *   public static void main(String[] args) {
083 *     Set<Service> services = ...;
084 *     ServiceManager manager = new ServiceManager(services);
085 *     manager.addListener(new Listener() {
086 *         public void stopped() {}
087 *         public void healthy() {
088 *           // Services have been initialized and are healthy, start accepting requests...
089 *         }
090 *         public void failure(Service service) {
091 *           // Something failed, at this point we could log it, notify a load balancer, or take
092 *           // some other action.  For now we will just exit.
093 *           System.exit(1);
094 *         }
095 *       },
096 *       MoreExecutors.directExecutor());
097 *
098 *     Runtime.getRuntime().addShutdownHook(new Thread() {
099 *       public void run() {
100 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
101 *         // requests.
102 *         try {
103 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
104 *         } catch (TimeoutException timeout) {
105 *           // stopping timed out
106 *         }
107 *       }
108 *     });
109 *     manager.startAsync();  // start all the services asynchronously
110 *   }
111 * }
112 * }</pre>
113 *
114 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
115 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
116 *
117 * @author Luke Sandberg
118 * @since 14.0
119 */
120@GwtIncompatible
121public final class ServiceManager implements ServiceManagerBridge {
122  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
123  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
124      new ListenerCallQueue.Event<Listener>() {
125        @Override
126        public void call(Listener listener) {
127          listener.healthy();
128        }
129
130        @Override
131        public String toString() {
132          return "healthy()";
133        }
134      };
135  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
136      new ListenerCallQueue.Event<Listener>() {
137        @Override
138        public void call(Listener listener) {
139          listener.stopped();
140        }
141
142        @Override
143        public String toString() {
144          return "stopped()";
145        }
146      };
147
148  /**
149   * A listener for the aggregate state changes of the services that are under management. Users
150   * that need to listen to more fine-grained events (such as when each particular {@linkplain
151   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
152   * listeners} to each individual service.
153   *
154   * @author Luke Sandberg
155   * @since 15.0 (present as an interface in 14.0)
156   */
157  public abstract static class Listener {
158    /**
159     * Called when the service initially becomes healthy.
160     *
161     * <p>This will be called at most once after all the services have entered the {@linkplain
162     * State#RUNNING running} state. If any services fail during start up or {@linkplain
163     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
164     * started {@linkplain State#RUNNING running} then this method will not be called.
165     */
166    public void healthy() {}
167
168    /**
169     * Called when the all of the component services have reached a terminal state, either
170     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
171     */
172    public void stopped() {}
173
174    /**
175     * Called when a component service has {@linkplain State#FAILED failed}.
176     *
177     * @param service The service that failed.
178     */
179    public void failure(Service service) {}
180  }
181
182  /**
183   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
184   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
185   * be made {@code static} and its instances can be safely constructed and added in the {@link
186   * ServiceManager} constructor without having to close over the partially constructed {@link
187   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
188   */
189  private final ServiceManagerState state;
190
191  private final ImmutableList<Service> services;
192
193  /**
194   * Constructs a new instance for managing the given services.
195   *
196   * @param services The services to manage
197   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
198   *     are any duplicate services.
199   */
200  public ServiceManager(Iterable<? extends Service> services) {
201    ImmutableList<Service> copy = ImmutableList.copyOf(services);
202    if (copy.isEmpty()) {
203      // Having no services causes the manager to behave strangely. Notably, listeners are never
204      // fired. To avoid this we substitute a placeholder service.
205      logger.log(
206          Level.WARNING,
207          "ServiceManager configured with no services.  Is your application configured properly?",
208          new EmptyServiceManagerWarning());
209      copy = ImmutableList.<Service>of(new NoOpService());
210    }
211    this.state = new ServiceManagerState(copy);
212    this.services = copy;
213    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
214    for (Service service : copy) {
215      service.addListener(new ServiceListener(service, stateReference), directExecutor());
216      // We check the state after adding the listener as a way to ensure that our listener was added
217      // to a NEW service.
218      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
219    }
220    // We have installed all of our listeners and after this point any state transition should be
221    // correct.
222    this.state.markReady();
223  }
224
225  /**
226   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
227   * executor. The listener will not have previous state changes replayed, so it is suggested that
228   * listeners are added before any of the managed services are {@linkplain Service#startAsync
229   * started}.
230   *
231   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
232   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
233   * invoked in the same order as the underlying service enters those states. Additionally, at most
234   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
235   * may execute concurrently, and listeners may execute in an order different from the one in which
236   * they were registered.
237   *
238   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
239   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
240   * logged.
241   *
242   * <p>When selecting an executor, note that {@code directExecutor} is dangerous in some cases. See
243   * the discussion in the {@link ListenableFuture#addListener ListenableFuture.addListener}
244   * documentation.
245   *
246   * @param listener the listener to run when the manager changes state
247   * @param executor the executor in which the listeners callback methods will be run.
248   */
249  public void addListener(Listener listener, Executor executor) {
250    state.addListener(listener, executor);
251  }
252
253  /**
254   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
255   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
256   *
257   * @return this
258   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
259   *     method is called.
260   */
261  @CanIgnoreReturnValue
262  public ServiceManager startAsync() {
263    for (Service service : services) {
264      State state = service.state();
265      checkState(state == NEW, "Service %s is %s, cannot start it.", service, state);
266    }
267    for (Service service : services) {
268      try {
269        state.tryStartTiming(service);
270        service.startAsync();
271      } catch (IllegalStateException e) {
272        // This can happen if the service has already been started or stopped (e.g. by another
273        // service or listener). Our contract says it is safe to call this method if
274        // all services were NEW when it was called, and this has already been verified above, so we
275        // don't propagate the exception.
276        logger.log(Level.WARNING, "Unable to start Service " + service, e);
277      }
278    }
279    return this;
280  }
281
282  /**
283   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
284   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
285   * running} state.
286   *
287   * @throws IllegalStateException if the service manager reaches a state from which it cannot
288   *     become {@linkplain #isHealthy() healthy}.
289   */
290  public void awaitHealthy() {
291    state.awaitHealthy();
292  }
293
294  /**
295   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
296   * than the given time. The manager will become healthy after all the component services have
297   * reached the {@linkplain State#RUNNING running} state.
298   *
299   * @param timeout the maximum time to wait
300   * @param unit the time unit of the timeout argument
301   * @throws TimeoutException if not all of the services have finished starting within the deadline
302   * @throws IllegalStateException if the service manager reaches a state from which it cannot
303   *     become {@linkplain #isHealthy() healthy}.
304   */
305  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
306  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
307    state.awaitHealthy(timeout, unit);
308  }
309
310  /**
311   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
312   * being managed.
313   *
314   * @return this
315   */
316  @CanIgnoreReturnValue
317  public ServiceManager stopAsync() {
318    for (Service service : services) {
319      service.stopAsync();
320    }
321    return this;
322  }
323
324  /**
325   * Waits for the all the services to reach a terminal state. After this method returns all
326   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
327   * Service.State#FAILED failed}.
328   */
329  public void awaitStopped() {
330    state.awaitStopped();
331  }
332
333  /**
334   * Waits for the all the services to reach a terminal state for no more than the given time. After
335   * this method returns all services will either be {@linkplain Service.State#TERMINATED
336   * terminated} or {@linkplain Service.State#FAILED failed}.
337   *
338   * @param timeout the maximum time to wait
339   * @param unit the time unit of the timeout argument
340   * @throws TimeoutException if not all of the services have stopped within the deadline
341   */
342  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
343  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
344    state.awaitStopped(timeout, unit);
345  }
346
347  /**
348   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
349   *
350   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
351   * get detailed information about which services are not running.
352   */
353  public boolean isHealthy() {
354    for (Service service : services) {
355      if (!service.isRunning()) {
356        return false;
357      }
358    }
359    return true;
360  }
361
362  /**
363   * Provides a snapshot of the current state of all the services under management.
364   *
365   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
366   * correspond to a point in time view of the services.
367   *
368   * @since 29.0 (present with return type {@code ImmutableMultimap} since 14.0)
369   */
370  @Override
371  public ImmutableSetMultimap<State, Service> servicesByState() {
372    return state.servicesByState();
373  }
374
375  /**
376   * Returns the service load times. This value will only return startup times for services that
377   * have finished starting.
378   *
379   * @return Map of services and their corresponding startup time in millis, the map entries will be
380   *     ordered by startup time.
381   */
382  public ImmutableMap<Service, Long> startupTimes() {
383    return state.startupTimes();
384  }
385
386  @Override
387  public String toString() {
388    return MoreObjects.toStringHelper(ServiceManager.class)
389        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
390        .toString();
391  }
392
393  /**
394   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
395   * accessed by instances of {@link ServiceListener}.
396   */
397  private static final class ServiceManagerState {
398    final Monitor monitor = new Monitor();
399
400    @GuardedBy("monitor")
401    final SetMultimap<State, Service> servicesByState =
402        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
403
404    @GuardedBy("monitor")
405    final Multiset<State> states = servicesByState.keys();
406
407    @GuardedBy("monitor")
408    final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
409
410    /**
411     * These two booleans are used to mark the state as ready to start.
412     *
413     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
414     * correctly installed
415     *
416     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
417     * transition has been performed.
418     *
419     * <p>Together, they allow us to enforce that all services have their listeners installed prior
420     * to any service performing a transition, then we can fail in the ServiceManager constructor
421     * rather than in a Service.Listener callback.
422     */
423    @GuardedBy("monitor")
424    boolean ready;
425
426    @GuardedBy("monitor")
427    boolean transitioned;
428
429    final int numberOfServices;
430
431    /**
432     * Controls how long to wait for all the services to either become healthy or reach a state from
433     * which it is guaranteed that it can never become healthy.
434     */
435    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
436
437    @WeakOuter
438    final class AwaitHealthGuard extends Monitor.Guard {
439      AwaitHealthGuard() {
440        super(ServiceManagerState.this.monitor);
441      }
442
443      @Override
444      @GuardedBy("ServiceManagerState.this.monitor")
445      public boolean isSatisfied() {
446        // All services have started or some service has terminated/failed.
447        return states.count(RUNNING) == numberOfServices
448            || states.contains(STOPPING)
449            || states.contains(TERMINATED)
450            || states.contains(FAILED);
451      }
452    }
453
454    /** Controls how long to wait for all services to reach a terminal state. */
455    final Monitor.Guard stoppedGuard = new StoppedGuard();
456
457    @WeakOuter
458    final class StoppedGuard extends Monitor.Guard {
459      StoppedGuard() {
460        super(ServiceManagerState.this.monitor);
461      }
462
463      @Override
464      @GuardedBy("ServiceManagerState.this.monitor")
465      public boolean isSatisfied() {
466        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
467      }
468    }
469
470    /** The listeners to notify during a state transition. */
471    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
472
473    /**
474     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
475     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
476     * responsibility to only call {@link #markReady()} if all services were new at the time this
477     * method was called and when all the listeners were installed.
478     */
479    ServiceManagerState(ImmutableCollection<Service> services) {
480      this.numberOfServices = services.size();
481      servicesByState.putAll(NEW, services);
482    }
483
484    /**
485     * Attempts to start the timer immediately prior to the service being started via {@link
486     * Service#startAsync()}.
487     */
488    void tryStartTiming(Service service) {
489      monitor.enter();
490      try {
491        Stopwatch stopwatch = startupTimers.get(service);
492        if (stopwatch == null) {
493          startupTimers.put(service, Stopwatch.createStarted());
494        }
495      } finally {
496        monitor.leave();
497      }
498    }
499
500    /**
501     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
502     * been observed yet.
503     */
504    void markReady() {
505      monitor.enter();
506      try {
507        if (!transitioned) {
508          // nothing has transitioned since construction, good.
509          ready = true;
510        } else {
511          // This should be an extremely rare race condition.
512          List<Service> servicesInBadStates = Lists.newArrayList();
513          for (Service service : servicesByState().values()) {
514            if (service.state() != NEW) {
515              servicesInBadStates.add(service);
516            }
517          }
518          throw new IllegalArgumentException(
519              "Services started transitioning asynchronously before "
520                  + "the ServiceManager was constructed: "
521                  + servicesInBadStates);
522        }
523      } finally {
524        monitor.leave();
525      }
526    }
527
528    void addListener(Listener listener, Executor executor) {
529      listeners.addListener(listener, executor);
530    }
531
532    void awaitHealthy() {
533      monitor.enterWhenUninterruptibly(awaitHealthGuard);
534      try {
535        checkHealthy();
536      } finally {
537        monitor.leave();
538      }
539    }
540
541    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
542      monitor.enter();
543      try {
544        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
545          throw new TimeoutException(
546              "Timeout waiting for the services to become healthy. The "
547                  + "following services have not started: "
548                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
549        }
550        checkHealthy();
551      } finally {
552        monitor.leave();
553      }
554    }
555
556    void awaitStopped() {
557      monitor.enterWhenUninterruptibly(stoppedGuard);
558      monitor.leave();
559    }
560
561    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
562      monitor.enter();
563      try {
564        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
565          throw new TimeoutException(
566              "Timeout waiting for the services to stop. The following "
567                  + "services have not stopped: "
568                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
569        }
570      } finally {
571        monitor.leave();
572      }
573    }
574
575    ImmutableSetMultimap<State, Service> servicesByState() {
576      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
577      monitor.enter();
578      try {
579        for (Entry<State, Service> entry : servicesByState.entries()) {
580          if (!(entry.getValue() instanceof NoOpService)) {
581            builder.put(entry);
582          }
583        }
584      } finally {
585        monitor.leave();
586      }
587      return builder.build();
588    }
589
590    ImmutableMap<Service, Long> startupTimes() {
591      List<Entry<Service, Long>> loadTimes;
592      monitor.enter();
593      try {
594        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
595        // N.B. There will only be an entry in the map if the service has started
596        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
597          Service service = entry.getKey();
598          Stopwatch stopWatch = entry.getValue();
599          if (!stopWatch.isRunning() && !(service instanceof NoOpService)) {
600            loadTimes.add(Maps.immutableEntry(service, stopWatch.elapsed(MILLISECONDS)));
601          }
602        }
603      } finally {
604        monitor.leave();
605      }
606      Collections.sort(
607          loadTimes,
608          Ordering.natural()
609              .onResultOf(
610                  new Function<Entry<Service, Long>, Long>() {
611                    @Override
612                    public Long apply(Entry<Service, Long> input) {
613                      return input.getValue();
614                    }
615                  }));
616      return ImmutableMap.copyOf(loadTimes);
617    }
618
619    /**
620     * Updates the state with the given service transition.
621     *
622     * <p>This method performs the main logic of ServiceManager in the following steps.
623     *
624     * <ol>
625     *   <li>Update the {@link #servicesByState()}
626     *   <li>Update the {@link #startupTimers}
627     *   <li>Based on the new state queue listeners to run
628     *   <li>Run the listeners (outside of the lock)
629     * </ol>
630     */
631    void transitionService(final Service service, State from, State to) {
632      checkNotNull(service);
633      checkArgument(from != to);
634      monitor.enter();
635      try {
636        transitioned = true;
637        if (!ready) {
638          return;
639        }
640        // Update state.
641        checkState(
642            servicesByState.remove(from, service),
643            "Service %s not at the expected location in the state map %s",
644            service,
645            from);
646        checkState(
647            servicesByState.put(to, service),
648            "Service %s in the state map unexpectedly at %s",
649            service,
650            to);
651        // Update the timer
652        Stopwatch stopwatch = startupTimers.get(service);
653        if (stopwatch == null) {
654          // This means the service was started by some means other than ServiceManager.startAsync
655          stopwatch = Stopwatch.createStarted();
656          startupTimers.put(service, stopwatch);
657        }
658        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
659          // N.B. if we miss the STARTING event then we may never record a startup time.
660          stopwatch.stop();
661          if (!(service instanceof NoOpService)) {
662            logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
663          }
664        }
665        // Queue our listeners
666
667        // Did a service fail?
668        if (to == FAILED) {
669          enqueueFailedEvent(service);
670        }
671
672        if (states.count(RUNNING) == numberOfServices) {
673          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
674          // they are not guaranteed to get 'true', because any service could fail right now.
675          enqueueHealthyEvent();
676        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
677          enqueueStoppedEvent();
678        }
679      } finally {
680        monitor.leave();
681        // Run our executors outside of the lock
682        dispatchListenerEvents();
683      }
684    }
685
686    void enqueueStoppedEvent() {
687      listeners.enqueue(STOPPED_EVENT);
688    }
689
690    void enqueueHealthyEvent() {
691      listeners.enqueue(HEALTHY_EVENT);
692    }
693
694    void enqueueFailedEvent(final Service service) {
695      listeners.enqueue(
696          new ListenerCallQueue.Event<Listener>() {
697            @Override
698            public void call(Listener listener) {
699              listener.failure(service);
700            }
701
702            @Override
703            public String toString() {
704              return "failed({service=" + service + "})";
705            }
706          });
707    }
708
709    /** Attempts to execute all the listeners in {@link #listeners}. */
710    void dispatchListenerEvents() {
711      checkState(
712          !monitor.isOccupiedByCurrentThread(),
713          "It is incorrect to execute listeners with the monitor held.");
714      listeners.dispatch();
715    }
716
717    @GuardedBy("monitor")
718    void checkHealthy() {
719      if (states.count(RUNNING) != numberOfServices) {
720        IllegalStateException exception =
721            new IllegalStateException(
722                "Expected to be healthy after starting. The following services are not running: "
723                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
724        throw exception;
725      }
726    }
727  }
728
729  /**
730   * A {@link Service} that wraps another service and times how long it takes for it to start and
731   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
732   * the state transitions.
733   */
734  private static final class ServiceListener extends Service.Listener {
735    final Service service;
736    // We store the state in a weak reference to ensure that if something went wrong while
737    // constructing the ServiceManager we don't pointlessly keep updating the state.
738    final WeakReference<ServiceManagerState> state;
739
740    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
741      this.service = service;
742      this.state = state;
743    }
744
745    @Override
746    public void starting() {
747      ServiceManagerState state = this.state.get();
748      if (state != null) {
749        state.transitionService(service, NEW, STARTING);
750        if (!(service instanceof NoOpService)) {
751          logger.log(Level.FINE, "Starting {0}.", service);
752        }
753      }
754    }
755
756    @Override
757    public void running() {
758      ServiceManagerState state = this.state.get();
759      if (state != null) {
760        state.transitionService(service, STARTING, RUNNING);
761      }
762    }
763
764    @Override
765    public void stopping(State from) {
766      ServiceManagerState state = this.state.get();
767      if (state != null) {
768        state.transitionService(service, from, STOPPING);
769      }
770    }
771
772    @Override
773    public void terminated(State from) {
774      ServiceManagerState state = this.state.get();
775      if (state != null) {
776        if (!(service instanceof NoOpService)) {
777          logger.log(
778              Level.FINE,
779              "Service {0} has terminated. Previous state was: {1}",
780              new Object[] {service, from});
781        }
782        state.transitionService(service, from, TERMINATED);
783      }
784    }
785
786    @Override
787    public void failed(State from, Throwable failure) {
788      ServiceManagerState state = this.state.get();
789      if (state != null) {
790        // Log before the transition, so that if the process exits in response to server failure,
791        // there is a higher likelihood that the cause will be in the logs.
792        boolean log = !(service instanceof NoOpService);
793        if (log) {
794          logger.log(
795              Level.SEVERE,
796              "Service " + service + " has failed in the " + from + " state.",
797              failure);
798        }
799        state.transitionService(service, from, FAILED);
800      }
801    }
802  }
803
804  /**
805   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
806   * that the {@link ServiceManager} functions properly even when it is managing no services.
807   *
808   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
809   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
810   * logging statements.
811   */
812  private static final class NoOpService extends AbstractService {
813    @Override
814    protected void doStart() {
815      notifyStarted();
816    }
817
818    @Override
819    protected void doStop() {
820      notifyStopped();
821    }
822  }
823
824  /** This is never thrown but only used for logging. */
825  private static final class EmptyServiceManagerWarning extends Throwable {}
826}