001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014package com.google.common.util.concurrent;
015
016import static com.google.common.base.Preconditions.checkArgument;
017import static com.google.common.base.Preconditions.checkNotNull;
018import static com.google.common.base.Preconditions.checkState;
019import static com.google.common.base.Predicates.equalTo;
020import static com.google.common.base.Predicates.in;
021import static com.google.common.base.Predicates.instanceOf;
022import static com.google.common.base.Predicates.not;
023import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
024import static com.google.common.util.concurrent.Service.State.FAILED;
025import static com.google.common.util.concurrent.Service.State.NEW;
026import static com.google.common.util.concurrent.Service.State.RUNNING;
027import static com.google.common.util.concurrent.Service.State.STARTING;
028import static com.google.common.util.concurrent.Service.State.STOPPING;
029import static com.google.common.util.concurrent.Service.State.TERMINATED;
030import static java.util.concurrent.TimeUnit.MILLISECONDS;
031
032import com.google.common.annotations.Beta;
033import com.google.common.annotations.GwtIncompatible;
034import com.google.common.base.Function;
035import com.google.common.base.MoreObjects;
036import com.google.common.base.Stopwatch;
037import com.google.common.collect.Collections2;
038import com.google.common.collect.ImmutableCollection;
039import com.google.common.collect.ImmutableList;
040import com.google.common.collect.ImmutableMap;
041import com.google.common.collect.ImmutableMultimap;
042import com.google.common.collect.ImmutableSet;
043import com.google.common.collect.ImmutableSetMultimap;
044import com.google.common.collect.Lists;
045import com.google.common.collect.Maps;
046import com.google.common.collect.MultimapBuilder;
047import com.google.common.collect.Multimaps;
048import com.google.common.collect.Multiset;
049import com.google.common.collect.Ordering;
050import com.google.common.collect.SetMultimap;
051import com.google.common.util.concurrent.Service.State;
052import com.google.errorprone.annotations.CanIgnoreReturnValue;
053import com.google.j2objc.annotations.WeakOuter;
054import java.lang.ref.WeakReference;
055import java.util.Collections;
056import java.util.EnumSet;
057import java.util.List;
058import java.util.Map;
059import java.util.Map.Entry;
060import java.util.concurrent.Executor;
061import java.util.concurrent.TimeUnit;
062import java.util.concurrent.TimeoutException;
063import java.util.logging.Level;
064import java.util.logging.Logger;
065import javax.annotation.concurrent.GuardedBy;
066
067/**
068 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
069 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
070 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
071 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
072 * mechanism.
073 *
074 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
075 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
076 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
077 * when appropriate and {@link #awaitHealthy} will still work as expected.
078 *
079 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
080 * <pre>   {@code
081 * class Server {
082 *   public static void main(String[] args) {
083 *     Set<Service> services = ...;
084 *     ServiceManager manager = new ServiceManager(services);
085 *     manager.addListener(new Listener() {
086 *         public void stopped() {}
087 *         public void healthy() {
088 *           // Services have been initialized and are healthy, start accepting requests...
089 *         }
090 *         public void failure(Service service) {
091 *           // Something failed, at this point we could log it, notify a load balancer, or take
092 *           // some other action.  For now we will just exit.
093 *           System.exit(1);
094 *         }
095 *       },
096 *       MoreExecutors.directExecutor());
097 *
098 *     Runtime.getRuntime().addShutdownHook(new Thread() {
099 *       public void run() {
100 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
101 *         // requests.
102 *         try {
103 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
104 *         } catch (TimeoutException timeout) {
105 *           // stopping timed out
106 *         }
107 *       }
108 *     });
109 *     manager.startAsync();  // start all the services asynchronously
110 *   }
111 * }}</pre>
112 *
113 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
114 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
115 *
116 * @author Luke Sandberg
117 * @since 14.0
118 */
119@Beta
120@GwtIncompatible
121public final class ServiceManager {
122  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
123  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
124      new ListenerCallQueue.Event<Listener>() {
125        @Override
126        public void call(Listener listener) {
127          listener.healthy();
128        }
129
130        @Override
131        public String toString() {
132          return "healthy()";
133        }
134      };
135  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
136      new ListenerCallQueue.Event<Listener>() {
137        @Override
138        public void call(Listener listener) {
139          listener.stopped();
140        }
141
142        @Override
143        public String toString() {
144          return "stopped()";
145        }
146      };
147
148  /**
149   * A listener for the aggregate state changes of the services that are under management. Users
150   * that need to listen to more fine-grained events (such as when each particular
151   * {@linkplain Service service} starts, or terminates), should attach {@linkplain Service.Listener
152   * service listeners} to each individual service.
153   *
154   * @author Luke Sandberg
155   * @since 15.0 (present as an interface in 14.0)
156   */
157  @Beta // Should come out of Beta when ServiceManager does
158  public abstract static class Listener {
159    /**
160     * Called when the service initially becomes healthy.
161     *
162     * <p>This will be called at most once after all the services have entered the
163     * {@linkplain State#RUNNING running} state. If any services fail during start up or
164     * {@linkplain State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other
165     * services have started {@linkplain State#RUNNING running} then this method will not be called.
166     */
167    public void healthy() {}
168
169    /**
170     * Called when the all of the component services have reached a terminal state, either
171     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
172     */
173    public void stopped() {}
174
175    /**
176     * Called when a component service has {@linkplain State#FAILED failed}.
177     *
178     * @param service The service that failed.
179     */
180    public void failure(Service service) {}
181  }
182
183  /**
184   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
185   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
186   * be made {@code static} and its instances can be safely constructed and added in the
187   * {@link ServiceManager} constructor without having to close over the partially constructed
188   * {@link ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
189   */
190  private final ServiceManagerState state;
191  private final ImmutableList<Service> services;
192
193  /**
194   * Constructs a new instance for managing the given services.
195   *
196   * @param services The services to manage
197   *
198   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
199   *     are any duplicate services.
200   */
201  public ServiceManager(Iterable<? extends Service> services) {
202    ImmutableList<Service> copy = ImmutableList.copyOf(services);
203    if (copy.isEmpty()) {
204      // Having no services causes the manager to behave strangely. Notably, listeners are never
205      // fired. To avoid this we substitute a placeholder service.
206      logger.log(
207          Level.WARNING,
208          "ServiceManager configured with no services.  Is your application configured properly?",
209          new EmptyServiceManagerWarning());
210      copy = ImmutableList.<Service>of(new NoOpService());
211    }
212    this.state = new ServiceManagerState(copy);
213    this.services = copy;
214    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
215    for (Service service : copy) {
216      service.addListener(new ServiceListener(service, stateReference), directExecutor());
217      // We check the state after adding the listener as a way to ensure that our listener was added
218      // to a NEW service.
219      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
220    }
221    // We have installed all of our listeners and after this point any state transition should be
222    // correct.
223    this.state.markReady();
224  }
225
226  /**
227   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
228   * executor. The listener will not have previous state changes replayed, so it is suggested that
229   * listeners are added before any of the managed services are {@linkplain Service#startAsync
230   * started}.
231   *
232   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
233   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
234   * invoked in the same order as the underlying service enters those states. Additionally, at most
235   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
236   * may execute concurrently, and listeners may execute in an order different from the one in which
237   * they were registered.
238   *
239   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
240   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
241   * logged.
242   *
243   * <p>For fast, lightweight listeners that would be safe to execute in any thread, consider
244   * calling {@link #addListener(Listener)}.
245   *
246   * @param listener the listener to run when the manager changes state
247   * @param executor the executor in which the listeners callback methods will be run.
248   */
249  public void addListener(Listener listener, Executor executor) {
250    state.addListener(listener, executor);
251  }
252
253  /**
254   * Registers a {@link Listener} to be run when this {@link ServiceManager} changes state. The
255   * listener will not have previous state changes replayed, so it is suggested that listeners are
256   * added before any of the managed services are {@linkplain Service#startAsync started}.
257   *
258   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
259   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
260   * invoked in the same order as the underlying service enters those states. Additionally, at most
261   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
262   * may execute concurrently, and listeners may execute in an order different from the one in which
263   * they were registered.
264   *
265   * <p>RuntimeExceptions thrown by a listener will be caught and logged.
266   *
267   * @param listener the listener to run when the manager changes state
268   */
269  public void addListener(Listener listener) {
270    state.addListener(listener, directExecutor());
271  }
272
273  /**
274   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
275   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
276   *
277   * @return this
278   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
279   *     method is called.
280   */
281  @CanIgnoreReturnValue
282  public ServiceManager startAsync() {
283    for (Service service : services) {
284      State state = service.state();
285      checkState(state == NEW, "Service %s is %s, cannot start it.", service, state);
286    }
287    for (Service service : services) {
288      try {
289        state.tryStartTiming(service);
290        service.startAsync();
291      } catch (IllegalStateException e) {
292        // This can happen if the service has already been started or stopped (e.g. by another
293        // service or listener). Our contract says it is safe to call this method if
294        // all services were NEW when it was called, and this has already been verified above, so we
295        // don't propagate the exception.
296        logger.log(Level.WARNING, "Unable to start Service " + service, e);
297      }
298    }
299    return this;
300  }
301
302  /**
303   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
304   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
305   * running} state.
306   *
307   * @throws IllegalStateException if the service manager reaches a state from which it cannot
308   *     become {@linkplain #isHealthy() healthy}.
309   */
310  public void awaitHealthy() {
311    state.awaitHealthy();
312  }
313
314  /**
315   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
316   * than the given time. The manager will become healthy after all the component services have
317   * reached the {@linkplain State#RUNNING running} state.
318   *
319   * @param timeout the maximum time to wait
320   * @param unit the time unit of the timeout argument
321   * @throws TimeoutException if not all of the services have finished starting within the deadline
322   * @throws IllegalStateException if the service manager reaches a state from which it cannot
323   *     become {@linkplain #isHealthy() healthy}.
324   */
325  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
326    state.awaitHealthy(timeout, unit);
327  }
328
329  /**
330   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
331   * being managed.
332   *
333   * @return this
334   */
335  @CanIgnoreReturnValue
336  public ServiceManager stopAsync() {
337    for (Service service : services) {
338      service.stopAsync();
339    }
340    return this;
341  }
342
343  /**
344   * Waits for the all the services to reach a terminal state. After this method returns all
345   * services will either be {@linkplain Service.State#TERMINATED terminated} or
346   * {@linkplain Service.State#FAILED failed}.
347   */
348  public void awaitStopped() {
349    state.awaitStopped();
350  }
351
352  /**
353   * Waits for the all the services to reach a terminal state for no more than the given time. After
354   * this method returns all services will either be {@linkplain Service.State#TERMINATED
355   * terminated} or {@linkplain Service.State#FAILED failed}.
356   *
357   * @param timeout the maximum time to wait
358   * @param unit the time unit of the timeout argument
359   * @throws TimeoutException if not all of the services have stopped within the deadline
360   */
361  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
362    state.awaitStopped(timeout, unit);
363  }
364
365  /**
366   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
367   *
368   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
369   * get detailed information about which services are not running.
370   */
371  public boolean isHealthy() {
372    for (Service service : services) {
373      if (!service.isRunning()) {
374        return false;
375      }
376    }
377    return true;
378  }
379
380  /**
381   * Provides a snapshot of the current state of all the services under management.
382   *
383   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
384   * correspond to a point in time view of the services.
385   */
386  public ImmutableMultimap<State, Service> servicesByState() {
387    return state.servicesByState();
388  }
389
390  /**
391   * Returns the service load times. This value will only return startup times for services that
392   * have finished starting.
393   *
394   * @return Map of services and their corresponding startup time in millis, the map entries will be
395   *     ordered by startup time.
396   */
397  public ImmutableMap<Service, Long> startupTimes() {
398    return state.startupTimes();
399  }
400
401  @Override
402  public String toString() {
403    return MoreObjects.toStringHelper(ServiceManager.class)
404        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
405        .toString();
406  }
407
408  /**
409   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
410   * accessed by instances of {@link ServiceListener}.
411   */
412  private static final class ServiceManagerState {
413    final Monitor monitor = new Monitor();
414
415    @GuardedBy("monitor")
416    final SetMultimap<State, Service> servicesByState =
417        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
418
419    @GuardedBy("monitor")
420    final Multiset<State> states = servicesByState.keys();
421
422    @GuardedBy("monitor")
423    final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
424
425    /**
426     * These two booleans are used to mark the state as ready to start.
427     *
428     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
429     * correctly installed
430     *
431     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
432     * transition has been performed.
433     *
434     * <p>Together, they allow us to enforce that all services have their listeners installed prior
435     * to any service performing a transition, then we can fail in the ServiceManager constructor
436     * rather than in a Service.Listener callback.
437     */
438    @GuardedBy("monitor")
439    boolean ready;
440
441    @GuardedBy("monitor")
442    boolean transitioned;
443
444    final int numberOfServices;
445
446    /**
447     * Controls how long to wait for all the services to either become healthy or reach a state from
448     * which it is guaranteed that it can never become healthy.
449     */
450    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
451
452    @WeakOuter
453    final class AwaitHealthGuard extends Monitor.Guard {
454      AwaitHealthGuard() {
455        super(ServiceManagerState.this.monitor);
456      }
457
458      @Override
459      @GuardedBy("ServiceManagerState.this.monitor")
460      public boolean isSatisfied() {
461        // All services have started or some service has terminated/failed.
462        return states.count(RUNNING) == numberOfServices
463            || states.contains(STOPPING)
464            || states.contains(TERMINATED)
465            || states.contains(FAILED);
466      }
467    }
468
469    /**
470     * Controls how long to wait for all services to reach a terminal state.
471     */
472    final Monitor.Guard stoppedGuard = new StoppedGuard();
473
474    @WeakOuter
475    final class StoppedGuard extends Monitor.Guard {
476      StoppedGuard() {
477        super(ServiceManagerState.this.monitor);
478      }
479
480      @Override
481      @GuardedBy("ServiceManagerState.this.monitor")
482      public boolean isSatisfied() {
483        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
484      }
485    }
486
487    /** The listeners to notify during a state transition. */
488    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
489
490    /**
491     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
492     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
493     * responsibility to only call {@link #markReady()} if all services were new at the time this
494     * method was called and when all the listeners were installed.
495     */
496    ServiceManagerState(ImmutableCollection<Service> services) {
497      this.numberOfServices = services.size();
498      servicesByState.putAll(NEW, services);
499    }
500
501    /**
502     * Attempts to start the timer immediately prior to the service being started via
503     * {@link Service#startAsync()}.
504     */
505    void tryStartTiming(Service service) {
506      monitor.enter();
507      try {
508        Stopwatch stopwatch = startupTimers.get(service);
509        if (stopwatch == null) {
510          startupTimers.put(service, Stopwatch.createStarted());
511        }
512      } finally {
513        monitor.leave();
514      }
515    }
516
517    /**
518     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
519     * been observed yet.
520     */
521    void markReady() {
522      monitor.enter();
523      try {
524        if (!transitioned) {
525          // nothing has transitioned since construction, good.
526          ready = true;
527        } else {
528          // This should be an extremely rare race condition.
529          List<Service> servicesInBadStates = Lists.newArrayList();
530          for (Service service : servicesByState().values()) {
531            if (service.state() != NEW) {
532              servicesInBadStates.add(service);
533            }
534          }
535          throw new IllegalArgumentException(
536              "Services started transitioning asynchronously before "
537                  + "the ServiceManager was constructed: "
538                  + servicesInBadStates);
539        }
540      } finally {
541        monitor.leave();
542      }
543    }
544
545    void addListener(Listener listener, Executor executor) {
546      listeners.addListener(listener, executor);
547    }
548
549    void awaitHealthy() {
550      monitor.enterWhenUninterruptibly(awaitHealthGuard);
551      try {
552        checkHealthy();
553      } finally {
554        monitor.leave();
555      }
556    }
557
558    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
559      monitor.enter();
560      try {
561        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
562          throw new TimeoutException(
563              "Timeout waiting for the services to become healthy. The "
564                  + "following services have not started: "
565                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
566        }
567        checkHealthy();
568      } finally {
569        monitor.leave();
570      }
571    }
572
573    void awaitStopped() {
574      monitor.enterWhenUninterruptibly(stoppedGuard);
575      monitor.leave();
576    }
577
578    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
579      monitor.enter();
580      try {
581        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
582          throw new TimeoutException(
583              "Timeout waiting for the services to stop. The following "
584                  + "services have not stopped: "
585                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
586        }
587      } finally {
588        monitor.leave();
589      }
590    }
591
592    ImmutableMultimap<State, Service> servicesByState() {
593      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
594      monitor.enter();
595      try {
596        for (Entry<State, Service> entry : servicesByState.entries()) {
597          if (!(entry.getValue() instanceof NoOpService)) {
598            builder.put(entry);
599          }
600        }
601      } finally {
602        monitor.leave();
603      }
604      return builder.build();
605    }
606
607    ImmutableMap<Service, Long> startupTimes() {
608      List<Entry<Service, Long>> loadTimes;
609      monitor.enter();
610      try {
611        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
612        // N.B. There will only be an entry in the map if the service has started
613        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
614          Service service = entry.getKey();
615          Stopwatch stopWatch = entry.getValue();
616          if (!stopWatch.isRunning() && !(service instanceof NoOpService)) {
617            loadTimes.add(Maps.immutableEntry(service, stopWatch.elapsed(MILLISECONDS)));
618          }
619        }
620      } finally {
621        monitor.leave();
622      }
623      Collections.sort(
624          loadTimes,
625          Ordering.natural()
626              .onResultOf(
627                  new Function<Entry<Service, Long>, Long>() {
628                    @Override
629                    public Long apply(Entry<Service, Long> input) {
630                      return input.getValue();
631                    }
632                  }));
633      return ImmutableMap.copyOf(loadTimes);
634    }
635
636    /**
637     * Updates the state with the given service transition.
638     *
639     * <p>This method performs the main logic of ServiceManager in the following steps.
640     * <ol>
641     * <li>Update the {@link #servicesByState()}
642     * <li>Update the {@link #startupTimers}
643     * <li>Based on the new state queue listeners to run
644     * <li>Run the listeners (outside of the lock)
645     * </ol>
646     */
647    void transitionService(final Service service, State from, State to) {
648      checkNotNull(service);
649      checkArgument(from != to);
650      monitor.enter();
651      try {
652        transitioned = true;
653        if (!ready) {
654          return;
655        }
656        // Update state.
657        checkState(
658            servicesByState.remove(from, service),
659            "Service %s not at the expected location in the state map %s",
660            service,
661            from);
662        checkState(
663            servicesByState.put(to, service),
664            "Service %s in the state map unexpectedly at %s",
665            service,
666            to);
667        // Update the timer
668        Stopwatch stopwatch = startupTimers.get(service);
669        if (stopwatch == null) {
670          // This means the service was started by some means other than ServiceManager.startAsync
671          stopwatch = Stopwatch.createStarted();
672          startupTimers.put(service, stopwatch);
673        }
674        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
675          // N.B. if we miss the STARTING event then we may never record a startup time.
676          stopwatch.stop();
677          if (!(service instanceof NoOpService)) {
678            logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
679          }
680        }
681        // Queue our listeners
682
683        // Did a service fail?
684        if (to == FAILED) {
685          enqueueFailedEvent(service);
686        }
687
688        if (states.count(RUNNING) == numberOfServices) {
689          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
690          // they are not guaranteed to get 'true', because any service could fail right now.
691          enqueueHealthyEvent();
692        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
693          enqueueStoppedEvent();
694        }
695      } finally {
696        monitor.leave();
697        // Run our executors outside of the lock
698        dispatchListenerEvents();
699      }
700    }
701
702    void enqueueStoppedEvent() {
703      listeners.enqueue(STOPPED_EVENT);
704    }
705
706    void enqueueHealthyEvent() {
707      listeners.enqueue(HEALTHY_EVENT);
708    }
709
710    void enqueueFailedEvent(final Service service) {
711      listeners.enqueue(
712          new ListenerCallQueue.Event<Listener>() {
713            @Override
714            public void call(Listener listener) {
715              listener.failure(service);
716            }
717
718            @Override
719            public String toString() {
720              return "failed({service=" + service + "})";
721            }
722          });
723    }
724
725    /** Attempts to execute all the listeners in {@link #listeners}. */
726    void dispatchListenerEvents() {
727      checkState(
728          !monitor.isOccupiedByCurrentThread(),
729          "It is incorrect to execute listeners with the monitor held.");
730      listeners.dispatch();
731    }
732
733    @GuardedBy("monitor")
734    void checkHealthy() {
735      if (states.count(RUNNING) != numberOfServices) {
736        IllegalStateException exception =
737            new IllegalStateException(
738                "Expected to be healthy after starting. The following services are not running: "
739                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
740        for (Service service : servicesByState.get(State.FAILED)) {
741          exception.addSuppressed(new FailedService(service));
742        }
743        throw exception;
744      }
745    }
746  }
747
748  /**
749   * A {@link Service} that wraps another service and times how long it takes for it to start and
750   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
751   * the state transitions.
752   */
753  private static final class ServiceListener extends Service.Listener {
754    final Service service;
755    // We store the state in a weak reference to ensure that if something went wrong while
756    // constructing the ServiceManager we don't pointlessly keep updating the state.
757    final WeakReference<ServiceManagerState> state;
758
759    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
760      this.service = service;
761      this.state = state;
762    }
763
764    @Override
765    public void starting() {
766      ServiceManagerState state = this.state.get();
767      if (state != null) {
768        state.transitionService(service, NEW, STARTING);
769        if (!(service instanceof NoOpService)) {
770          logger.log(Level.FINE, "Starting {0}.", service);
771        }
772      }
773    }
774
775    @Override
776    public void running() {
777      ServiceManagerState state = this.state.get();
778      if (state != null) {
779        state.transitionService(service, STARTING, RUNNING);
780      }
781    }
782
783    @Override
784    public void stopping(State from) {
785      ServiceManagerState state = this.state.get();
786      if (state != null) {
787        state.transitionService(service, from, STOPPING);
788      }
789    }
790
791    @Override
792    public void terminated(State from) {
793      ServiceManagerState state = this.state.get();
794      if (state != null) {
795        if (!(service instanceof NoOpService)) {
796          logger.log(
797              Level.FINE,
798              "Service {0} has terminated. Previous state was: {1}",
799              new Object[] {service, from});
800        }
801        state.transitionService(service, from, TERMINATED);
802      }
803    }
804
805    @Override
806    public void failed(State from, Throwable failure) {
807      ServiceManagerState state = this.state.get();
808      if (state != null) {
809        // Log before the transition, so that if the process exits in response to server failure,
810        // there is a higher likelihood that the cause will be in the logs.
811        boolean log = !(service instanceof NoOpService);
812        /*
813         * We have already exposed startup exceptions to the user in the form of suppressed
814         * exceptions. We don't need to log those exceptions again.
815         */
816        log &= from != State.STARTING;
817        if (log) {
818          logger.log(
819              Level.SEVERE,
820              "Service " + service + " has failed in the " + from + " state.",
821              failure);
822        }
823        state.transitionService(service, from, FAILED);
824      }
825    }
826  }
827
828  /**
829   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
830   * that the {@link ServiceManager} functions properly even when it is managing no services.
831   *
832   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
833   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
834   * logging statements.
835   */
836  private static final class NoOpService extends AbstractService {
837    @Override
838    protected void doStart() {
839      notifyStarted();
840    }
841
842    @Override
843    protected void doStop() {
844      notifyStopped();
845    }
846  }
847
848  /** This is never thrown but only used for logging. */
849  private static final class EmptyServiceManagerWarning extends Throwable {}
850
851  private static final class FailedService extends Throwable {
852    FailedService(Service service) {
853      super(
854          service.toString(),
855          service.failureCause(),
856          false /* don't enable suppression */,
857          false /* don't calculate a stack trace. */);
858    }
859  }
860}