001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package com.google.common.util.concurrent;
017
018import static com.google.common.base.Preconditions.checkArgument;
019import static com.google.common.base.Preconditions.checkNotNull;
020import static com.google.common.base.Preconditions.checkState;
021import static com.google.common.base.Predicates.equalTo;
022import static com.google.common.base.Predicates.in;
023import static com.google.common.base.Predicates.instanceOf;
024import static com.google.common.base.Predicates.not;
025import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
026import static com.google.common.util.concurrent.Service.State.FAILED;
027import static com.google.common.util.concurrent.Service.State.NEW;
028import static com.google.common.util.concurrent.Service.State.RUNNING;
029import static com.google.common.util.concurrent.Service.State.STARTING;
030import static com.google.common.util.concurrent.Service.State.STOPPING;
031import static com.google.common.util.concurrent.Service.State.TERMINATED;
032import static java.util.concurrent.TimeUnit.MILLISECONDS;
033
034import com.google.common.annotations.Beta;
035import com.google.common.base.Function;
036import com.google.common.base.MoreObjects;
037import com.google.common.base.Stopwatch;
038import com.google.common.collect.Collections2;
039import com.google.common.collect.ImmutableCollection;
040import com.google.common.collect.ImmutableList;
041import com.google.common.collect.ImmutableMap;
042import com.google.common.collect.ImmutableMultimap;
043import com.google.common.collect.ImmutableSet;
044import com.google.common.collect.ImmutableSetMultimap;
045import com.google.common.collect.Lists;
046import com.google.common.collect.Maps;
047import com.google.common.collect.MultimapBuilder;
048import com.google.common.collect.Multimaps;
049import com.google.common.collect.Multiset;
050import com.google.common.collect.Ordering;
051import com.google.common.collect.SetMultimap;
052import com.google.common.util.concurrent.ListenerCallQueue.Callback;
053import com.google.common.util.concurrent.Service.State;
054import com.google.j2objc.annotations.WeakOuter;
055
056import java.lang.ref.WeakReference;
057import java.util.ArrayList;
058import java.util.Collections;
059import java.util.EnumSet;
060import java.util.List;
061import java.util.Map;
062import java.util.Map.Entry;
063import java.util.concurrent.Executor;
064import java.util.concurrent.TimeUnit;
065import java.util.concurrent.TimeoutException;
066import java.util.logging.Level;
067import java.util.logging.Logger;
068
069import javax.annotation.concurrent.GuardedBy;
070
071/**
072 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
073 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
074 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
075 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
076 * mechanism.
077 *
078 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
079 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
080 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
081 * when appropriate and {@link #awaitHealthy} will still work as expected.
082 *
083 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
084 * <pre>   {@code
085 * class Server {
086 *   public static void main(String[] args) {
087 *     Set<Service> services = ...;
088 *     ServiceManager manager = new ServiceManager(services);
089 *     manager.addListener(new Listener() {
090 *         public void stopped() {}
091 *         public void healthy() {
092 *           // Services have been initialized and are healthy, start accepting requests...
093 *         }
094 *         public void failure(Service service) {
095 *           // Something failed, at this point we could log it, notify a load balancer, or take
096 *           // some other action.  For now we will just exit.
097 *           System.exit(1);
098 *         }
099 *       },
100 *       MoreExecutors.directExecutor());
101 *
102 *     Runtime.getRuntime().addShutdownHook(new Thread() {
103 *       public void run() {
104 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
105 *         // requests.
106 *         try {
107 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
108 *         } catch (TimeoutException timeout) {
109 *           // stopping timed out
110 *         }
111 *       }
112 *     });
113 *     manager.startAsync();  // start all the services asynchronously
114 *   }
115 * }}</pre>
116 *
117 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
118 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
119 *
120 * @author Luke Sandberg
121 * @since 14.0
122 */
123@Beta
124public final class ServiceManager {
125  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
126  private static final Callback<Listener> HEALTHY_CALLBACK = new Callback<Listener>("healthy()") {
127    @Override void call(Listener listener) {
128      listener.healthy();
129    }
130  };
131  private static final Callback<Listener> STOPPED_CALLBACK = new Callback<Listener>("stopped()") {
132    @Override void call(Listener listener) {
133      listener.stopped();
134    }
135  };
136
137  /**
138   * A listener for the aggregate state changes of the services that are under management. Users
139   * that need to listen to more fine-grained events (such as when each particular {@linkplain
140   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
141   * listeners} to each individual service.
142   *
143   * @author Luke Sandberg
144   * @since 15.0 (present as an interface in 14.0)
145   */
146  @Beta  // Should come out of Beta when ServiceManager does
147  public abstract static class Listener {
148    /**
149     * Called when the service initially becomes healthy.
150     *
151     * <p>This will be called at most once after all the services have entered the
152     * {@linkplain State#RUNNING running} state. If any services fail during start up or
153     * {@linkplain State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other
154     * services have started {@linkplain State#RUNNING running} then this method will not be called.
155     */
156    public void healthy() {}
157
158    /**
159     * Called when the all of the component services have reached a terminal state, either
160     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
161     */
162    public void stopped() {}
163
164    /**
165     * Called when a component service has {@linkplain State#FAILED failed}.
166     *
167     * @param service The service that failed.
168     */
169    public void failure(Service service) {}
170  }
171
172  /**
173   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
174   * service listeners}.  This is extracted into its own object so that {@link ServiceListener}
175   * could be made {@code static} and its instances can be safely constructed and added in the
176   * {@link ServiceManager} constructor without having to close over the partially constructed
177   * {@link ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
178   */
179  private final ServiceManagerState state;
180  private final ImmutableList<Service> services;
181
182  /**
183   * Constructs a new instance for managing the given services.
184   *
185   * @param services The services to manage
186   *
187   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
188   * are any duplicate services.
189   */
190  public ServiceManager(Iterable<? extends Service> services) {
191    ImmutableList<Service> copy = ImmutableList.copyOf(services);
192    if (copy.isEmpty()) {
193      // Having no services causes the manager to behave strangely. Notably, listeners are never
194      // fired.  To avoid this we substitute a placeholder service.
195      logger.log(Level.WARNING,
196          "ServiceManager configured with no services.  Is your application configured properly?",
197          new EmptyServiceManagerWarning());
198      copy = ImmutableList.<Service>of(new NoOpService());
199    }
200    this.state = new ServiceManagerState(copy);
201    this.services = copy;
202    WeakReference<ServiceManagerState> stateReference =
203        new WeakReference<ServiceManagerState>(state);
204    for (Service service : copy) {
205      service.addListener(new ServiceListener(service, stateReference), directExecutor());
206      // We check the state after adding the listener as a way to ensure that our listener was added
207      // to a NEW service.
208      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
209    }
210    // We have installed all of our listeners and after this point any state transition should be
211    // correct.
212    this.state.markReady();
213  }
214
215  /**
216   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
217   * executor. The listener will not have previous state changes replayed, so it is
218   * suggested that listeners are added before any of the managed services are
219   * {@linkplain Service#startAsync started}.
220   *
221   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
222   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
223   * invoked in the same order as the underlying service enters those states. Additionally, at most
224   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
225   * may execute concurrently, and listeners may execute in an order different from the one in which
226   * they were registered.
227   *
228   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
229   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
230   * logged.
231   *
232   * <p> For fast, lightweight listeners that would be safe to execute in any thread, consider
233   * calling {@link #addListener(Listener)}.
234   *
235   * @param listener the listener to run when the manager changes state
236   * @param executor the executor in which the listeners callback methods will be run.
237   */
238  public void addListener(Listener listener, Executor executor) {
239    state.addListener(listener, executor);
240  }
241
242  /**
243   * Registers a {@link Listener} to be run when this {@link ServiceManager} changes state. The
244   * listener will not have previous state changes replayed, so it is suggested that listeners are
245   * added before any of the managed services are {@linkplain Service#startAsync started}.
246   *
247   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
248   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
249   * invoked in the same order as the underlying service enters those states. Additionally, at most
250   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
251   * may execute concurrently, and listeners may execute in an order different from the one in which
252   * they were registered.
253   *
254   * <p>RuntimeExceptions thrown by a listener will be caught and logged.
255   *
256   * @param listener the listener to run when the manager changes state
257   */
258  public void addListener(Listener listener) {
259    state.addListener(listener, directExecutor());
260  }
261
262  /**
263   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed.
264   * It is only valid to call this method if all of the services are {@linkplain State#NEW new}.
265   *
266   * @return this
267   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
268   *     method is called.
269   */
270  public ServiceManager startAsync() {
271    for (Service service : services) {
272      State state = service.state();
273      checkState(state == NEW, "Service %s is %s, cannot start it.", service, state);
274    }
275    for (Service service : services) {
276      try {
277        state.tryStartTiming(service);
278        service.startAsync();
279      } catch (IllegalStateException e) {
280        // This can happen if the service has already been started or stopped (e.g. by another
281        // service or listener). Our contract says it is safe to call this method if
282        // all services were NEW when it was called, and this has already been verified above, so we
283        // don't propagate the exception.
284        logger.log(Level.WARNING, "Unable to start Service " + service, e);
285      }
286    }
287    return this;
288  }
289
290  /**
291   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}.  The manager
292   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
293   * running} state.
294   *
295   * @throws IllegalStateException if the service manager reaches a state from which it cannot
296   *     become {@linkplain #isHealthy() healthy}.
297   */
298  public void awaitHealthy() {
299    state.awaitHealthy();
300  }
301
302  /**
303   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
304   * than the given time.  The manager will become healthy after all the component services have
305   * reached the {@linkplain State#RUNNING running} state.
306   *
307   * @param timeout the maximum time to wait
308   * @param unit the time unit of the timeout argument
309   * @throws TimeoutException if not all of the services have finished starting within the deadline
310   * @throws IllegalStateException if the service manager reaches a state from which it cannot
311   *     become {@linkplain #isHealthy() healthy}.
312   */
313  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
314    state.awaitHealthy(timeout, unit);
315  }
316
317  /**
318   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
319   * being managed.
320   *
321   * @return this
322   */
323  public ServiceManager stopAsync() {
324    for (Service service : services) {
325      service.stopAsync();
326    }
327    return this;
328  }
329
330  /**
331   * Waits for the all the services to reach a terminal state. After this method returns all
332   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
333   * Service.State#FAILED failed}.
334   */
335  public void awaitStopped() {
336    state.awaitStopped();
337  }
338
339  /**
340   * Waits for the all the services to reach a terminal state for no more than the given time. After
341   * this method returns all services will either be {@linkplain Service.State#TERMINATED
342   * terminated} or {@linkplain Service.State#FAILED failed}.
343   *
344   * @param timeout the maximum time to wait
345   * @param unit the time unit of the timeout argument
346   * @throws TimeoutException if not all of the services have stopped within the deadline
347   */
348  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
349    state.awaitStopped(timeout, unit);
350  }
351
352  /**
353   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
354   *
355   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
356   * get detailed information about which services are not running.
357   */
358  public boolean isHealthy() {
359    for (Service service : services) {
360      if (!service.isRunning()) {
361        return false;
362      }
363    }
364    return true;
365  }
366
367  /**
368   * Provides a snapshot of the current state of all the services under management.
369   *
370   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
371   * correspond to a point in time view of the services.
372   */
373  public ImmutableMultimap<State, Service> servicesByState() {
374    return state.servicesByState();
375  }
376
377  /**
378   * Returns the service load times. This value will only return startup times for services that
379   * have finished starting.
380   *
381   * @return Map of services and their corresponding startup time in millis, the map entries will be
382   *     ordered by startup time.
383   */
384  public ImmutableMap<Service, Long> startupTimes() {
385    return state.startupTimes();
386  }
387
388  @Override public String toString() {
389    return MoreObjects.toStringHelper(ServiceManager.class)
390        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
391        .toString();
392  }
393
394  /**
395   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
396   * accessed by instances of {@link ServiceListener}.
397   */
398  private static final class ServiceManagerState {
399    final Monitor monitor = new Monitor();
400
401    @GuardedBy("monitor")
402    final SetMultimap<State, Service> servicesByState =
403        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
404
405    @GuardedBy("monitor")
406    final Multiset<State> states = servicesByState.keys();
407
408    @GuardedBy("monitor")
409    final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
410
411    /**
412     * These two booleans are used to mark the state as ready to start.
413     * {@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
414     *     correctly installed
415     * {@link #transitioned}: is set by {@link #transitionService} to indicate that some transition
416     *     has been performed.
417     *
418     * <p>Together, they allow us to enforce that all services have their listeners installed prior
419     * to any service performing a transition, then we can fail in the ServiceManager constructor
420     * rather than in a Service.Listener callback.
421     */
422    @GuardedBy("monitor")
423    boolean ready;
424
425    @GuardedBy("monitor")
426    boolean transitioned;
427
428    final int numberOfServices;
429
430    /**
431     * Controls how long to wait for all the services to either become healthy or reach a
432     * state from which it is guaranteed that it can never become healthy.
433     */
434    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
435
436    @WeakOuter
437    final class AwaitHealthGuard extends Monitor.Guard {
438      AwaitHealthGuard() {
439        super(ServiceManagerState.this.monitor);
440      }
441
442      @Override public boolean isSatisfied() {
443        // All services have started or some service has terminated/failed.
444        return states.count(RUNNING) == numberOfServices
445            || states.contains(STOPPING)
446            || states.contains(TERMINATED)
447            || states.contains(FAILED);
448      }
449    }
450
451    /**
452     * Controls how long to wait for all services to reach a terminal state.
453     */
454    final Monitor.Guard stoppedGuard = new StoppedGuard();
455
456    @WeakOuter
457    final class StoppedGuard extends Monitor.Guard {
458      StoppedGuard() {
459        super(ServiceManagerState.this.monitor);
460      }
461
462      @Override public boolean isSatisfied() {
463        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
464      }
465    }
466
467    /** The listeners to notify during a state transition. */
468    @GuardedBy("monitor")
469    final List<ListenerCallQueue<Listener>> listeners =
470        Collections.synchronizedList(new ArrayList<ListenerCallQueue<Listener>>());
471
472    /**
473     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
474     * until all the Listeners are installed and {@link #markReady()} is called.  It is our caller's
475     * responsibility to only call {@link #markReady()} if all services were new at the time this
476     * method was called and when all the listeners were installed.
477     */
478    ServiceManagerState(ImmutableCollection<Service> services) {
479      this.numberOfServices = services.size();
480      servicesByState.putAll(NEW, services);
481    }
482
483    /**
484     * Attempts to start the timer immediately prior to the service being started via
485     * {@link Service#startAsync()}.
486     */
487    void tryStartTiming(Service service) {
488      monitor.enter();
489      try {
490        Stopwatch stopwatch = startupTimers.get(service);
491        if (stopwatch == null) {
492          startupTimers.put(service, Stopwatch.createStarted());
493        }
494      } finally {
495        monitor.leave();
496      }
497    }
498
499    /**
500     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
501     * been observed yet.
502     */
503    void markReady() {
504      monitor.enter();
505      try {
506        if (!transitioned) {
507          // nothing has transitioned since construction, good.
508          ready = true;
509        } else {
510          // This should be an extremely rare race condition.
511          List<Service> servicesInBadStates = Lists.newArrayList();
512          for (Service service : servicesByState().values()) {
513            if (service.state() != NEW) {
514              servicesInBadStates.add(service);
515            }
516          }
517          throw new IllegalArgumentException("Services started transitioning asynchronously before "
518              + "the ServiceManager was constructed: " + servicesInBadStates);
519        }
520      } finally {
521        monitor.leave();
522      }
523    }
524
525    void addListener(Listener listener, Executor executor) {
526      checkNotNull(listener, "listener");
527      checkNotNull(executor, "executor");
528      monitor.enter();
529      try {
530        // no point in adding a listener that will never be called
531        if (!stoppedGuard.isSatisfied()) {
532          listeners.add(new ListenerCallQueue<Listener>(listener, executor));
533        }
534      } finally {
535        monitor.leave();
536      }
537    }
538
539    void awaitHealthy() {
540      monitor.enterWhenUninterruptibly(awaitHealthGuard);
541      try {
542        checkHealthy();
543      } finally {
544        monitor.leave();
545      }
546    }
547
548    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
549      monitor.enter();
550      try {
551        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
552          throw new TimeoutException("Timeout waiting for the services to become healthy. The "
553              + "following services have not started: "
554              + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
555        }
556        checkHealthy();
557      } finally {
558        monitor.leave();
559      }
560    }
561
562    void awaitStopped() {
563      monitor.enterWhenUninterruptibly(stoppedGuard);
564      monitor.leave();
565    }
566
567    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
568      monitor.enter();
569      try {
570        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
571          throw new TimeoutException("Timeout waiting for the services to stop. The following "
572              + "services have not stopped: "
573              + Multimaps.filterKeys(servicesByState,
574                  not(in(EnumSet.of(TERMINATED, FAILED)))));
575        }
576      } finally {
577        monitor.leave();
578      }
579    }
580
581    ImmutableMultimap<State, Service> servicesByState() {
582      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
583      monitor.enter();
584      try {
585        for (Entry<State, Service> entry : servicesByState.entries()) {
586          if (!(entry.getValue() instanceof NoOpService)) {
587            builder.put(entry);
588          }
589        }
590      } finally {
591        monitor.leave();
592      }
593      return builder.build();
594    }
595
596    ImmutableMap<Service, Long> startupTimes() {
597      List<Entry<Service, Long>> loadTimes;
598      monitor.enter();
599      try {
600        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
601        // N.B. There will only be an entry in the map if the service has started
602        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
603          Service service = entry.getKey();
604          Stopwatch stopWatch = entry.getValue();
605          if (!stopWatch.isRunning() && !(service instanceof NoOpService)) {
606            loadTimes.add(Maps.immutableEntry(service, stopWatch.elapsed(MILLISECONDS)));
607          }
608        }
609      } finally {
610        monitor.leave();
611      }
612      Collections.sort(loadTimes, Ordering.natural()
613          .onResultOf(new Function<Entry<Service, Long>, Long>() {
614            @Override public Long apply(Map.Entry<Service, Long> input) {
615              return input.getValue();
616            }
617          }));
618      return ImmutableMap.copyOf(loadTimes);
619    }
620
621    /**
622     * Updates the state with the given service transition.
623     *
624     * <p>This method performs the main logic of ServiceManager in the following steps.
625     * <ol>
626     *      <li>Update the {@link #servicesByState()}
627     *      <li>Update the {@link #startupTimers}
628     *      <li>Based on the new state queue listeners to run
629     *      <li>Run the listeners (outside of the lock)
630     * </ol>
631     */
632    void transitionService(final Service service, State from, State to) {
633      checkNotNull(service);
634      checkArgument(from != to);
635      monitor.enter();
636      try {
637        transitioned = true;
638        if (!ready) {
639          return;
640        }
641        // Update state.
642        checkState(servicesByState.remove(from, service),
643            "Service %s not at the expected location in the state map %s", service, from);
644        checkState(servicesByState.put(to, service),
645            "Service %s in the state map unexpectedly at %s", service, to);
646        // Update the timer
647        Stopwatch stopwatch = startupTimers.get(service);
648        if (stopwatch == null) {
649          // This means the service was started by some means other than ServiceManager.startAsync
650          stopwatch = Stopwatch.createStarted();
651          startupTimers.put(service, stopwatch);
652        }
653        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
654          // N.B. if we miss the STARTING event then we may never record a startup time.
655          stopwatch.stop();
656          if (!(service instanceof NoOpService)) {
657            logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
658          }
659        }
660        // Queue our listeners
661
662        // Did a service fail?
663        if (to == FAILED) {
664          fireFailedListeners(service);
665        }
666
667        if (states.count(RUNNING) == numberOfServices) {
668          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
669          // they are not guaranteed to get 'true', because any service could fail right now.
670          fireHealthyListeners();
671        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
672          fireStoppedListeners();
673        }
674      } finally {
675        monitor.leave();
676        // Run our executors outside of the lock
677        executeListeners();
678      }
679    }
680
681    @GuardedBy("monitor")
682    void fireStoppedListeners() {
683      STOPPED_CALLBACK.enqueueOn(listeners);
684    }
685
686    @GuardedBy("monitor")
687    void fireHealthyListeners() {
688      HEALTHY_CALLBACK.enqueueOn(listeners);
689    }
690
691    @GuardedBy("monitor")
692    void fireFailedListeners(final Service service) {
693      new Callback<Listener>("failed({service=" + service + "})") {
694        @Override void call(Listener listener) {
695          listener.failure(service);
696        }
697      }.enqueueOn(listeners);
698    }
699
700    /** Attempts to execute all the listeners in {@link #listeners}. */
701    void executeListeners() {
702      checkState(!monitor.isOccupiedByCurrentThread(),
703          "It is incorrect to execute listeners with the monitor held.");
704      // iterate by index to avoid concurrent modification exceptions
705      for (int i = 0; i < listeners.size(); i++) {
706        listeners.get(i).execute();
707      }
708    }
709
710    @GuardedBy("monitor")
711    void checkHealthy() {
712      if (states.count(RUNNING) != numberOfServices) {
713        IllegalStateException exception = new IllegalStateException(
714            "Expected to be healthy after starting. The following services are not running: "
715                + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
716        throw exception;
717      }
718    }
719  }
720
721  /**
722   * A {@link Service} that wraps another service and times how long it takes for it to start and
723   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)},
724   * to record the state transitions.
725   */
726  private static final class ServiceListener extends Service.Listener {
727    final Service service;
728    // We store the state in a weak reference to ensure that if something went wrong while
729    // constructing the ServiceManager we don't pointlessly keep updating the state.
730    final WeakReference<ServiceManagerState> state;
731
732    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
733      this.service = service;
734      this.state = state;
735    }
736
737    @Override public void starting() {
738      ServiceManagerState state = this.state.get();
739      if (state != null) {
740        state.transitionService(service, NEW, STARTING);
741        if (!(service instanceof NoOpService)) {
742          logger.log(Level.FINE, "Starting {0}.", service);
743        }
744      }
745    }
746
747    @Override public void running() {
748      ServiceManagerState state = this.state.get();
749      if (state != null) {
750        state.transitionService(service, STARTING, RUNNING);
751      }
752    }
753
754    @Override public void stopping(State from) {
755      ServiceManagerState state = this.state.get();
756      if (state != null) {
757        state.transitionService(service, from, STOPPING);
758      }
759    }
760
761    @Override public void terminated(State from) {
762      ServiceManagerState state = this.state.get();
763      if (state != null) {
764        if (!(service instanceof NoOpService)) {
765          logger.log(Level.FINE, "Service {0} has terminated. Previous state was: {1}",
766              new Object[] {service, from});
767        }
768        state.transitionService(service, from, TERMINATED);
769      }
770    }
771
772    @Override public void failed(State from, Throwable failure) {
773      ServiceManagerState state = this.state.get();
774      if (state != null) {
775        // Log before the transition, so that if the process exits in response to server failure,
776        // there is a higher likelihood that the cause will be in the logs.
777        if (!(service instanceof NoOpService) ) {
778          logger.log(Level.SEVERE, "Service " + service + " has failed in the " + from + " state.",
779              failure);
780        }
781        state.transitionService(service, from, FAILED);
782      }
783    }
784  }
785
786  /**
787   * A {@link Service} instance that does nothing.  This is only useful as a placeholder to
788   * ensure that the {@link ServiceManager} functions properly even when it is managing no services.
789   *
790   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
791   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
792   * logging statements.
793   */
794  private static final class NoOpService extends AbstractService {
795    @Override protected void doStart() { notifyStarted(); }
796    @Override protected void doStop() { notifyStopped(); }
797  }
798
799  /** This is never thrown but only used for logging. */
800  private static final class EmptyServiceManagerWarning extends Throwable {}
801}