001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package com.google.common.util.concurrent;
017
018import static com.google.common.base.Preconditions.checkArgument;
019import static com.google.common.base.Preconditions.checkNotNull;
020import static com.google.common.base.Preconditions.checkState;
021import static com.google.common.base.Predicates.instanceOf;
022import static com.google.common.base.Predicates.not;
023import static java.util.concurrent.TimeUnit.MILLISECONDS;
024
025import com.google.common.annotations.Beta;
026import com.google.common.base.Function;
027import com.google.common.base.Objects;
028import com.google.common.base.Stopwatch;
029import com.google.common.collect.Collections2;
030import com.google.common.collect.ImmutableList;
031import com.google.common.collect.ImmutableMap;
032import com.google.common.collect.ImmutableMultimap;
033import com.google.common.collect.Lists;
034import com.google.common.collect.Maps;
035import com.google.common.collect.Ordering;
036import com.google.common.util.concurrent.Service.State;
037
038import java.util.Collections;
039import java.util.List;
040import java.util.Map;
041import java.util.Map.Entry;
042import java.util.Set;
043import java.util.concurrent.Executor;
044import java.util.concurrent.TimeUnit;
045import java.util.concurrent.TimeoutException;
046import java.util.logging.Level;
047import java.util.logging.Logger;
048
049import javax.annotation.concurrent.GuardedBy;
050import javax.annotation.concurrent.Immutable;
051import javax.inject.Inject;
052import javax.inject.Singleton;
053
054/**
055 * A manager for monitoring and controlling a set of {@link Service services}. This class provides
056 * methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
057 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
058 * Additionally, users can monitor state transitions with the {@link Listener listener} mechanism.
059 *
060 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
061 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
062 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
063 * when appropriate and {@link #awaitHealthy} will still work as expected.
064 *
065 * <p>Here is a simple example of how to use a {@link ServiceManager} to start a server.
066 * <pre>   {@code
067 * class Server {
068 *   public static void main(String[] args) {
069 *     Set<Service> services = ...;
070 *     ServiceManager manager = new ServiceManager(services);
071 *     manager.addListener(new Listener() {
072 *         public void stopped() {}
073 *         public void healthy() {
074 *           // Services have been initialized and are healthy, start accepting requests...
075 *         }
076 *         public void failure(Service service) {
077 *           // Something failed, at this point we could log it, notify a load balancer, or take
078 *           // some other action.  For now we will just exit.
079 *           System.exit(1);
080 *         }
081 *       },
082 *       MoreExecutors.sameThreadExecutor());
083 *
084 *     Runtime.getRuntime().addShutdownHook(new Thread() {
085 *       public void run() {
086 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown 
087 *         // requests.
088 *         try {
089 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
090 *         } catch (TimeoutException timeout) {
091 *           // stopping timed out
092 *         }
093 *       }
094 *     });
095 *     manager.startAsync();  // start all the services asynchronously
096 *   }
097 * }}</pre>
098 *
099 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
100 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
101 *
102 * @author Luke Sandberg
103 * @since 14.0
104 */
105@Beta
106@Singleton
107public final class ServiceManager {
108  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
109  
110  /**
111   * A listener for the aggregate state changes of the services that are under management. Users
112   * that need to listen to more fine-grained events (such as when each particular
113   * {@link Service service} starts, or terminates), should attach {@link Service.Listener service
114   * listeners} to each individual service.
115   * 
116   * @author Luke Sandberg
117   * @since 15.0 (present as an interface in 14.0)
118   */
119  @Beta  // Should come out of Beta when ServiceManager does
120  public abstract static class Listener {
121    /** 
122     * Called when the service initially becomes healthy.
123     * 
124     * <p>This will be called at most once after all the services have entered the 
125     * {@linkplain State#RUNNING running} state. If any services fail during start up or 
126     * {@linkplain State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other 
127     * services have started {@linkplain State#RUNNING running} then this method will not be called.
128     */
129    public void healthy() {}
130    
131    /** 
132     * Called when the all of the component services have reached a terminal state, either 
133     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
134     */
135    public void stopped() {}
136    
137    /** 
138     * Called when a component service has {@linkplain State#FAILED failed}.
139     * 
140     * @param service The service that failed.
141     */
142    public void failure(Service service) {}
143  }
144  
145  /**
146   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener 
147   * service listeners}.  This is extracted into its own object so that {@link ServiceListener} 
148   * could be made {@code static} and its instances can be safely constructed and added in the 
149   * {@link ServiceManager} constructor without having to close over the partially constructed 
150   * {@link ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
151   */
152  private final ServiceManagerState state;
153  private final ImmutableMap<Service, ServiceListener> services;
154  
155  /**
156   * Constructs a new instance for managing the given services.
157   * 
158   * @param services The services to manage
159   * 
160   * @throws IllegalArgumentException if not all services are {@link State#NEW new} or if there are
161   *     any duplicate services.
162   */
163  public ServiceManager(Iterable<? extends Service> services) {
164    ImmutableList<Service> copy = ImmutableList.copyOf(services);
165    if (copy.isEmpty()) {
166      // Having no services causes the manager to behave strangely. Notably, listeners are never 
167      // fired.  To avoid this we substitute a placeholder service.
168      logger.log(Level.WARNING, 
169          "ServiceManager configured with no services.  Is your application configured properly?", 
170          new EmptyServiceManagerWarning());
171      copy = ImmutableList.<Service>of(new NoOpService());
172    }
173    this.state = new ServiceManagerState(copy.size());
174    ImmutableMap.Builder<Service, ServiceListener> builder = ImmutableMap.builder();
175    Executor executor = MoreExecutors.sameThreadExecutor();
176    for (Service service : copy) {
177      ServiceListener listener = new ServiceListener(service, state);
178      service.addListener(listener, executor);
179      // We check the state after adding the listener as a way to ensure that our listener was added
180      // to a NEW service.
181      checkArgument(service.state() == State.NEW, "Can only manage NEW services, %s", service);
182      builder.put(service, listener);
183    }
184    this.services = builder.build();
185  }
186  
187  /**
188   * Constructs a new instance for managing the given services. This constructor is provided so that
189   * dependency injection frameworks can inject instances of {@link ServiceManager}.
190   * 
191   * @param services The services to manage
192   * 
193   * @throws IllegalStateException if not all services are {@link State#NEW new}.
194   */
195  @Inject ServiceManager(Set<Service> services) {
196    this((Iterable<Service>) services);
197  }
198  
199  /**
200   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given 
201   * executor. The listener will not have previous state changes replayed, so it is 
202   * suggested that listeners are added before any of the managed services are 
203   * {@linkplain Service#start started}.
204   *
205   * <p>There is no guaranteed ordering of execution of listeners, but any listener added through 
206   * this method is guaranteed to be called whenever there is a state change.
207   *
208   * <p>Exceptions thrown by a listener will be propagated up to the executor. Any exception thrown 
209   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException} or an exception 
210   * thrown by {@linkplain MoreExecutors#sameThreadExecutor inline execution}) will be caught and
211   * logged.
212   * 
213   * <p> For fast, lightweight listeners that would be safe to execute in any thread, consider 
214   * calling {@link #addListener(Listener)}.
215   * 
216   * @param listener the listener to run when the manager changes state
217   * @param executor the executor in which the listeners callback methods will be run.
218   */
219  public void addListener(Listener listener, Executor executor) {
220    state.addListener(listener, executor);
221  }
222
223  /**
224   * Registers a {@link Listener} to be run when this {@link ServiceManager} changes state. The 
225   * listener will not have previous state changes replayed, so it is suggested that listeners are 
226   * added before any of the managed services are {@linkplain Service#start started}.
227   *
228   * <p>There is no guaranteed ordering of execution of listeners, but any listener added through 
229   * this method is guaranteed to be called whenever there is a state change.
230   *
231   * <p>Exceptions thrown by a listener will be will be caught and logged.
232   * 
233   * @param listener the listener to run when the manager changes state
234   */
235  public void addListener(Listener listener) {
236    state.addListener(listener, MoreExecutors.sameThreadExecutor());
237  }
238
239  /**
240   * Initiates service {@linkplain Service#start startup} on all the services being managed.  It is
241   * only valid to call this method if all of the services are {@linkplain State#NEW new}.
242   * 
243   * @return this
244   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the 
245   *     method is called.
246   */
247  public ServiceManager startAsync() {
248    for (Map.Entry<Service, ServiceListener> entry : services.entrySet()) {
249      Service service = entry.getKey();
250      State state = service.state();
251      checkState(state == State.NEW, "Service %s is %s, cannot start it.", service, 
252          state);
253    }
254    for (ServiceListener service : services.values()) {
255      try {
256        service.start();
257      } catch (IllegalStateException e) {
258        // This can happen if the service has already been started or stopped (e.g. by another 
259        // service or listener). Our contract says it is safe to call this method if
260        // all services were NEW when it was called, and this has already been verified above, so we
261        // don't propagate the exception.
262        logger.log(Level.WARNING, "Unable to start Service " + service.service, e);
263      }
264    }
265    return this;
266  }
267  
268  /**
269   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}.  The manager
270   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
271   * running} state.  
272   * 
273   * @throws IllegalStateException if the service manager reaches a state from which it cannot 
274   *     become {@linkplain #isHealthy() healthy}.
275   */
276  public void awaitHealthy() {
277    state.awaitHealthy();
278    checkState(isHealthy(), "Expected to be healthy after starting");
279  }
280  
281  /**
282   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more 
283   * than the given time.  The manager will become healthy after all the component services have 
284   * reached the {@linkplain State#RUNNING running} state. 
285   *
286   * @param timeout the maximum time to wait
287   * @param unit the time unit of the timeout argument
288   * @throws TimeoutException if not all of the services have finished starting within the deadline
289   * @throws IllegalStateException if the service manager reaches a state from which it cannot 
290   *     become {@linkplain #isHealthy() healthy}.
291   */
292  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
293    if (!state.awaitHealthy(timeout, unit)) {
294      // It would be nice to tell the caller who we are still waiting on, and this information is 
295      // likely to be in servicesByState(), however due to race conditions we can't actually tell 
296      // which services are holding up healthiness. The current set of NEW or STARTING services is
297      // likely to point out the culprit, but may not.  If we really wanted to solve this we could
298      // change state to track exactly which services have started and then we could accurately 
299      // report on this. But it is only for logging so we likely don't care.
300      throw new TimeoutException("Timeout waiting for the services to become healthy.");
301    }
302    checkState(isHealthy(), "Expected to be healthy after starting");
303  }
304
305  /**
306   * Initiates service {@linkplain Service#stop shutdown} if necessary on all the services being 
307   * managed. 
308   *    
309   * @return this
310   */
311  public ServiceManager stopAsync() {
312    for (Service service : services.keySet()) {
313      service.stop();
314    }
315    return this;
316  }
317 
318  /**
319   * Waits for the all the services to reach a terminal state. After this method returns all
320   * services will either be {@link Service.State#TERMINATED terminated} or 
321   * {@link Service.State#FAILED failed}
322   */
323  public void awaitStopped() {
324    state.awaitStopped();
325  }
326  
327  /**
328   * Waits for the all the services to reach a terminal state for no more than the given time. After
329   * this method returns all services will either be {@link Service.State#TERMINATED terminated} or 
330   * {@link Service.State#FAILED failed}
331   *
332   * @param timeout the maximum time to wait
333   * @param unit the time unit of the timeout argument
334   * @throws TimeoutException if not all of the services have stopped within the deadline
335   */
336  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
337    if (!state.awaitStopped(timeout, unit)) {
338      throw new TimeoutException("Timeout waiting for the services to stop.");
339    }
340  }
341  
342  /**
343   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.  
344   * 
345   * <p>Users who want more detailed information should use the {@link #servicesByState} method to 
346   * get detailed information about which services are not running.
347   */
348  public boolean isHealthy() {
349    for (Service service : services.keySet()) {
350      if (!service.isRunning()) {
351        return false;
352      }
353    }
354    return true;
355  }
356  
357  /**
358   * Provides a snapshot of the current state of all the services under management.
359   * 
360   * <p>N.B. This snapshot it not guaranteed to be consistent, i.e. the set of states returned may
361   * not correspond to any particular point in time view of the services. 
362   */
363  public ImmutableMultimap<State, Service> servicesByState() {
364    ImmutableMultimap.Builder<State, Service> builder = ImmutableMultimap.builder();
365    for (Service service : services.keySet()) {
366      if (!(service instanceof NoOpService)) {
367        builder.put(service.state(), service);
368      }
369    }
370    return builder.build();
371  }
372  
373  /**
374   * Returns the service load times. This value will only return startup times for services that
375   * have finished starting.
376   *
377   * @return Map of services and their corresponding startup time in millis, the map entries will be
378   *     ordered by startup time.
379   */
380  public ImmutableMap<Service, Long> startupTimes() { 
381    List<Entry<Service, Long>> loadTimes = Lists.newArrayListWithCapacity(services.size());
382    for (Map.Entry<Service, ServiceListener> entry : services.entrySet()) {
383      Service service = entry.getKey();
384      State state = service.state();
385      if (state != State.NEW & state != State.STARTING & !(service instanceof NoOpService)) {
386        loadTimes.add(Maps.immutableEntry(service, entry.getValue().startupTimeMillis()));
387      }
388    }
389    Collections.sort(loadTimes, Ordering.<Long>natural()
390        .onResultOf(new Function<Entry<Service, Long>, Long>() {
391          @Override public Long apply(Map.Entry<Service, Long> input) {
392            return input.getValue();
393          }
394        }));
395    ImmutableMap.Builder<Service, Long> builder = ImmutableMap.builder();
396    for (Entry<Service, Long> entry : loadTimes) {
397      builder.put(entry);
398    }
399    return builder.build();
400  }
401  
402  @Override public String toString() {
403    return Objects.toStringHelper(ServiceManager.class)
404        .add("services", Collections2.filter(services.keySet(), not(instanceOf(NoOpService.class))))
405        .toString();
406  }
407  
408  /**
409   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be 
410   * accessed by instances of {@link ServiceListener}.
411   */
412  private static final class ServiceManagerState {
413    final Monitor monitor = new Monitor();
414    final int numberOfServices;
415    /** The number of services that have not finished starting up. */
416    @GuardedBy("monitor")
417    int unstartedServices;
418    /** The number of services that have not reached a terminal state. */
419    @GuardedBy("monitor")
420    int unstoppedServices;
421    /** 
422     * Controls how long to wait for all the service manager to either become healthy or reach a 
423     * state where it is guaranteed that it can never become healthy.
424     */
425    final Monitor.Guard awaitHealthGuard = new Monitor.Guard(monitor) {
426      @Override public boolean isSatisfied() {
427        // All services have started or some service has terminated/failed.
428        return unstartedServices == 0 | unstoppedServices != numberOfServices;
429      }
430    };
431    /**
432     * Controls how long to wait for all services to reach a terminal state.
433     */
434    final Monitor.Guard stoppedGuard = new Monitor.Guard(monitor) {
435      @Override public boolean isSatisfied() {
436        return unstoppedServices == 0;
437      }
438    };
439    /** The listeners to notify during a state transition. */
440    @GuardedBy("monitor")
441    final List<ListenerExecutorPair> listeners = Lists.newArrayList();
442    /**
443     * The queue of listeners that are waiting to be executed.
444     *
445     * <p>Enqueue operations should be protected by {@link #monitor} while dequeue operations are
446     * not protected. Holding {@link #monitor} while enqueuing ensures that listeners in the queue
447     * are in the correct order and {@link ExecutionQueue} ensures that they are executed in the 
448     * correct order.
449     */
450    @GuardedBy("monitor")
451    final ExecutionQueue queuedListeners = new ExecutionQueue();
452    
453    ServiceManagerState(int numberOfServices) {
454      this.numberOfServices = numberOfServices;
455      this.unstoppedServices = numberOfServices;
456      this.unstartedServices = numberOfServices;
457    }
458    
459    void addListener(Listener listener, Executor executor) {
460      checkNotNull(listener, "listener");
461      checkNotNull(executor, "executor");
462      monitor.enter();
463      try {
464        // no point in adding a listener that will never be called
465        if (unstartedServices > 0 || unstoppedServices > 0) {
466          listeners.add(new ListenerExecutorPair(listener, executor));
467        }
468      } finally {
469        monitor.leave();
470      }
471    }
472    
473    void awaitHealthy() {
474      monitor.enterWhenUninterruptibly(awaitHealthGuard);
475      monitor.leave();
476    }
477    
478    boolean awaitHealthy(long timeout, TimeUnit unit) {
479      if (monitor.enterWhenUninterruptibly(awaitHealthGuard, timeout, unit)) {
480        monitor.leave();
481        return true;
482      }
483      return false;
484    }
485    
486    void awaitStopped() {
487      monitor.enterWhenUninterruptibly(stoppedGuard);
488      monitor.leave();
489    }
490    
491    boolean awaitStopped(long timeout, TimeUnit unit) {
492      if (monitor.enterWhenUninterruptibly(stoppedGuard, timeout, unit)) {
493        monitor.leave();
494        return true;
495      }
496      return false;
497    }
498    
499    /**
500     * This should be called when a service finishes starting up.
501     * 
502     * @param currentlyHealthy whether or not the service that finished starting was healthy at the 
503     *        time that it finished starting. 
504     */
505    @GuardedBy("monitor")
506    private void serviceFinishedStarting(Service service, boolean currentlyHealthy) {
507      checkState(unstartedServices > 0, 
508          "All services should have already finished starting but %s just finished.", service);
509      unstartedServices--;
510      if (currentlyHealthy && unstartedServices == 0 && unstoppedServices == numberOfServices) {
511        // This means that the manager is currently healthy, or at least it should have been
512        // healthy at some point from some perspective. Calling isHealthy is not currently
513        // guaranteed to return true because any service could fail right now. However, the
514        // happens-before relationship enforced by the monitor ensures that this method was called
515        // before either serviceTerminated or serviceFailed, so we know that the manager was at 
516        // least healthy for some period of time. Furthermore we are guaranteed that this call to
517        // healthy() will be before any call to terminated() or failure(Service) on the listener.
518        // So it is correct to execute the listener's health() callback.
519        for (final ListenerExecutorPair pair : listeners) {
520          queuedListeners.add(new Runnable() {
521            @Override public void run() {
522              pair.listener.healthy();
523            }
524          }, pair.executor);
525        }
526      }
527    }
528    
529    /**
530     * This should be called when a service is {@linkplain State#TERMINATED terminated}.
531     */
532    @GuardedBy("monitor")
533    private void serviceTerminated(Service service) {
534      serviceStopped(service);
535    }
536    
537    /**
538     * This should be called when a service is {@linkplain State#FAILED failed}.
539     */
540    @GuardedBy("monitor")
541    private void serviceFailed(final Service service) {
542      for (final ListenerExecutorPair pair : listeners) {
543        queuedListeners.add(new Runnable() {
544          @Override public void run() {
545            pair.listener.failure(service);
546          }
547        }, pair.executor);
548      }
549      serviceStopped(service);
550    }
551    
552    /**
553     * Should be called whenever a service reaches a terminal state (
554     * {@linkplain State#TERMINATED terminated} or 
555     * {@linkplain State#FAILED failed}).
556     */
557    @GuardedBy("monitor")
558    private void serviceStopped(Service service) {
559      checkState(unstoppedServices > 0, 
560          "All services should have already stopped but %s just stopped.", service);
561      unstoppedServices--;
562      if (unstoppedServices == 0) {
563        checkState(unstartedServices == 0, 
564            "All services are stopped but %d services haven't finished starting", 
565            unstartedServices);
566        for (final ListenerExecutorPair pair : listeners) {
567          queuedListeners.add(new Runnable() {
568            @Override public void run() {
569              pair.listener.stopped();
570            }
571          }, pair.executor);
572        }
573        // no more listeners could possibly be called, so clear them out
574        listeners.clear();
575      }
576    }
577
578    /** Attempts to execute all the listeners in {@link #queuedListeners}. */
579    private void executeListeners() {
580      checkState(!monitor.isOccupiedByCurrentThread(), 
581          "It is incorrect to execute listeners with the monitor held.");
582      queuedListeners.execute();
583    }
584  }
585
586  /**
587   * A {@link Service} that wraps another service and times how long it takes for it to start and 
588   * also calls the {@link ServiceManagerState#serviceFinishedStarting}, 
589   * {@link ServiceManagerState#serviceTerminated} and {@link ServiceManagerState#serviceFailed}
590   * according to its current state.
591   */
592  private static final class ServiceListener extends Service.Listener {
593    @GuardedBy("watch")  // AFAICT Stopwatch is not thread safe so we need to protect accesses
594    final Stopwatch watch = Stopwatch.createUnstarted();
595    final Service service;
596    final ServiceManagerState state;
597    
598    /**
599     * @param service the service that 
600     */
601    ServiceListener(Service service, ServiceManagerState state) {
602      this.service = service;
603      this.state = state;
604    }
605    
606    @Override public void starting() {
607      // This can happen if someone besides the ServiceManager starts the service, in this case
608      // our timings may be inaccurate.
609      startTimer();
610    }
611    
612    @Override public void running() {
613      state.monitor.enter();
614      try {
615        finishedStarting(true);
616      } finally {
617        state.monitor.leave();
618        state.executeListeners();
619      }
620    }
621    
622    @Override public void stopping(State from) {
623      if (from == State.STARTING) {
624        state.monitor.enter();
625        try {
626          finishedStarting(false);
627        } finally {
628          state.monitor.leave();
629          state.executeListeners();
630        }
631      }
632    }
633    
634    @Override public void terminated(State from) {
635      if (!(service instanceof NoOpService)) {
636        logger.log(Level.FINE, "Service {0} has terminated. Previous state was: {1}", 
637            new Object[] {service, from});
638      }
639      state.monitor.enter();
640      try {
641        if (from == State.NEW) {
642          // startTimer is idempotent, so this is safe to call and it may be necessary if no one has
643          // started the timer yet.
644          startTimer(); 
645          finishedStarting(false);
646        }
647        state.serviceTerminated(service);
648      } finally {
649        state.monitor.leave();
650        state.executeListeners();
651      }
652    }
653    
654    @Override public void failed(State from, Throwable failure) {
655      logger.log(Level.SEVERE, "Service " + service + " has failed in the " + from + " state.", 
656          failure);
657      state.monitor.enter();
658      try {
659        if (from == State.STARTING) {
660          finishedStarting(false);
661        }
662        state.serviceFailed(service);
663      } finally {
664        state.monitor.leave();
665        state.executeListeners();
666      }
667    }
668    
669    /** 
670     * Stop the stopwatch, log the startup time and decrement the startup latch
671     *  
672     * @param currentlyHealthy whether or not the service that finished starting is currently 
673     *        healthy 
674     */
675    @GuardedBy("monitor")
676    void finishedStarting(boolean currentlyHealthy) {
677      synchronized (watch) {
678        watch.stop();
679        if (!(service instanceof NoOpService)) {
680          logger.log(Level.FINE, "Started {0} in {1} ms.", 
681              new Object[] {service, startupTimeMillis()});
682        }
683      }
684      state.serviceFinishedStarting(service, currentlyHealthy);
685    }
686    
687    void start() {
688      startTimer();
689      service.startAsync();
690    }
691  
692    /** Start the timer if it hasn't been started. */
693    void startTimer() {
694      synchronized (watch) {
695        if (!watch.isRunning()) { // only start the watch once.
696          watch.start();
697          if (!(service instanceof NoOpService)) {
698            logger.log(Level.FINE, "Starting {0}.", service);
699          }
700        }
701      }
702    }
703    
704    /** Returns the amount of time it took for the service to finish starting in milliseconds. */
705    long startupTimeMillis() {
706      synchronized (watch) {
707        return watch.elapsed(MILLISECONDS);
708      }
709    }
710  }
711  
712  /** Simple value object binding a listener to its executor. */
713  @Immutable private static final class ListenerExecutorPair {
714    final Listener listener;
715    final Executor executor;
716    
717    ListenerExecutorPair(Listener listener, Executor executor) {
718      this.listener = listener;
719      this.executor = executor;
720    }
721  }
722  
723  /**
724   * A {@link Service} instance that does nothing.  This is only useful as a placeholder to
725   * ensure that the {@link ServiceManager} functions properly even when it is managing no services.
726   * 
727   * <p>The use of this class is considered an implementation detail of the class and as such it is
728   * excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all 
729   * logging statements.
730   */
731  private static final class NoOpService extends AbstractService {
732    @Override protected void doStart() { notifyStarted(); }
733    @Override protected void doStop() { notifyStopped(); }
734  }
735  
736  /** This is never thrown but only used for logging. */
737  private static final class EmptyServiceManagerWarning extends Throwable {}
738}