TransactionManager can be initialized but unusable if other components are not initialized

[pkoenig10]

See internal link: https://g.p.b/foundry/mp/pull/12089#discussion_r1462867

AtlasDB has a number of components which need to be initialized (one of which is the transaction manager). If the initialization does not succeed on the first attempt, then another attempt will be scheduled for 10 second later. https://github.com/palantir/atlasdb/blob/b0e43310490ac1b04d309e1854ffc52ac403bfc6/atlasdb-commons/src/main/java/com/palantir/async/initializer/AsyncInitializer.java#L89-L102

Unfortunately, the initializations are all independent. This means that the transaction manager can be initialized but unusable if another required component (like the PersistentTimestampServiceImpl in the example above) is not yet initialized.

com.palantir.exception.NotInitializedException: The PersistentTimestampService is not initialized yet
	at com.palantir.async.initializer.AsyncInitializer.checkInitialized(AsyncInitializer.java:138)
	at com.palantir.timestamp.PersistentTimestampServiceImpl$InitializingWrapper.delegate(PersistentTimestampServiceImpl.java:33)
	at com.palantir.timestamp.AutoDelegate_PersistentTimestampService.getFreshTimestamp(AutoDelegate_PersistentTimestampService.java:36)
	at jdk.internal.reflect.GeneratedMethodAccessor166.invoke(Unknown Source)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:564)
	at com.palantir.leader.proxy.AwaitingLeadershipProxy.handleInvocation(AwaitingLeadershipProxy.java:134)
	at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86)
	at com.sun.proxy.$Proxy89.getFreshTimestamp(Unknown Source)
	at jdk.internal.reflect.GeneratedMethodAccessor166.invoke(Unknown Source)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:564)
	at com.palantir.atlasdb.factory.LocalOrRemoteProxy.handleInvocation(LocalOrRemoteProxy.java:64)
	at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86)
	at com.sun.proxy.$Proxy91.getFreshTimestamp(Unknown Source)
	at com.palantir.lock.impl.LegacyTimelockService.lockImmutableTimestamp(LegacyTimelockService.java:87)
	at com.palantir.lock.impl.LegacyTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$0(LegacyTimelockService.java:123)
	at java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104)
	at java.util.stream.IntPipeline$Head.forEach(IntPipeline.java:593)
	at com.palantir.lock.impl.LegacyTimelockService.startIdentifiedAtlasDbTransactionBatch(LegacyTimelockService.java:122)
	at com.palantir.lock.client.TimeLockClient.lambda$startIdentifiedAtlasDbTransactionBatch$0(TimeLockClient.java:112)
	at com.palantir.lock.client.TimeLockClient.executeOnTimeLock(TimeLockClient.java:173)
	at com.palantir.lock.client.TimeLockClient.startIdentifiedAtlasDbTransactionBatch(TimeLockClient.java:112)
	at com.palantir.lock.client.ProfilingTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$2(ProfilingTimelockService.java:122)
	at com.palantir.lock.client.ProfilingTimelockService.runTaskTimed(ProfilingTimelockService.java:174)
	at com.palantir.lock.client.ProfilingTimelockService.startIdentifiedAtlasDbTransactionBatch(ProfilingTimelockService.java:121)
	at com.palantir.atlasdb.factory.timelock.TimestampCorroboratingTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$3(TimestampCorroboratingTimelockService.java:76)
	at com.palantir.atlasdb.factory.timelock.TimestampCorroboratingTimelockService.checkAndUpdateLowerBound(TimestampCorroboratingTimelockService.java:94)
	at com.palantir.atlasdb.factory.timelock.TimestampCorroboratingTimelockService.startIdentifiedAtlasDbTransactionBatch(TimestampCorroboratingTimelockService.java:75)
	at com.palantir.atlasdb.transaction.impl.InstrumentedTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$2(InstrumentedTimelockService.java:88)
	at com.palantir.atlasdb.transaction.impl.InstrumentedTimelockService.executeWithRecord(InstrumentedTimelockService.java:136)
	at com.palantir.atlasdb.transaction.impl.InstrumentedTimelockService.startIdentifiedAtlasDbTransactionBatch(InstrumentedTimelockService.java:88)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.startTransactions(SnapshotTransactionManager.java:182)
	at com.palantir.atlasdb.transaction.impl.SerializableTransactionManager.startTransactions(SerializableTransactionManager.java:62)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.lambda$runTaskWithConditionThrowOnConflict$1(SnapshotTransactionManager.java:168)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.runTimed(SnapshotTransactionManager.java:517)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.runTaskWithConditionThrowOnConflict(SnapshotTransactionManager.java:167)
	at com.palantir.atlasdb.transaction.impl.SerializableTransactionManager.runTaskWithConditionThrowOnConflict(SerializableTransactionManager.java:62)
	at com.palantir.atlasdb.transaction.impl.AbstractConditionAwareTransactionManager.lambda$runTaskWithConditionWithRetry$0(AbstractConditionAwareTransactionManager.java:50)
	at com.github.rholder.retry.AttemptTimeLimiters$NoAttemptTimeLimit.call(AttemptTimeLimiters.java:78)
	at com.github.rholder.retry.Retryer.call(Retryer.java:160)
	at com.palantir.atlasdb.transaction.impl.TransactionRetryStrategy.runWithRetry(TransactionRetryStrategy.java:84)
	at com.palantir.atlasdb.transaction.impl.AbstractConditionAwareTransactionManager.runTaskWithConditionWithRetry(AbstractConditionAwareTransactionManager.java:47)
	at com.palantir.tritium.proxy.InstrumentedTransactionManager$8.runTaskWithConditionWithRetry(Unknown Source)
	at com.palantir.atlas.migration2.api.MigrationAwareTransactionManagerImpl.runTaskWithConditionWithRetry(MigrationAwareTransactionManagerImpl.java:126)
	at com.palantir.atlas.migration2.api.MigrationAwareTransactionManagerImpl.runTaskWithRetry(MigrationAwareTransactionManagerImpl.java:131)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:564)
	at com.palantir.atlas.migration2.api.QosAwareProxy.handleInvocation(QosAwareProxy.java:34)
	at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86)
	at com.sun.proxy.$Proxy186.runTaskWithRetry(Unknown Source)