only add semaphore to waitingList once per version

TrackLastVersion is called once per store during IN_PROGRESS. Each call creates
a new semaphore and overwrites lastVersionTransactionsDone, orphaning the
previous one in the waitingList. DecrementActiveTransactions only releases the
last one. ProcessWaitingListAsync blocks forever on the orphaned semaphore.

Since both stores share the same transaction counter, we only need one
semaphore per version. If TrackLastVersion has already been called for a given
version, subsequent calls return immediately.

Includes a regression test that fails without the fix.

Author: KiKoS0

libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineDriver.cs

@@ -66,6 +66,12 @@ void DecrementActiveTransactions(long txnVersion)
 
         internal void TrackLastVersion(long version)
         {
+            // Only create and enqueue one semaphore per version, if we create a
+            // new one on each call, the earlier semaphore is orphaned in the waitingList
+            // and never released, and we permanently block ProcessWaitingListAsync.
+            if (lastVersion == version)
+                return;
+
             if (GetNumActiveTransactions(version) > 0)
             {
                 // Set version number first, then create semaphore

libs/storage/Tsavorite/cs/test/StateMachineDriverTests.cs

@@ -401,4 +401,49 @@ public async ValueTask GrowIndexVersionSwitchTxnTest(
             [Values] bool useTimingFuzzing)
             => await DoGrowIndexVersionSwitchEquivalenceCheck(indexSize, useTimingFuzzing).ConfigureAwait(false);
     }
+
+    /// <summary>
+    /// Regression test for checkpoint deadlock with two-store checkpoints.
+    ///
+    /// TrackLastVersion is called once per store during the IN_PROGRESS phase.
+    /// Without the fix, the second call overwrites lastVersionTransactionsDone,
+    /// orphaning the first semaphore in the waitingList. ProcessWaitingListAsync
+    /// then waits on it forever.
+    /// </summary>
+    [TestFixture]
+    public class TrackLastVersionTwoStoreDeadlock
+    {
+        [Test]
+        public async Task TrackLastVersionCalledTwiceDoesNotDeadlock()
+        {
+            var epoch = new LightEpoch();
+            var driver = new StateMachineDriver(epoch);
+
+            // Simulate an active transaction (e.g. Lua script touching both stores)
+            var txnVersion = driver.AcquireTransactionVersion();
+
+            // GlobalAfterEnteringState calls TrackLastVersion once per store
+            driver.TrackLastVersion(txnVersion);  // MainStore
+            driver.TrackLastVersion(txnVersion);  // ObjectStore
+
+            // Transaction completes
+            driver.EndTransaction(txnVersion);
+
+            // Verify all waitingList semaphores are released (not orphaned)
+            var waitingList = (System.Collections.Generic.List<System.Threading.SemaphoreSlim>)
+                typeof(StateMachineDriver)
+                    .GetField("waitingList", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
+                    .GetValue(driver);
+
+            ClassicAssert.AreEqual(1, waitingList.Count,
+                "Expected 1 semaphore in waitingList, not 2. " +
+                "Two means the second TrackLastVersion call created a new semaphore " +
+                "that overwrote the first, orphaning it.");
+
+            var acquired = await waitingList[0].WaitAsync(System.TimeSpan.FromSeconds(5));
+            ClassicAssert.IsTrue(acquired,
+                "Semaphore was not released after EndTransaction. " +
+                "This causes ProcessWaitingListAsync to deadlock permanently.");
+        }
+    }
 }