title: 1 out of N threads never joining
body: <p>I have thread pool implementation where whenever I try to stop/join the pool there is always one random thread in the pool that will not stop (<code>state == Running</code>) when I call <code>Stop()</code> on the pool.</p>
  
  <p>I cannot see why, I only have one lock, I notify whoever might be blocked waiting for <code>Dequeue</code> with <code>Monitor.PulseAll</code> in <code>Stop</code>. The debugger clearly shows most of them got the message, it is just always 1 out of N that is still running...</p>
  
  <p>Here is a minimal implementation of the pool</p>
  
  <pre><code>using System;
  using System.Collections.Generic;
  using System.Linq;
  using System.Threading;
  using System.Threading.Tasks;
  
  namespace MultiThreading
  {
      public class WorkerHub
      {
          private readonly object _listMutex = new object();
          private readonly Queue&lt;TaskWrapper&gt; _taskQueue;
          private readonly List&lt;Thread&gt; _threads;
          private int _runCondition;
          private readonly Dictionary&lt;string, int&gt; _statistics;
  
          public WorkerHub(int count = 4)
          {
              _statistics = new Dictionary&lt;string, int&gt;();
              _taskQueue = new Queue&lt;TaskWrapper&gt;();
              _threads = new List&lt;Thread&gt;();
              InitializeThreads(count);
          }
  
          private bool ShouldRun
          {
              get =&gt; Interlocked.CompareExchange(ref _runCondition, 1, 1) == 1;
              set
              {
                  if (value)
                      Interlocked.CompareExchange(ref _runCondition, 1, 0);
                  else
                      Interlocked.CompareExchange(ref _runCondition, 0, 1);
              }
          }
  
          private void InitializeThreads(int count)
          {
              Action threadHandler = () =&gt;
              {
                  while (ShouldRun)
                  {
                      var wrapper = Dequeue();
                      if (wrapper != null)
                      {
                          wrapper.FunctionBinding.Invoke();
                          _statistics[Thread.CurrentThread.Name] += 1;
                      }
                  }
              };
  
              for (var i = 0; i &lt; count; ++i)
              {
                  var t = new Thread(() =&gt; { threadHandler.Invoke(); });
                  t.Name = $"WorkerHub Thread#{i}";
                  _statistics[t.Name] = 0;
                  _threads.Add(t);
              }
          }
  
  
          public Task Enqueue(Action work)
          {
              var tcs = new TaskCompletionSource&lt;bool&gt;();
              var wrapper = new TaskWrapper();
  
              Action workInvoker = () =&gt;
              {
                  try
                  {
                      work.Invoke();
                      tcs.TrySetResult(true);
                  }
                  catch (Exception e)
                  {
                      tcs.TrySetException(e);
                  }
              };
              Action workCanceler = () =&gt; { tcs.TrySetCanceled(); };
              wrapper.FunctionBinding = workInvoker;
              wrapper.CancelBinding = workCanceler;
  
  
              lock (_taskQueue)
              {
                  _taskQueue.Enqueue(wrapper);
                  Monitor.PulseAll(_taskQueue);
              }
  
  
              return tcs.Task;
          }
  
          private TaskWrapper Dequeue()
          {
              lock (_listMutex)
              {
                  while (_taskQueue.Count == 0)
                  {
                      if (!ShouldRun)
                          return null;
                      Monitor.Wait(_listMutex);
                  }
  
                  _taskQueue.TryDequeue(out var wrapper);
                  return wrapper;
              }
          }
  
          public void Stop()
          {
              ShouldRun = false;
  
              //Wake up whoever is waiting for dequeue
              lock (_listMutex)
              {
                  Monitor.PulseAll(_listMutex);
              }
  
              foreach (var thread in _threads)
              {
                  thread.Join();
              }
              var sum = _statistics.Sum(pair =&gt; pair.Value) * 1.0;
              foreach (var stat in _statistics)
              {
                  Console.WriteLine($"{stat.Key} ran {stat.Value} functions, {stat.Value/sum * 100} percent of the total.");
              }
          }
  
          public void Start()
          {
              ShouldRun = true;
              foreach (var thread in _threads) thread.Start();
          }
      }
  }
  </code></pre>
  
  <p>With a test run</p>
  
  <pre><code>public static async Task Main(string[] args)
      {
          var hub = new WorkerHub();
          var tasks = Enumerable.Range(0, (int) 100).Select(x =&gt; hub.Enqueue(() =&gt; Sum(x)))
              .ToArray();
          var sw = new Stopwatch();
          sw.Start();
          hub.Start();
          await Task.WhenAll(tasks);
          hub.Stop();
          sw.Start();
          Console.WriteLine($"Work took: {sw.ElapsedMilliseconds}ms.");
      }
  
      public static int Sum(int n)
      {
          var sum = 0;
          for (var i = 0; i &lt;= n; ++i) sum += i;
          Console.WriteLine($"Sum of numbers up to {n} is {sum}");
          return sum;
      }
  </code></pre>
  
  <p>Am I missing something fundamental? Please note this is not production code (phew) but stuff I am just missing around with so you might find more than 1 issue :)</p>