AD Login failure when using Docker and multiple replicas

I have an nginx reverse proxy sitting in front of an aspnetcore application that authenticates against AzureAD, When I run in a swarm with a single replica all works fine, however when I have more than 1 replica I get the following Error

Exception: An error was encountered while handling the remote login.
Microsoft.AspNetCore.Authentication.RemoteAuthenticationHandler<TOptions>.HandleRequestAsync()
Microsoft.AspNetCore.Authentication.AuthenticationMiddleware.Invoke(HttpContext context)
Microsoft.AspNetCore.StaticFiles.StaticFileMiddleware.Invoke(HttpContext context)
Microsoft.AspNetCore.Builder.Extensions.MapWhenMiddleware.Invoke(HttpContext context)
Microsoft.AspNetCore.Diagnostics.DeveloperExceptionPageMiddleware.Invoke(HttpContext context)

Below is my Startup.cs

using System.IO;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.AzureAD.UI;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Authorization;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using DataAccess.Models;
using Newtonsoft.Json.Serialization;
using Microsoft.AspNetCore.HttpOverrides;
using System.Globalization;
using Microsoft.Net.Http.Headers;
using System.Data.SqlClient;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
using Newtonsoft.Json;
using System.Linq;

namespace www {
    public class Startup {
        public Startup(IConfiguration configuration) {

            CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");

            Configuration = configuration;

            var builder = new ConfigurationBuilder()
                                .AddJsonFile("appsettings.json")
                                .AddEnvironmentVariables()
                                ;
            Configuration = builder.Build();

            TradingContext.ConnectionString = $"Server={Configuration.GetSection("SQL_SERVER").Value};Database=Trading;User ID={Configuration.GetSection("SQL_USER").Value};Password={Configuration.GetSection("SQL_PASSWORD").Value};MultipleActiveResultSets=true;";

        }

        public IConfiguration Configuration { get; }

        public void ConfigureServices(IServiceCollection services) {

            services.AddHealthChecks()
                .AddCheck("SqlServer", () => {
                    try {
                        using (var db = new TradingContext()) {
                            var x = db.InstrumentRoots.Count();
                        }
                    }
                    catch {
                        return HealthCheckResult.Unhealthy();
                    }

                    return HealthCheckResult.Healthy();
                }
            );

            services.AddMvc().AddJsonOptions(options => {
                var resolver = options.SerializerSettings.ContractResolver;
                if (resolver != null) {
                    var res = (DefaultContractResolver)resolver;
                    res.NamingStrategy = null;
                }
            });

            services.Configure<ForwardedHeadersOptions>(options => {
                options.ForwardedHeaders =
                    ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
            });

            services.Configure<EmailSettings>(Configuration.GetSection("EmailSettings"));
            services.AddTransient<IEmailSender, AuthMessageSender>();

            services.Configure<CookiePolicyOptions>(options => {
                options.CheckConsentNeeded = context => true;
                options.MinimumSameSitePolicy = Microsoft.AspNetCore.Http.SameSiteMode.None;
            });

            services.AddDbContext<TradingContext>();

            services.AddAuthentication(AzureADDefaults.AuthenticationScheme)
                .AddAzureAD(options => Configuration.Bind("AzureAd", options));

            services.AddMvc(options => {
                var policy = new AuthorizationPolicyBuilder()
                    .RequireAuthenticatedUser()
                    .Build();
                options.Filters.Add(new AuthorizeFilter(policy));
            })
            .AddRazorPagesOptions(options => {
                options.Conventions.AddPageRoute("/Dashboard/Index", "");
            })
            .SetCompatibilityVersion(CompatibilityVersion.Version_2_2);
        }

        public void Configure(IApplicationBuilder app, IHostingEnvironment env) {
            if (env.IsDevelopment()) {
                app.UseDeveloperExceptionPage();
            }
            else {
                app.UseExceptionHandler("/Error");
            }

            var options = new HealthCheckOptions();
            options.ResponseWriter = async (c, r) => {
                c.Response.ContentType = "application/json";

                var result = JsonConvert.SerializeObject(new {
                    status = r.Status.ToString(),
                    errors = r.Entries.Select(e => new { key = e.Key, value = e.Value.Status.ToString() })
                });
                await c.Response.WriteAsync(result);
            };
            app.UseHealthChecks("/healthcheck", options);

            app.UseCookiePolicy();

            app.UseStaticFiles(new StaticFileOptions {
                OnPrepareResponse = ctx =>
                {
                    const int durationInSeconds = 60 * 60 * 24;
                    ctx.Context.Response.Headers[HeaderNames.CacheControl] =
                        "public,max-age=" + durationInSeconds;
                }
            });

            var fordwardedHeaderOptions = new ForwardedHeadersOptions {
                ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
            };
            fordwardedHeaderOptions.KnownNetworks.Clear();
            fordwardedHeaderOptions.KnownProxies.Clear();

            app.UseForwardedHeaders(fordwardedHeaderOptions);

            app.UseAuthentication();

            app.UseMvc();
        }
    }
}

Below is my nginx.config

events {

}

http {
    proxy_buffer_size   128k;
    proxy_buffers   4 256k;
    proxy_busy_buffers_size   256k;
    large_client_header_buffers 4 16k;
    gzip on;

    server {
        listen      80;
        listen [::]:80;
        server_name xxxx;

        location / {
            rewrite ^ https://$host$request_uri? permanent;
        }

        #for certbot challenges (renewal process)
        location ~ /.well-known/acme-challenge {
            allow all;
            root /data/letsencrypt;
        }
    }

    #https://xxxx
    server {
        server_name xxxx;
        listen 443 ssl http2;

        server_tokens off;

        ssl_buffer_size 8k;
        ssl_dhparam /etc/ssl/certs/dhparam-2048.pem;

        ssl_protocols TLSv1.3 TLSv1.2;
        ssl_prefer_server_ciphers on;
        ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5;

        ssl_ecdh_curve secp384r1;
        ssl_session_tickets off;

        # OCSP stapling
        ssl_stapling on;
        ssl_stapling_verify on;
        resolver 8.8.8.8 8.8.4.4;

        ssl_certificate /etc/letsencrypt/live/xxxxx/fullchain.pem;
        ssl_certificate_key /etc/letsencrypt/live/xxxxx/privkey.pem;

        location / {
            proxy_pass http://www:5000;
            proxy_http_version  1.1;
            proxy_set_header    Upgrade $http_upgrade;
            proxy_set_header    Connection keep-alive;
            proxy_set_header    Host $http_host;
            proxy_cache_bypass  $http_upgrade;
            proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header    X-Forwarded-Proto $scheme;
            fastcgi_buffers     16 16k;
            fastcgi_buffer_size 32k;

            #security headers
            add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload";
            add_header X-XSS-Protection "1; mode=block" always;
            add_header X-Content-Type-Options "nosniff" always;
            add_header X-Frame-Options "DENY" always;
            #CSP
            #add_header Content-Security-Policy "frame-src 'self'; default-src 'self'; script-src 'self' 'unsafe-inline' https://maxcdn.bootstrapcdn.com https://ajax.googleapis.com; img-src 'self'; style-src 'self' https://maxcdn.bootstrapcdn.com; font-src 'self' data: https://maxcdn.bootstrapcdn.com; form-action 'self'; upgrade-insecure-requests;" always;
            add_header Referrer-Policy "strict-origin-when-cross-origin" always;
        }
    }
}

Any Idea what I am missing?

Source: StackOverflow