ory kratos csrf cookie not being sent behind nginx ssl

Question 1

I am using Go with ory kratos in docker and everything works fine on my machine on localhost. Auth works, all cookies are send and set and i can call my backend from SPA and be authenticated.

The problem is that on live server behind nginx and ssl, apparently one cookie is not being sent from my js client (only ory_kratos_session is being sent and not xxx_csrf_token cookie) and it fails in function bellow with cookie missing error.

it uses official go sdk: kratos-client-go

Go AuthRequired middleware

func ExtractKratosCookiesFromRequest(r *http.Request) (csrf, session *http.Cookie, cookieHeader string) {
    cookieHeader = r.Header.Get("Cookie")

    cookies := r.Cookies()
    for _, c := range cookies {
        if c != nil {
            if ok := strings.HasSuffix(c.Name, string("csrf_token")); ok {
                csrf = c
            }
        }
    }

    sessionCookie, _ := r.Cookie("ory_kratos_session")
    if sessionCookie != nil {
        session = sessionCookie
    }

    return
}

func AuthRequired(w http.ResponseWriter, r *http.Request) error {
    csrfCookie, sessionCookie, cookieHeader := ExtractKratosCookiesFromRequest(r)
    if (csrfCookie == nil || sessionCookie == nil) || (csrfCookie.Value == "" || sessionCookie.Value == "") {
        return errors.New("Cookie missing")
    }

    req := kratos.PublicApi.Whoami(r.Context()).Cookie(cookieHeader)
    kratosSession, _, err := req.Execute()
    if err != nil {
        return errors.New("Whoami error")
    }
    
    return nil
}

My js http client has option: credentials: 'include'.

In devtools panel i see only 1 cookie (ory_kratos_session) after register/login.

So what is failing is that request is only sending ory_kratos_session and not xxx_csrf_token cookie (which works on localhost in kratos --dev mode, and cookie is vidisble in devtools panel)

Request Info

General:

Request URL: https://example.com/api/v1/users/1/donations
Request Method: GET
Status Code: 401 Unauthorized
Remote Address: 217.163.23.144:443
Referrer Policy: strict-origin-when-cross-origin

Request Headers:

accept: application/json
Accept-Encoding: gzip, deflate, br
Accept-Language: en-US,en;q=0.9
Connection: keep-alive
content-type: application/json; charset=UTF-8
Cookie: ory_kratos_session=MTYyMjA0NjEyMHxEdi1CQkFFQ180SUFBUkFCRUFBQVJfLUNBQUVHYzNSeWFXNW5EQThBRFhObGMzTnBiMjVmZEc5clpXNEdjM1J5YVc1bkRDSUFJRFo0Y2tKUFNFUmxZWFpsV21kaFdVbFZjMFU0VVZwcFkxbDNPRFpoY1ZOeXyInl242jY9c2FDQmykJrjLTNLg-sPFv2y04Qfl3uDfpA==
Host: example.com
Referer: https://example.com/dashboard/donations
sec-ch-ua: " Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"
sec-ch-ua-mobile: ?0
Sec-Fetch-Dest: empty
Sec-Fetch-Mode: cors
Sec-Fetch-Site: same-origin
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36

Response Headers:

Connection: keep-alive
Content-Length: 175
Content-Type: application/json
Date: Wed, 26 May 2021 17:12:27 GMT
Server: nginx/1.18.0 (Ubuntu)
Vary: Origin

docker-compose.yml

version: "3.8"

services:
  # --------------------------------------------------------------------------------
  api-server:
    build:
      context: .
      dockerfile: ./dockerfiles/app.dockerfile
    container_name: api-server
    restart: always
    volumes:
      - ./:/app
    ports:
      - 3001:3001
    networks:
      - intranet
    depends_on:
      - postgresd
  # --------------------------------------------------------------------------------
  postgresd:
    image: postgres:13.3-alpine
    container_name: postgresd
    restart: always
    environment:
      - POSTGRES_DB=test
      - POSTGRES_USER=test
      - POSTGRES_PASSWORD=test
    volumes:
      - postgres-data:/var/lib/postgresql/data
    ports:
      - 5432:5432
    networks:
      - intranet
  # --------------------------------------------------------------------------------
  kratos-migrate:
    image: oryd/kratos:v0.6.2-alpha.1
    container_name: kratos-migrate
    restart: on-failure
    environment:
      - DSN=postgres://test:test@postgresd:5432/test?sslmode=disable&max_conns=20&max_idle_conns=4
    volumes:
      - type: bind
        source: ./kratos/config
        target: /etc/config/kratos
    command:
      [
        "migrate",
        "sql",
        "--read-from-env",
        "--config",
        "/etc/config/kratos/kratos.yml",
        "--yes",
      ]
    networks:
      - intranet
    depends_on:
      - postgresd
  # --------------------------------------------------------------------------------
  kratos:
    image: oryd/kratos:v0.6.2-alpha.1
    container_name: kratos
    restart: unless-stopped
    environment:
      - DSN=postgres://test:test@postgresd:5432/test?sslmode=disable&max_conns=20&max_idle_conns=4
    command: ["serve", "--config", "/etc/config/kratos/kratos.yml"]
    volumes:
      - type: bind
        source: ./kratos/config
        target: /etc/config/kratos
    ports:
      - 4433:4433
      - 4434:4434
    networks:
      - intranet
    depends_on:
      - postgress
      - kratos-migrate
  # --------------------------------------------------------------------------------

volumes:
  postgres-data:

networks:
  intranet:
    driver: bridge

kratos.yml

version: v0.6.2-alpha.1

dsn: postgres://test:test@postgresd:5432/test?sslmode=disable&max_conns=20&max_idle_conns=4

serve:
  public:
    base_url: https://example.com/kratos/
    cors:
      enabled: true
      debug: true
      allow_credentials: true
      options_passthrough: true
      allowed_origins:
        - https://example.com
      allowed_methods:
        - POST
        - GET
        - PUT
        - PATCH
        - DELETE
        - OPTIONS
      allowed_headers:
        - Authorization
        - Cookie
        - Origin
        - X-Session-Token
      exposed_headers:
        - Content-Type
        - Set-Cookie
  admin:
    base_url: https://example.com/kratos/

selfservice:
  default_browser_return_url: https://example.com
  whitelisted_return_urls:
    - https://example.com
    - https://example.com/dashboard
    - https://example.com/auth/login
  methods:
    password:
      enabled: true
    oidc:
      enabled: false
    link:
      enabled: true
    profile:
      enabled: true
  flows:
    error:
      ui_url: https://example.com/error
    settings:
      ui_url: https://example.com/dashboard/profile
      privileged_session_max_age: 15m
    recovery:
      enabled: true
      ui_url: https://example.com/auth/recovery
      after:
        default_browser_return_url: https://example.com/auth/login
    verification:
      enabled: true
      ui_url: https://example.com/auth/verification
      after:
        default_browser_return_url: https://example.com
    logout:
      after:
        default_browser_return_url: https://example.com
    login:
      ui_url: https://example.com/auth/login
      lifespan: 10m
    registration:
      lifespan: 10m
      ui_url: https://example.com/auth/registration
      after:
        password:
          hooks:
            - hook: session
          default_browser_return_url: https://example.com/auth/login
        default_browser_return_url: https://example.com/auth/login
        oidc:
          hooks:
            - hook: session

secrets:
  cookie:
    - fdwfhgwjfgwf9286f24tf29ft

session:
  lifespan: 24h
  cookie:
    domain: example.com # i tried also with http:// and https://
    same_site: Lax

hashers:
  argon2:
    parallelism: 1
    memory: 128MB
    iterations: 1
    salt_length: 16
    key_length: 16

identity:
  default_schema_url: file:///etc/config/kratos/identity.schema.json

courier:
  smtp:
    connection_uri: smtp://user:pwd@smtp.mailtrap.io:2525
    from_name: test
    from_address: office@test.com

watch-courier: true

log:
  level: debug
  format: text
  leak_sensitive_values: true

My Go rest api has these cors options:

ALLOWED_ORIGINS=https://example.com
ALLOWED_METHODS=GET,POST,PUT,PATCH,DELETE,HEAD,OPTIONS
ALLOWED_HEADERS=Content-Type,Authorization,Cookie,Origin,X-Session-Token,X-CSRF-Token,Vary
EXPOSED_HEADERS=Content-Type,Authorization,Content-Length,Cache-Control,Content-Language,Content-Range,Set-Cookie,Pragma,Expires,Last-Modified,X-Session-Token,X-CSRF-Token
MAX_AGE=86400
ALLOW_CREDENTIALS=true

nginx default

upstream go-api {
    server 127.0.0.1:3001;
}

upstream kratos {
    server 127.0.0.1:4433;
}

upstream kratos-admin {
    server 127.0.0.1:4434;
}

server {
        server_name example.com www.example.com;

        location / {
                root /var/www/website;
                try_files $uri $uri/ /index.html;
        }
  
        location /api/ {
                 proxy_pass http://go-api;
                 proxy_http_version 1.1;
                 proxy_set_header Host $host;
                 proxy_set_header X-Real-IP $remote_addr;
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                 proxy_set_header X-Forwarded-Port $server_port;
                 proxy_set_header x-forwarded-proto $scheme;
                 proxy_set_header Upgrade $http_upgrade;
                 proxy_set_header Connection 'upgrade';
                 proxy_cache_bypass $http_upgrade;
        }

        location /kratos/ {
                 proxy_pass http://kratos/;
                 proxy_http_version 1.1;
                 proxy_set_header Host $host;
                 proxy_set_header X-Real-IP $remote_addr;
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                 proxy_set_header X-Forwarded-Port $server_port;
                 proxy_set_header x-forwarded-proto $scheme;
                 proxy_set_header Upgrade $http_upgrade;
                 proxy_set_header Connection 'upgrade';
                 proxy_cache_bypass $http_upgrade;
        }

       location /kratos-admin/ {
                 proxy_pass http://kratos-admin/;
                 proxy_http_version 1.1;
                 proxy_set_header Host $host;
                 proxy_set_header X-Real-IP $remote_addr;
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                 proxy_set_header X-Forwarded-Port $server_port;
                 proxy_set_header x-forwarded-proto $scheme;
                 proxy_set_header Upgrade $http_upgrade;
                 proxy_set_header Connection 'upgrade';
                 proxy_cache_bypass $http_upgrade;
         }

    listen [::]:443 ssl ipv6only=on; # managed by Certbot
    listen 443 ssl; # managed by Certbot
    certs go here...
}

I don't understand why it is not working on live server, it has to be something with ssl

this is my http client that i am using (ky.js but it doesn't matter its the same as fetch)

const options = {
  prefixUrl: 'https://example.com/api/v1',
  headers: {
    'Content-Type': 'application/json; charset=UTF-8',
    Accept: 'application/json',
  },
  timeout: 5000,
  mode: 'cors',
  credentials: 'include',
};

export const apiClient = ky.create(options);

i'm just calling my backend protected route that is protected with AuthRequired middleware, nothing special:

function createTodo(data) {
  return apiClient.post(`todos`, { json: data }).json();
}

ory/kratos-client (js sdk) is configured like this:

const conf = new Configuration({
  basePath: 'https://example.com/kratos',
  // these are axios options (kratos js sdk uses axios under the hood)
  baseOptions: { 
    withCredentials: true,
    timeout: 5000,
  },
});

export const kratos = new PublicApi(conf);

It's strange that in firefox i see 2 cookies in devtools panel but not in chrome.

this is csrf one:

aHR0cHM6Ly9hbmltb25kLnh5ei9rcmF0b3Mv_csrf_token:"Kx+PXWeoxsDNxQFGZBgvlTJScg9VIYEB+6cTrC0zsA0="
Created:"Thu, 27 May 2021 10:21:45 GMT"
Domain:".example.com"
Expires / Max-Age:"Fri, 27 May 2022 10:22:32 GMT"
HostOnly:false
HttpOnly:true
Last Accessed:"Thu, 27 May 2021 10:22:32 GMT"
Path:"/kratos/"
SameSite:"None"
Secure:true
Size: 91

This is the session cookie:

ory_kratos_session:"MTYyMjExMDk1MnxEdi1CQkFFQ180SUFBUkFCRUFBQVJfLUNBQUVHYzNSeWFXNW5EQThBRFhObGMzTnBiMjVmZEc5clpXNEdjM1J5YVc1bkRDSUFJRFZYV25Jd05HaEpTR28xVHpaT1kzTXlSSGxxVHpaaWQyUTVRamhIY2paM3zb24EtkN6Bmv_lRZa7YSRBOYvUGYSUBmZ7RIkDsm4Oyw=="
Created:"Thu, 27 May 2021 10:22:32 GMT"
Domain:".example.com"
Expires / Max-Age:"Thu, 08 Jul 2021 01:22:32 GMT"
HostOnly:false
HttpOnly:true
Last Accessed:"Thu, 27 May 2021 10:22:32 GMT"
Path:"/"
SameSite:"Lax"
Secure:true
Size:234

I thought it was something related to the timezones in the containers, i also mounted this volume in all of them: -v /etc/localtime:/etc/localtime:ro

PS

The problem is that whenever i docker-compose restart kratos, things get broken, somehow apparently old csrf_token is being used. How is this supposed to be used, i can't just tell my users hey go to your browser and delete all cache and cookies. when i prune everything it works, but once i restarted nginx and it didn't work after that (same is after docker-compose restart)... very strange

this guy had the same problem here: csrf problem after restart

Question 2

I did manage to solve this, it works fine without --dev flag in production and doesn't get screwed up even after restarting the services and changing everything.

Maybe it was even my react form where i used defaultValue for the csrf token input

Now whenever window.location url changes or whenever csrf_token changes, it should use the latest value for the csrf_token

const [csrf, setCsrf] = React.useState('');

useEffect(() => {
  if (flowResponse !== null) {
    const csrfVal = flowResponse?.ui?.nodes?.find?.(n => n.attributes.name === 'csrf_token')?.attributes.value;
    setCsrf(csrfVal);
  }
}, [flowResponse, csrf]);

<input type='hidden' name='csrf_token' value={csrf} readOnly required />

the worst thing is that it could have also been a trailing slash or something so small that i am not sure exactly what caused it.

Here all post all the configuration that worked for me:

could have been that i tried before with this kratos url being http://127.0.0.1:4433 or http://kratos:4433 and it didn't work (even though i was switching between these 3 haha)

init kratos client

conf := kratos.NewConfiguration()
conf.Servers[0].URL = "https://example.com/kratos/"
kratosClient := kratos.NewAPIClient(conf)

kratos.yml

version: v0.6.2-alpha.1

dsn: postgres://test:test@postgresd:5432/test?sslmode=disable&max_conns=20&max_idle_conns=4

serve:
  public:
    base_url: https://example.com/kratos/
    cors:
      enabled: true
      debug: true
      allow_credentials: true
      options_passthrough: true
      max_age: 0
      allowed_origins:
        - https://example.com
      allowed_methods:
        - POST
        - GET
        - PUT
        - PATCH
        - DELETE
        - OPTIONS
      allowed_headers:
        - Authorization
        - Cookie
        - Origin
        - X-Session-Token
      exposed_headers:
        - Content-Type
        - Set-Cookie
  admin:
    base_url: http://127.0.0.1:4434/

selfservice:
  default_browser_return_url: https://example.com
  whitelisted_return_urls:
    - https://example.com
    - https://example.com/dashboard
    - https://example.com/auth/login
  methods:
    password:
      enabled: true
    oidc:
      enabled: false
    link:
      enabled: true
    profile:
      enabled: true
  flows:
    error:
      ui_url: https://example.com/error
    settings:
      ui_url: https://example.com/dashboard/profile
      privileged_session_max_age: 15m
    recovery:
      enabled: true
      ui_url: https://example.com/auth/recovery
      after:
        default_browser_return_url: https://example.com/auth/login
    verification:
      enabled: true
      ui_url: https://example.com/auth/verification
      after:
        default_browser_return_url: https://example.com
    logout:
      after:
        default_browser_return_url: https://example.com
    login:
      ui_url: https://example.com/auth/login
      lifespan: 10m
    registration:
      lifespan: 10m
      ui_url: https://example.com/auth/registration
      after:
        password:
          hooks:
            - hook: session
          default_browser_return_url: https://example.com/auth/login
        default_browser_return_url: https://example.com/auth/login
        oidc:
          hooks:
            - hook: session

secrets:
  cookie:
    - veRy_S3cRet_tHinG

session:
  lifespan: 24h
  cookie:
    domain: example.com
    same_site: Lax
    path: /           
// <- i didn't have path before, not sure if it changes anything but it works (before csrf cookie had path /kratos and now when it works it has path /, same as session_cookie)

hashers:
  argon2:
    parallelism: 1
    memory: 128MB
    iterations: 1
    salt_length: 16
    key_length: 16

identity:
  default_schema_url: file:///etc/config/kratos/identity.schema.json

courier:
  smtp:
    connection_uri: smtp://user:pwd@smtp.mailtrap.io:2525
    from_name: example
    from_address: office@example.cpm

watch-courier: true

log:
  level: debug
  format: text
  leak_sensitive_values: true

nginx.conf

server {
    server_name example.com www.example.com;

    location / {
       root /var/www/public;
       try_files $uri $uri/ /index.html;
    }

    location /api/ {
      proxy_pass http://127.0.0.1:3001; // backend api url
      proxy_http_version 1.1;
      proxy_set_header Host $host;
      proxy_set_header X-Real-IP $remote_addr;
      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_set_header X-Forwarded-Port $server_port;
      proxy_set_header x-forwarded-proto $scheme;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection 'upgrade';
      proxy_cache_bypass $http_upgrade;
    }

    location /kratos/ {
      proxy_pass http://127.0.0.1:4433/;  // kratos public url
      proxy_http_version 1.1;
      proxy_set_header Host $host;
      proxy_set_header X-Real-IP $remote_addr;
      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_set_header X-Forwarded-Port $server_port;
      proxy_set_header x-forwarded-proto $scheme;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection 'upgrade';
      proxy_cache_bypass $http_upgrade;
    }
       
    listen [::]:443 ssl ipv6only=on; # managed by Certbot
    listen 443 ssl; # managed by Certbot
    certs...
}

server {
    if ($host = www.example.com) {
        return 301 https://$host$request_uri;
    } # managed by Certbot

    if ($host = example.com) {
        return 301 https://$host$request_uri;
    } # managed by Certbot

   listen 80 default_server;
   listen [::]:80 default_server;
   server_name example.com www.example.com;
   return 404; # managed by Certbot
}

ory kratos csrf cookie not being sent behind nginx ssl

weixin_0010034

Answer a question

Go AuthRequired middleware

Request Info

docker-compose.yml

kratos.yml

nginx default

PS

Answers

init kratos client

kratos.yml

nginx.conf

所有评论(0)

weixin_0010034