Securing a NATS Cluster with cfssl

Secure NATS Cluster in Kubernetes using the NATS Operator

Features

• Clients TLS setup
• TLS based auth certs via secret
  • Reloading supported by only updating secret
• Routes TLS setup
• Advertising public IP per NATS server for external access

Creating the Certificates

Generating the Root CA Certs

{
    "CN": "nats.io",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "nats.io"
        }
    ]
}

(
  cd certs
  # CA certs
  cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
)

Setup the profiles for the Root CA, we will have 3 main profiles: one for the clients connecting, one for the servers, and another one for the full mesh routing connections between the servers.

{
    "signing": {
        "default": {
            "expiry": "43800h"
        },
        "profiles": {
            "server": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            },
            "client": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "route": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}

Generating the NATS server certs

First we generate the certificates for the server.

{
    "CN": "nats.io",
    "hosts": [
        "localhost",
        "*.nats-cluster.default.svc",
        "*.nats-cluster-mgmt.default.svc",
        "nats-cluster",
        "nats-cluster-mgmt",
        "nats-cluster.default.svc",
        "nats-cluster-mgmt.default.svc",
        "nats-cluster.default.svc.cluster.local",
        "nats-cluster-mgmt.default.svc.cluster.local",
        "*.nats-cluster.default.svc.cluster.local",
        "*.nats-cluster-mgmt.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "Operator"
        }
    ]
}

(
  # Generating the peer certificates
  cd certs
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server
)

Generating the NATS server routes certs

We will also be setting up TLS for the full mesh routes.

{
    "CN": "nats.io",
    "hosts": [
        "localhost",
        "*.nats-cluster.default.svc",
        "*.nats-cluster-mgmt.default.svc",
        "nats-cluster",
        "nats-cluster-mgmt",
        "nats-cluster.default.svc",
        "nats-cluster-mgmt.default.svc",
        "nats-cluster.default.svc.cluster.local",
        "nats-cluster-mgmt.default.svc.cluster.local",
        "*.nats-cluster.default.svc.cluster.local",
        "*.nats-cluster-mgmt.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "Operator"
        }
    ]
}

# Generating the peer certificates
(
  cd certs
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=route route.json | cfssljson -bare route
)

Generating the certs for the clients (CNCF && ACME)

{
    "CN": "nats.io",
    "hosts": [""],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "OU": "CNCF"
        }
    ]
}

(
  cd certs
  # Generating NATS client certs
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
)

Kubectl Create

cd certs kubectl create secret generic nats-tls-example --from-file=ca.pem --from-file=server-key.pem --from-file=server.pem kubectl create secret generic nats-tls-routes-example --from-file=ca.pem --from-file=route-key.pem --from-file=route.pem kubectl create secret generic nats-tls-client-example --from-file=ca.pem --from-file=client-key.pem --from-file=client.pem

Create the Auth secret

{
  "users": [
    { "username": "CN=nats.io,OU=ACME" },
    { "username": "CN=nats.io,OU=CNCF",
      "permissions": {
    "publish": ["hello.*"],
    "subscribe": ["hello.world"]
      }
    }
  ],
  "default_permissions": {
    "publish": ["SANDBOX.*"],
    "subscribe": ["PUBLIC.>"]
  }
}

kubectl create secret generic nats-tls-users --from-file=users.json

Create a cluster with TLS

echo '
apiVersion: "nats.io/v1alpha2"
kind: "NatsCluster"
metadata:
  name: "nats-cluster"
spec:
  size: 3
  # Using custom edge nats server image for TLS verify and map support.
  serverImage: "wallyqs/nats-server"
  version: "edge-2.0.0-RC5"
  tls:
    enableHttps: true
    # Certificates to secure the NATS client connections:
    serverSecret: "nats-tls-example"
    # Certificates to secure the routes.
    routesSecret: "nats-tls-routes-example"
  auth:
    tlsVerifyAndMap: true
    clientsAuthSecret: "nats-tls-users"
    # How long to wait for authentication
    clientsAuthTimeout: 5
  pod:
    # To be able to reload the secret changes
    enableConfigReload: true
    reloaderImage: connecteverything/nats-server-config-reloader
    # Bind the port 4222 as the host port to allow external access.
    enableClientsHostPort: true
    # Initializer container that resolves the external IP from the
    # container where it is running.
    advertiseExternalIP: true
    # Image of container that resolves external IP from K8S API
    bootconfigImage: "wallyqs/nats-boot-config"
    bootconfigImageTag: "0.5.0"
  # Service account required to be able to find the external IP
  template:
    spec:
      serviceAccountName: "nats-server"
' | kubectl apply -f -

Create APP using certs

Adding a new pod which uses the certificates

Development

package main
import (
    "encoding/json"
    "flag"
    "fmt"
    "log"
    "time"
    "github.com/nats-io/go-nats"
    "github.com/nats-io/nuid"
)
func main() {
    var (
        serverList     string
        rootCACertFile string
        clientCertFile string
        clientKeyFile  string
    )
    flag.StringVar(&serverList, "s", "tls://nats-1.nats-cluster.default.svc:4222", "List of NATS of servers available")
    flag.StringVar(&rootCACertFile, "cacert", "./certs/ca.pem", "Root CA Certificate File")
    flag.StringVar(&clientCertFile, "cert", "./certs/client.pem", "Client Certificate File")
    flag.StringVar(&clientKeyFile, "key", "./certs/client-key.pem", "Client Private key")
    flag.Parse()
    log.Println("NATS endpoint:", serverList)
    log.Println("Root CA:", rootCACertFile)
    log.Println("Client Cert:", clientCertFile)
    log.Println("Client Key:", clientKeyFile)
    // Connect options
    rootCA := nats.RootCAs(rootCACertFile)
    clientCert := nats.ClientCert(clientCertFile, clientKeyFile)
    alwaysReconnect := nats.MaxReconnects(-1)
    var nc *nats.Conn
    var err error
    for {
        nc, err = nats.Connect(serverList, rootCA, clientCert, alwaysReconnect)
        if err != nil {
            log.Printf("Error while connecting to NATS, backing off for a sec... (error: %s)", err)
            time.Sleep(1 * time.Second)
            continue
        }
        break
    }
    nc.Subscribe("discovery.*.status", func(m *nats.Msg) {
        log.Printf("[Received on %q] %s", m.Subject, string(m.Data))
    })
    discoverySubject := fmt.Sprintf("discovery.%s.status", nuid.Next())
    info := struct {
        InMsgs        uint64   `json:"in_msgs"`
        OutMsgs       uint64   `json:"out_msgs"`
        Reconnects    uint64   `json:"reconnects"`
        CurrentServer string   `json:"current_server"`
        Servers       []string `json:"servers"`
    }{}
    for range time.NewTicker(1 * time.Second).C {
        stats := nc.Stats()
        info.InMsgs = stats.InMsgs
        info.OutMsgs = stats.OutMsgs
        info.Reconnects = stats.Reconnects
        info.CurrentServer = nc.ConnectedUrl()
        info.Servers = nc.Servers()
        payload, err := json.Marshal(info)
        if err != nil {
            log.Printf("Error marshalling data: %s", err)
        }
        err = nc.Publish(discoverySubject, payload)
        if err != nil {
            log.Printf("Error during publishing: %s", err)
        }
        nc.Flush()
    }
}

FROM golang:1.11.0-alpine3.8 AS builder
COPY . /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
WORKDIR /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app
RUN apk add --update git
RUN go get -u github.com/nats-io/go-nats
RUN go get -u github.com/nats-io/nuid
RUN CGO_ENABLED=0 go build -o nats-client-app -v -a ./client.go
FROM scratch
COPY --from=builder /go/src/github.com/nats-io/nats-kubernetes/examples/nats-cluster-routes-tls/app/nats-client-app /nats-client-app
ENTRYPOINT ["/nats-client-app"]

docker build . -t wallyqs/nats-client-app
docker run wallyqs/nats-client-app
docker push wallyqs/nats-client-app

Pod spec

echo ' apiVersion: apps/v1beta2 kind: Deployment
## The name of the deployment
metadata: name: nats-client-app
spec:
## This selector has to match the template.metadata.labels section
## which is below in the PodSpec
selector: matchLabels: name: nats-client-app
## Number of instances
replicas: 1
## PodSpec
template: metadata: labels: name: nats-client-app spec: volumes:
* name: "client-tls-certs"
  secret:
    secretName: "nats-tls-client-example"
  containers:
* name: nats-client-app
  command: \["/nats-client-app", "-s", "tls://nats-cluster.default.svc:4222", "-cacert", '/etc/nats-client-tls-certs/ca.pem', '-cert', '/etc/nats-client-tls-certs/client.pem', '-key', '/etc/nats-client-tls-certs/client-key.pem'\]
  image: wallyqs/nats-client-app:latest
  imagePullPolicy: Always
  volumeMounts:
  * name: "client-tls-certs"
    mountPath: "/etc/nats-client-tls-certs/"
    ' \| kubectl apply -f -