Kibana Enterprise ROR with Let's Encrypt


(Craig Sands) #1

I’m trying to configure SSL between ES and Kibana with a Let’s Encrypt cert. But I keep getting “Error: custom client factory is already set, unable to replace the current one.” Below is my configuration (I’m using docker-compose for POC). At the end is the full traceback.

Set Env Vars

export KEY_PASS=readonlyrest
export KEYSTORE_PASS=readonlyrest

Generate Certs

docker run -it --rm \
  -v ${PWD}/certs:/etc/letsencrypt/archive \
  -v /home/`logname`/.aws/credentials:/root/.aws/credentials \
  certbot/dns-route53 \
    certonly \
    --register-unsafely-without-email \
    --agree-tos \
    --server https://acme-v02.api.letsencrypt.org/directory \
    --dns-route53 \
    -d *.example.com

Bundle certs into PKCS keystore

openssl \
  pkcs12 \
  -export \
  -in certs/fullchain1.pem \
  -inkey certs/privkey1.pem \
  -out keystore.p12 \
  -password pass:${KEY_PASS}

Create JKS keystore from PKCS bundle

keytool \
  -importkeystore \
  -deststorepass ${KEYSTORE_PASS} \
  -destkeypass ${KEY_PASS} \
  -destkeystore keystore.jks \
  -srckeystore keystore.p12 \
  -srcstoretype PKCS12 \
  -srcstorepass ${KEYSTORE_PASS}

ES Dockerfile:

FROM docker.elastic.co/elasticsearch/elasticsearch:6.4.2
COPY readonlyrest-1.16.28_es6.4.2.zip /tmp/ror.zip
RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:///tmp/ror.zip

Kibana Dockerfile:

FROM docker.elastic.co/kibana/kibana:6.4.2
COPY readonlyrest_kbn_enterprise-1.16.27_es6.4.2.zip /tmp/ror.zip
RUN /usr/share/kibana/bin/kibana-plugin install file:///tmp/ror.zip
RUN touch optimize/bundles/readonlyrest_kbn.style.css

readonlyrest.yml:

readonlyrest:
  ssl:
    keystore_file: "keystore.jks"
    keystore_pass: "${KEYSTORE_PASS}"
    key_pass: "${KEY_PASS}"
    key_alias: "1"  #Is this correct?
  access_control_rules:
  - name: "::KIBANA-SRV::"
    auth_key: kibana:kibana

docker-compose.yml:

version: '3'
services:
  es01:
    build: ./builds/elasticsearch-readonlyrest
    environment:
      - node.name=es01
      - discovery.zen.minimum_master_nodes=1
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Dcom.readonlyrest.reloadsettingsonboot=true"
      - http.type=ssl_netty4
      - xpack.license.self_generated.type=trial
      - xpack.monitoring.enabled=true
      - xpack.security.enabled=false
      - KEY_PASS
      - KEYSTORE_PASS
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata1:/usr/share/elasticsearch/data
      - ./readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml
      - ./keystore.jks:/usr/share/elasticsearch/config/keystore.jks
    ports:
      - 9200:9200
    networks:
      esnet:
  kibana:
    build: ./builds/kibana-readonlyrest
    environment:
      ELASTICSEARCH_URL: https://es01:9200
      ELASTICSEARCH_USERNAME: kibana
      ELASTICSEARCH_PASSWORD: kibana
      ELASTICSEARCH_SSL_CERTIFICATE: /usr/share/kibana/config/certs/fullchain1.pem
      ELASTICSEARCH_SSL_KEY: /usr/share/kibana/config/certs/privkey1.pem
      ELASTICSEARCH_SSL_VERIFICATIONMODE: none
      XPACK_MONITORING_ENABLED: "true"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./certs:/usr/share/kibana/config/certs
    ports:
      - 5601:5601
    depends_on:
      - es01
    networks:
      esnet:
volumes:
  esdata1:
    driver: local
networks:
  esnet:

ERROR Traceback:

kibana_1  | {"type":"error","@timestamp":"2018-11-08T15:10:57Z","tags":["fatal"],"pid":1,"level":"fatal","error":{"message":"custom client factory is already set, unable to replace the current one","name":"Error","stack":"Error: custom client factory is already set, unable to replace the current one\n    at ScopedSavedObjectsClientProvider.setClientFactory (/usr/share/kibana/src/server/saved_objects/service/lib/scoped_client_provider.js:52:13)\n    at Object.setScopedSavedObjectsClientFactory (/usr/share/kibana/src/server/saved_objects/service/create_saved_objects_service.js:80:75)\n    at k (/usr/share/kibana/plugins/readonlyrest_kbn/index.js:1:3432)\n    at Plugin.init [as externalInit] (/usr/share/kibana/plugins/readonlyrest_kbn/index.js:1:3401)\n    at asyncRegister (/usr/share/kibana/src/server/plugins/lib/plugin.js:72:20)\n    at Object.register (/usr/share/kibana/src/server/plugins/lib/plugin.js:77:7)\n    at Object.target [as register] (/usr/share/kibana/node_modules/hapi/node_modules/joi/lib/object.js:77:34)\n    at each (/usr/share/kibana/node_modules/hapi/lib/plugin.js:317:14)\n    at iterate (/usr/share/kibana/node_modules/items/lib/index.js:36:13)\n    at Object.exports.serial (/usr/share/kibana/node_modules/items/lib/index.js:39:9)\n    at module.exports.internals.Server.internals.Plugin.register (/usr/share/kibana/node_modules/hapi/lib/plugin.js:320:11)\n    at Promise (/usr/share/kibana/node_modules/hapi/lib/promises.js:24:16)\n    at new Promise (<anonymous>)\n    at Object.exports.wrap (/usr/share/kibana/node_modules/hapi/lib/promises.js:13:12)\n    at module.exports.internals.Server.internals.Plugin.register (/usr/share/kibana/node_modules/hapi/lib/plugin.js:164:25)\n    at Plugin.init (/usr/share/kibana/src/server/plugins/lib/plugin.js:82:28)\n    at Plugin.init (/usr/share/kibana/node_modules/lodash/index.js:7363:25)\n    at callPluginHook (/usr/share/kibana/src/server/plugins/lib/call_plugin_hook.js:37:25)\n    at <anonymous>"},"message":"custom client factory is already set, unable to replace the current one"}
kibana_1  | FATAL Error: custom client factory is already set, unable to replace the current one
kibana_1  |     at ScopedSavedObjectsClientProvider.setClientFactory (/usr/share/kibana/src/server/saved_objects/service/lib/scoped_client_provider.js:52:13)
kibana_1  |     at Object.setScopedSavedObjectsClientFactory (/usr/share/kibana/src/server/saved_objects/service/create_saved_objects_service.js:80:75)
kibana_1  |     at k (/usr/share/kibana/plugins/readonlyrest_kbn/index.js:1:3432)
kibana_1  |     at Plugin.init [as externalInit] (/usr/share/kibana/plugins/readonlyrest_kbn/index.js:1:3401)
kibana_1  |     at asyncRegister (/usr/share/kibana/src/server/plugins/lib/plugin.js:72:20)
kibana_1  |     at Object.register (/usr/share/kibana/src/server/plugins/lib/plugin.js:77:7)
kibana_1  |     at Object.target [as register] (/usr/share/kibana/node_modules/hapi/node_modules/joi/lib/object.js:77:34)
kibana_1  |     at each (/usr/share/kibana/node_modules/hapi/lib/plugin.js:317:14)
kibana_1  |     at iterate (/usr/share/kibana/node_modules/items/lib/index.js:36:13)
kibana_1  |     at Object.exports.serial (/usr/share/kibana/node_modules/items/lib/index.js:39:9)
kibana_1  |     at module.exports.internals.Server.internals.Plugin.register (/usr/share/kibana/node_modules/hapi/lib/plugin.js:320:11)
kibana_1  |     at Promise (/usr/share/kibana/node_modules/hapi/lib/promises.js:24:16)
kibana_1  |     at new Promise (<anonymous>)
kibana_1  |     at Object.exports.wrap (/usr/share/kibana/node_modules/hapi/lib/promises.js:13:12)
kibana_1  |     at module.exports.internals.Server.internals.Plugin.register (/usr/share/kibana/node_modules/hapi/lib/plugin.js:164:25)
kibana_1  |     at Plugin.init (/usr/share/kibana/src/server/plugins/lib/plugin.js:82:28)
kibana_1  |     at Plugin.init (/usr/share/kibana/node_modules/lodash/index.js:7363:25)
kibana_1  |     at callPluginHook (/usr/share/kibana/src/server/plugins/lib/call_plugin_hook.js:37:25)
kibana_1  |     at <anonymous>
docker-examples_kibana_1 exited with code 1

(Craig Sands) #2

Figured out the issue. I skipped over the documentation where you also had to specify xpack.security.enabled=“false” in the kibana.yml. So here is the new kibana env:

ELASTICSEARCH_URL: https://es01:9200
ELASTICSEARCH_USERNAME: kibana
ELASTICSEARCH_PASSWORD: kibana
ELASTICSEARCH_SSL_CERTIFICATE: /usr/share/kibana/config/certs/fullchain1.pem
ELASTICSEARCH_SSL_KEY: /usr/share/kibana/config/certs/privkey1.pem
ELASTICSEARCH_SSL_VERIFICATIONMODE: none
XPACK_MONITORING_ENABLED: "true"
XPACK_SECURITY_ENABLED: "false"

(Simone Scarduzio) #3

Well done @craigsands, your experience will serve well other users!