Hi,
I am trying to test the ROR Kibana pluging 1.29.0 on ES 7.10.2, Red Hat 7.6. I have some error issues with Kibana Stack Monitoring. When I open Kibana Stack Monitoring, I get a popup message “Error fetching alert status. Internal Server Error”. But it seems the Stack monitoring displays all monitored data correctly (cpu, mem, disk etc). I found these errors in logs:
kibana.log
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“http”],“pid”:1059,“message”:“{ Error: Insufficient user permissions for monitoring data\n at handleAuthError (/opt/app/elastic/kibana-7.10.2-linux-x86_64/x-pack/plugins/monitoring/server/lib/errors/auth_errors.js:48:30)\n at handleError (/opt/app/elastic/kibana-7.10.2-linux-x86_64/x-pack/plugins/monitoring/server/lib/errors/handle_error.js:24:45)\n at npRoute.router.post (/opt/app/elastic/kibana-7.10.2-linux-x86_64/x-pack/plugins/monitoring/server/routes/api/v1/alerts/status.js:65:37)\n at process._tickCallback (internal/process/next_tick.js:68:7)\n data: null,\n isBoom: true,\n isServer: false,\n output:\n { statusCode: 403,\n payload:\n { statusCode: 403,\n error: ‘Forbidden’,\n message: ‘Insufficient user permissions for monitoring data’ },\n headers: {} },\n reformat: [Function],\n typeof: [Function: forbidden] }”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“log”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:[“error”,“elasticsearch”,“data”],“pid”:1059,“message”:“[ResponseError]: forbidden”}
{“type”:“error”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:,“pid”:1059,“level”:“error”,“error”:{“message”:“Internal Server Error”,“name”:“Error”,“stack”:“Error: Internal Server Error\n at HapiResponseAdapter.toInternalError (/opt/app/elastic/kibana-7.10.2-linux-x86_64/src/core/server/http/router/response_adapter.js:69:19)\n at Router.handle (/opt/app/elastic/kibana-7.10.2-linux-x86_64/src/core/server/http/router/router.js:177:34)\n at process._tickCallback (internal/process/next_tick.js:68:7)”},“url”:{“protocol”:null,“slashes”:null,“auth”:null,“host”:null,“port”:null,“hostname”:null,“hash”:null,“search”:null,“query”:{},“pathname”:“/api/monitoring/v1/alert/F4_29skjQImO80K5UeOu-w/status”,“path”:“/api/monitoring/v1/alert/F4_29skjQImO80K5UeOu-w/status”,“href”:“/api/monitoring/v1/alert/F4_29skjQImO80K5UeOu-w/status”},“message”:“Internal Server Error”}
{“type”:“response”,“@timestamp”:“2021-05-10T06:59:32Z”,“tags”:,“pid”:1059,“method”:“post”,“statusCode”:500,“req”:{“url”:“/api/monitoring/v1/alert/F4_29skjQImO80K5UeOu-w/status”,“method”:“post”,“headers”:{“host”:“elk1:5601”,“kbn-version”:“7.10.2”,“user-agent”:“Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36”,“content-type”:“application/json”,“accept”:“/”,“origin”:“http://elk1:5601”,“referer”:“http://elk1:5601/app/monitoring",“accept-encoding”:"gzip, deflate”,“accept-language”:“sk-SK,sk;q=0.9,cs;q=0.8,en-US;q=0.7,en;q=0.6”,“connection”:“close”,“x-ror-pkp-kibana-token”:“3dd58dyy56epkbz6fvwnudsq6uk601”,“content-length”:“55”,“accept-charset”:“utf-8”},“remoteAddress”:“127.0.0.1”,“userAgent”:“Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36”,“referer”:“http://elk1:5601/app/monitoring"},“res”:{“statusCode”:500,“responseTime”:239,“contentLength”:9},“message”:"POST /api/monitoring/v1/alert/F4_29skjQImO80K5UeOu-w/status 500 239ms - 9.0B”}
{“type”:“response”,“@timestamp”:“2021-05-10T06:59:31Z”,“tags”:,“pid”:1059,“method”:“post”,“statusCode”:200,“req”:{“url”:“/api/monitoring/v1/clusters/F4_29skjQImO80K5UeOu-w”,“method”:“post”,“headers”:{“host”:“elk1:5601”,“accept”:“application/json, text/plain, /”,“kbn-version”:“7.10.2”,“user-agent”:“Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36”,“content-type”:“application/json;charset=UTF-8”,“origin”:“http://elk1:5601”,“referer”:“http://elk1:5601/app/monitoring",“accept-encoding”:"gzip, deflate”,“accept-language”:“sk-SK,sk;q=0.9,cs;q=0.8,en-US;q=0.7,en;q=0.6”,“connection”:“close”,“x-ror-pkp-kibana-token”:“3dd58dyy56epkbz6fvwnudsq6uk601”,“content-length”:“101”,“accept-charset”:“utf-8”},“remoteAddress”:“127.0.0.1”,“userAgent”:“Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36”,“referer”:“http://elk1:5601/app/monitoring"},“res”:{“statusCode”:200,“responseTime”:390,“contentLength”:9},“message”:"POST /api/monitoring/v1/clusters/F4_29skjQImO80K5UeOu-w 200 390ms - 9.0B”}
elasticsearch.log
[2021-05-10T08:59:32,181][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1166431550-1169813989#2432, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=408.0 B> , HDR:Accept-Charset=utf-8, Content-Length=408, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,195][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:207464428-2090030670#2449, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=399.0 B> , HDR:Accept-Charset=utf-8, Content-Length=399, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,198][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1051026762–2021555209#2450, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=400.0 B> , HDR:Accept-Charset=utf-8, Content-Length=400, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,201][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1300418923-889423801#2455, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=404.0 B> , HDR:Accept-Charset=utf-8, Content-Length=404, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,199][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1734600423–1337757654#2452, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=403.0 B> , HDR:Accept-Charset=utf-8, Content-Length=403, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,214][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:406989666-1376100776#2469, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=413.0 B> , HDR:Accept-Charset=utf-8, Content-Length=413, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,217][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1978942833-39636153#2471, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=406.0 B> , HDR:Accept-Charset=utf-8, Content-Length=406, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,219][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1669870861-514056536#2475, TYP:GetRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/get, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:GET, PTH:/.kibana/_doc/space:default, CNT:<N/A>, HDR:Accept-Charset=utf-8, Host=elk1:9200, connection=close, content-length=0, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,224][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1904266883-1345573647#2482, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=420.0 B> , HDR:Accept-Charset=utf-8, Content-Length=420, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,226][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1536736492–569193305#2484, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:elk1/32, XFF:null, DA:elk1/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=415.0 B> , HDR:Accept-Charset=utf-8, Content-Length=415, Host=elk1:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.10.0-rc.1 (linux 3.10.0-957.el7.x86_64-x64; Node.js v10.23.1), x-elastic-product-origin=kibana, x-opaque-id=7d10d11b-1b6f-40b1-95a2-4d565ac46a04, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]], }[0m
[2021-05-10T08:59:32,228][INFO ][t.b.r.a.l.AccessControlLoggingDecorator] [elk1] [35mFORBIDDEN by default req={ ID:1158955939-77205228#2487, TYP:SearchRequest, CGR:N/A, USR:[no i
My confs:
elasticsearch.yml:
cluster.name: elk
node.name: elk1
path.data: /opt/app/elastic/elasticsearch/data
path.logs: /opt/app/elastic/elasticsearch/logs
bootstrap.memory_lock: true
network.host: elk1
discovery.seed_hosts: [‘elk1’]
xpack.security.enabled: false
readonlyrest.yml:
readonlyrest:
enable: true # optional, defaults=true if at least 1 “access_control_rules” block
prompt_for_basic_auth: false
access_control_rules:
# Kibana rules
- name: "::KIBANA-SRV::"
auth_key: kibana:kibana
type: allow
verbosity: error
kibana.yml
elasticsearch.username: kibana
elasticsearch.password: kibana
xpack.security.enabled: false
status.allowAnonymous: true
cluster.initial_master_nodes: elk1