{
"_index": "readonlyrest_audit-2021.05",
"_type": "_doc",
"_id": "314953487--1404332622#19829097",
"_score": 1,
"_source": {
"headers": [
"x-elastic-product-origin",
"user-agent",
"Content-Length",
"x-elastic-client-meta",
"content-type",
"Accept-Charset",
"connection",
"Host"
],
"acl_history": "[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]] (Longer stuff here, but the Kibana important part is this)",
"origin": "x.x.x.x/32",
"match": false,
"final_state": "FORBIDDEN",
"destination": "x.x.x.x/32",
"task_id": 19829097,
"type": "SearchRequest",
"req_method": "POST",
"content": "{\"seq_no_primary_term\":true,\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"should\":[{\"range\":{\"api_key_pending_invalidation.createdAt\":{\"lte\":\"2021-05-14T11:06:20.400Z\"}}}],\"minimum_should_match\":1}},{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"term\":{\"type\":\"api_key_pending_invalidation\"}}],\"must_not\":[{\"exists\":{\"field\":\"namespace\"}},{\"exists\":{\"field\":\"namespaces\"}}]}}],\"minimum_should_match\":1}}]}},\"sort\":[{\"api_key_pending_invalidation.createdAt\":{\"order\":\"asc\",\"unmapped_type\":\"date\"}}]}",
"path": "/.kibana/_search",
"indices": [],
"@timestamp": "2021-05-14T12:06:20Z",
"content_len_kb": 0,
"correlation_id": "28031639-1ee5-42f1-941d-2f4e15fd9818",
"processingMillis": 2,
"action": "indices:data/read/search",
"block": "default",
"id": "314953487--1404332622#19829097",
"content_len": 480
},
"fields": {
"@timestamp": [
"2021-05-14T12:06:20.000Z"
]
}
}
{"type": "server", "timestamp": "2021-05-14T14:17:18,665+02:00", "level": "DEBUG", "component": "o.e.a.a.c.n.t.c.TransportCancelTasksAction", "cluster.name": "XXX", "node.name": "nlhrl1tsmc04", "message": "failed to execute on node [o6WNXxsIRzGTtAoLj9HmzA]", "cluster.uuid": "XMFAmE8rSOydA8jmOkoHYg", "node.id": "o6WNXxsIRzGTtAoLj9HmzA" ,
"stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [nlhrl1tsmc04][x.x.x.x:9300][cluster:admin/tasks/cancel[n]]",
"Caused by: org.elasticsearch.ResourceNotFoundException: task [o6WNXxsIRzGTtAoLj9HmzA:19881274] is not found",
"at org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction.processTasks(TransportCancelTasksAction.java:63) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction.processTasks(TransportCancelTasksAction.java:34) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.action.support.tasks.TransportTasksAction.nodeOperation(TransportTasksAction.java:90) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.action.support.tasks.TransportTasksAction.access$900(TransportTasksAction.java:52) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.action.support.tasks.TransportTasksAction$NodeTransportHandler.messageReceived(TransportTasksAction.java:310) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.action.support.tasks.TransportTasksAction$NodeTransportHandler.messageReceived(TransportTasksAction.java:306) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:61) ~[elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.transport.TransportService$8.doRun(TransportService.java:912) [elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.11.2.jar:7.11.2]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.11.2.jar:7.11.2]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]"] }
{
"cluster.name": "XXX",
"cluster.uuid": "XMFAmE8rSOydA8jmOkoHYg",
"component": "t.b.r.a.l.AccessControlLoggingDecorator",
"level": "INFO",
"message": "\u001b[35mFORBIDDEN by default req={ ID:1796573475--743041753#19881274, TYP:SearchRequest, CGR:N/A, USR:[no info about user], BRS:true, KDX:null, ACT:indices:data/read/search, OA:x.x.x.x/32, XFF:null, DA:x.x.x.x/32, IDX:.kibana, MET:POST, PTH:/.kibana/_search, CNT:<OMITTED, LENGTH=480.0 B> , HDR:Accept-Charset=utf-8, Content-Length=480, Host=x.x.x.x:9200, connection=close, content-type=application/json, user-agent=elasticsearch-js/7.11.0-rc.1 (linux 3.10.0-1160.15.2.el7.x86_64-x64; Node.js v14.16.0), x-elastic-client-meta=es=7.11.0-rc.1,js=14.16.0,t=7.11.0-rc.1,hc=14.16.0, x-elastic-product-origin=kibana, HIS:[::KIBANA-SRV::-> RULES:[auth_key->false] RESOLVED:[indices=.kibana]] (also here more config, not relevant }\u001b[0m",
"node.id": "o6WNXxsIRzGTtAoLj9HmzA",
"node.name": "YYY",
"timestamp": "2021-05-14T14:17:18,664+02:00",
"type": "server"
}
We noticed failed cancel tasks during testing.
Looks like Kibana Alert is not using the defined credentials towards Elasticsearch for checking expired API keys within Alerting plugin?