python-logstash-async
python-logstash-async copied to clipboard
eht16
Not working with Django
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"logstash": {
"()": "logstash_async.formatter.DjangoLogstashFormatter",
"message_type": "django-logstash",
"fqdn": False,
"extra_prefix": None,
"extra": {
"application": "my-app",
"project_path": os.getcwd(),
"environment": "test",
},
},
},
"handlers": {
"logstash": {
"level": "DEBUG",
"class": "logstash_async.handler.AsynchronousLogstashHandler",
"formatter": "logstash",
"transport": "logstash_async.transport.TcpTransport",
"host": "xxx.xxx.xxx.xxx",
"port": xxxx,
"database_path": "{}/logstash.db".format(os.getcwd()),
},
},
"loggers": {
"django.request": {
"handlers": ["logstash"],
"level": "DEBUG",
"propagate": True,
},
},
}
Hello, I tried your library in my Django Rest Framework application following the documentation, but nothing is working.
I know that my elastic and logstash is fully working because I'm using it with a lot of other application.
Do you have an idea why it's not working?
The configuration looks good to me, I don't see an error.
You could try adding a root logger with a console handler to check if there are any errors logged.
You don't have any TLS settings configured, are you sure your Logstash instance expects plain text JSON via TCP?
Do you see any events in the database file? If it is a transport problem, then you should see events in the database. If it is a logging configuration problem, the database is probably empty.
~~My database file is not empty, so looks like there are events in it.~~ see: #issuecomment-1283747520
I just switched the protocol to send information to Logstash using the HTTP Protocol (#usage-with-httptransport)
But same issue, I've got only the WARNING and ERROR logging levels.
From what I see, my SQLite database is empty:
So I also think it's a logging configuration problem.
But, requests are logged in the console with the INFO logging level, so I don't understand why the INFO logging level is not handled by Logstash 🤔
It's maybe an issue with gunicorn logging 🤔
python-logstash-async logs any errors it encounters to STDERR if no specific logger is configured. So, either check the STDERR of your application for errors or configure a root logger as suggested in my post above. With a configured root logger, or the specific logger named "LogProcessingWorker", you should see the errors logged to whatever handler you specify.
Example:
...
'loggers': {
'root': {
'handlers':['console'],
'level':'DEBUG',
'propagate': False,
},
...
I finally made it working! 🚀
If you want, you can add the following documentation to yours.
You can close this issue if you have no question on my work 👍
Click to see the full explanation
My LOGGING settings.py looks like this:
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"logstash": {
"()": MyLogstashFormatter,
"fqdn": False,
# To disable grouping of the extra items and have them on the top level of the log event message, simply set this option to None or the empty string.
"extra_prefix": None,
"metadata": {"beat": "django-api", "version": "1"},
"extra": {
"application": "my-django-app",
"version": "v1",
"project_path": os.getcwd(),
"environment": os.environ["ENVIRONMENT"],
},
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
},
"logstash": {
"level": "DEBUG",
"class": "logstash_async.handler.AsynchronousLogstashHandler",
"formatter": "logstash",
"transport": "logstash_async.transport.TcpTransport",
"host": "my_logstash_host.local",
"port": 1234,
"database_path": "{}/logstash.db".format(os.getcwd()),
},
},
"filters": {
"require_request": {
"()": RequireRequestFilter,
},
"require_response": {
"()": RequireResponseFilter,
},
},
"root": {
"handlers": ["console"],
"level": "INFO",
},
"loggers": {
"django": {
"handlers": ["console"],
"level": "INFO",
"propagate": False,
},
"django.request": {
"handlers": ["logstash"],
"level": "WARNING",
"propagate": True,
"filters": ["require_request", "require_response"],
},
},
}
I had to create one logging middleware that looks like this:
import contextlib
import logging
import traceback
LOGGER = logging.getLogger("django.request")
class LoggingMiddleware:
def __init__(self, get_response):
self.get_response = get_response
self.stacktrace = None
def process_exception(self, _request, _exception):
self.stacktrace = traceback.format_exc()
return None
def __call__(self, request):
with contextlib.suppress(UnicodeDecodeError, AttributeError):
body = request.body.decode("utf-8")
response = self.get_response(request)
log_func = LOGGER.info
if response.status_code in range(400, 499):
log_func = LOGGER.warning
if response.status_code in range(500, 599):
log_func = LOGGER.error
log_func(
"%s",
request,
extra={
"request": request,
"response": response,
"body": body or "",
"stack_trace": self.stacktrace or "",
},
)
return response
Two filters to remove message that has no request or response object:
import logging
class RequireRequestFilter(logging.Filter):
def filter(self, record):
return hasattr(record, "request") and hasattr(record.request, "META")
class RequireResponseFilter(logging.Filter):
def filter(self, record):
return hasattr(record, "response")
And then I have a custom formatter to display the data I want:
import json
from typing import Any, Dict
from django.core.exceptions import DisallowedHost
from django.http import HttpRequest
from logstash_async.formatter import DjangoLogstashFormatter
def get_host(request: HttpRequest) -> str:
try:
return request.get_host()
except DisallowedHost:
if "HTTP_HOST" in request.META:
return request.META["HTTP_HOST"]
return request.META["SERVER_NAME"]
def get_interesting_fields(record) -> Dict[str, Any]:
request: HttpRequest = record.request
body: str = record.body if hasattr(record, "body") else ""
fields = {
"request": {
"method": request.method,
"path": request.get_full_path(),
"body": body,
"headers": json.dumps(dict(request.headers)),
"user_agent": request.META.get("HTTP_USER_AGENT", ""),
"host": get_host(request),
"uri": request.build_absolute_uri(),
"remote_address": request.META.get("REMOTE_ADDR", ""),
"referer": request.META.get("HTTP_REFERER", ""),
}
}
response = record.response
fields["response"] = {
"status_code": response.status_code,
"headers": json.dumps(dict(response.headers)),
}
return fields
UNWANTED_KEYS = [
"body",
"request",
"response",
"req_user",
"req_method",
"req_useragent",
"req_remote_address",
"req_host",
"req_uri",
"req_user",
"req_method",
"req_referer",
]
def filter_unwanted_fields(message: Dict[str, Any]) -> Dict[str, Any]:
return {key: value for key, value in message.items() if key not in UNWANTED_KEYS}
class MyLogstashFormatter(DjangoLogstashFormatter):
def format(self, record):
message = json.loads(super().format(record))
# noinspection PyTypeChecker
django_fields = get_interesting_fields(record)
message.update({"rest": django_fields})
if hasattr(record, "stack_trace") and record.stack_trace:
message.update({"stack_trace": record.stack_trace})
message = filter_unwanted_fields(message)
return json.dumps(message, ensure_ascii=True)