0

Previously, you need to install the mongodb first before using this django mongodb log;

This installation is for Ubunntu 16.04

sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
sudo apt-get update
sudo apt-get install -y mongodb-org
sudo systemctl start mongod
sudo systemctl status mongod
sudo systemctl enable mongod
sudo ufw allow from 127.0.0.1/32 to any port 27017
sudo ufw status

Install django-mongolog

pip install django-mongolog

Add mongolog in installed apps in your project_name/settings.py

INSTALLED_APPS = (
    ...
    'mongolog',
)

project_name/logs.py

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime

from django.conf import settings

# All logging handlers configurations.
# 'propagate': False = mean is the error logs ins't duplicates to another file logs.

NOW = datetime.datetime.now()
DAY_NAME = NOW.strftime('%A').lower()

MAXIMUM_FILE_LOGS = 1024 * 1024 * 10  # 10 MB
BACKUP_COUNT = 5


LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '[%(levelname)s] %(asctime)s %(name)s: %(message)s'
        },
    },
    'handlers': {
        'mongolog': {
            'level': 'DEBUG',
            'class': 'mongolog.SimpleMongoLogHandler',

            # Set the connection string to the mongo instance.
            # mongodb://[username]:[password]@[host]:[port]/[database]
            'connection': settings.MONGODB_CONNECTION_URL,

            # define mongo collection the log handler should use.  Default is mongolog
            # This is useful if you want different handlers to use different collections
            'collection': 'mongolog',
            'formatter': 'standard',
        },
        'mail_admins_handler': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'email_backend': 'django.core.mail.backends.smtp.EmailBackend'
        },
    },
    'root': {
        'handlers': ['mongolog'],
        'level': 'DEBUG'
    },
    'loggers': {
        'django.request': {
            'handlers': [
                'mongolog',
                'mail_admins_handler'
            ],
            'level': 'DEBUG',
            'propagate': False
        },
    }
}

project_name/middleware.py

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import logging

from django.http import Http404
from django.http.request import RawPostDataException
from django.utils.deprecation import MiddlewareMixin


class SessionLogMiddleware(MiddlewareMixin):
    """
    this middleware to create a log
    by current user
    """

    def save_mongodb_logging(self, request, response=None, exception=None, status_code=None):
        """
        {
            "_id" : ObjectId("5d2d7200b2b76e29fc94080f"),
            "module" : "middleware",
            "line" : 94,
            "thread" : NumberLong(140471566464768),
            "created" : ISODate("2019-07-16T13:43:12.820Z"),
            "level" : "INFO",
            "path" : "/home/path/to/project/utils/middleware.py",
            "msg" : {
                "REQUEST_DATA" : {
                    "start_time" : "2019-07-04 16:38:01",
                    "finish_time" : "2019-07-04 16:39:22",
                    "quiz_choices_id" : [5, 30, 1, 3, 9]
                },
                "METHOD" : "POST",
                "URL" : "http://localhost:8000/api/quiz",
                "RESPONSE_DATA" : {
                    "message" : "Answer successfully created!",
                    "customer_quiz" : {
                        "customer_quiz_id" : 34,
                        "created_at" : "2019-07-16T13:43:12.538089",
                        "total_correct_answers" : 3,
                        "customer" : 1,
                        "start_time" : "2019-07-16T13:43:12.538062",
                        "total_incorrect_answers" : 2,
                        "deleted_at" : null,
                        "finish_time" : "2019-07-04T16:39:22",
                        "updated_at" : "2019-07-16T13:43:12.538098"
                    },
                    "action" : true
                },
                "HEADERS" : {
                    "Cache-Control" : "no-cache",
                    "Accept-Encoding" : "gzip, deflate",
                    "User-Agent" : "PostmanRuntime/7.6.1",
                    "Connection" : "keep-alive",
                    "Content-Type" : "application/json",
                    "Cookie" : "sessionid=ugs3iaw9ltqtdrh65rhfbpo2ct6uq83o",
                    "Accept" : "*/*",
                    "Content-Length" : "116",
                    "Host" : "localhost:8000",
                    "Authorization" : "Token  5dace55366d8f36102c04c7b9fbdbdd7352f2ffc",
                    "Postman-Token" : "61873e1e-0c1b-43f0-b048-a84bbd87ca4f"
                },
                "USER" : {
                    "email" : "[email protected]",
                    "phone_number" : "0838183777313",
                    "username" : "username-1",
                    "id" : 1
                },
                "ERROR_MESSAGE": null,
                "STATUS_CODE": 200
            },
            "func" : "save_mongodb_logging",
            "dates" : [
                ISODate("2019-07-16T13:43:12.820Z")
            ],
            "name" : "logger:mongolog",
            "counter" : 1,
            "uuid" : "ebe40b9a7bfb598ab3ec1c473fb93e31",
            "process" : 26388,
            "filename" : "middleware.py"
        }
        """
        # this `user` usage for models.User or models.Customer
        user = request._cached_user if hasattr(request, '_cached_user') else request.user
        request_data = None
        response_data = None

        if request.method in ['POST', 'PUT', 'DELETE']:

            def clean_text(text):
                if isinstance(text, bytes):
                    return text.decode('utf-8')\
                               .replace('\\n', '')\
                               .replace('\\t', '')\
                               .replace('\\r', '')
                return str(text)

            if hasattr(request, 'body'):
                try:
                    request_data = json.loads(clean_text(request.body))
                except RawPostDataException:
                    # doesn't recorded
                    pass
                except Exception:
                    request_data = clean_text(request.body)

            if hasattr(response, 'content'):
                try:
                    response_data = json.loads(clean_text(response.content))
                except RawPostDataException:
                    # doesn't recorded
                    pass
                except Exception:
                    response_data = clean_text(response.content)

        log_data = {
            'HEADERS': eval(str(request.headers)),
            'METHOD': request.method,
            'USER': {
                'id': user.pk if hasattr(user, 'pk') else None,
                'username': user.username if hasattr(user, 'username') else None,
                'phone_number': user.phone_number if hasattr(user, 'phone_number') else None,
                'email': user.email if hasattr(user, 'email') else None
            },
            'URL': request.build_absolute_uri(),
            'REQUEST_DATA': request_data,
            'RESPONSE_DATA': response_data,
            'ERROR_MESSAGE': exception,
            'STATUS_CODE': status_code
        }

        log = logging.getLogger('logger:mongolog')
        log.error(log_data) if exception else log.debug(log_data)

    def process_exception(self, request, exception):
        print(type(exception), exception)  # just for debug
        status_code = 404 if isinstance(exception, Http404) else 500

        self.save_mongodb_logging(request,
                                  exception=exception,
                                  status_code=status_code)
        return  # no return anything

    def process_response(self, request, response):
        if not response.status_code >= 400:
            response_data = {'request': request,
                             'response': response,
                             'status_code': response.status_code}
            self.save_mongodb_logging(**response_data)
        return response

management/commands/clear_logs.py

# -*- coding: utf-8 -*-

import datetime
import pymongo

from django.conf import settings
from django.core.management.base import BaseCommand


NOW_DATE = datetime.datetime.now()


class Command(BaseCommand):
    help = "Command to clear the mongodb logs"

    def is_max(self, database, collection):
        """
        function to check whenever `sizeOnDisk`
        was greater than `settings.MONGODB_MAX_BYTES`

        :param `database` is mongodb client connection.
        :param `collection` is collection to check his size.
        :return boolean

        >>> list(db.list_databases())
        [{'name': 'admin', 'sizeOnDisk': 143360.0, 'empty': False},
         {'name': 'local', 'sizeOnDisk': 65536.0, 'empty': False},
         {'name': 'mongolog', 'sizeOnDisk': 204800.0, 'empty': False}]
        """
        for database_info in database.list_databases():
            if isinstance(database_info, dict):
                if database_info.get('name') == collection:
                    if database_info.get('sizeOnDisk') > settings.MONGODB_MAX_BYTES:
                        return True
        return False

    def clear_logs(self, collection='mongolog'):
        """
        function to clean the database `mongolog`
        """
        database = pymongo.MongoClient(settings.MONGODB_CONNECTION_URL)
        if self.is_max(database, collection):
            expiration_date = NOW_DATE - datetime.timedelta(days=settings.MONGODB_MAX_DAYS)
            collection = database.mongolog.get_collection(collection)
            collection.delete_many({"created": {"$lte": expiration_date}})

        message = '[i] mongolog successfully cleaned, for logs before %s-days ago!' % settings.MONGODB_MAX_DAYS
        self.stdout.write(message)

    def handle(self, *args, **kwargs):
        # noqa
        self.clear_logs()

project_name/settings.py

MONGODB_CONNECTION_URL = 'mongodb://db_user:[email protected]:27017/mongolog'
MONGODB_MAX_BYTES = 1073741824 * 2  # 2 GB
MONGODB_MAX_DAYS = 30 * 3           # 3 months


MIDDLEWARE = [
    ....

    'apps.utils.middleware.SessionLogMiddleware',
]

And the last step is setup the cronjobs.

crontab -e

# Clear mongolog every day at 01:00 AM, see: https://crontab.guru/every-day-at-1am
0 1 * * * source /path/to/env-project/bin/activate && cd /path/to/env-project/project_name/ && ./manage.py clear_logs
logging mongodb database django

Your Answer

blog comments powered by Disqus