[change] Implement map dashboard using netjsongraph.js #393

[totallynotvaishnav]

trafficstars

closes #393

Checks:

• [x] I have manually tested the proposed changes
• [ ] I have written new test cases to avoid regressions (if necessary)
• [ ] I have updated the documentation (e.g. README.rst)

[coveralls]

Coverage decreased (-0.009%) to 98.782% when pulling fab54a0513b07eb9f628988b13b8fe89ec739356 on issues/393-use-netjsongraph-for-dashboard into 17cedfbb030111a92e3102ec18df0540e73438d8 on master.

[totallynotvaishnav]

Can we have more neutral colors for the buttons that allow to control the map? I am referring to full screen, zoom in and zoom out. The reason for this request is that it is a lot easier to change the main theme colors if we use netutral colors for the map.

Yeah sure. I will apply the similar colors that was present in the previous version.

Is it possible to maintain those features?

Yeah I will add those features.

[totallynotvaishnav]

Screenshots

[nemesifier]

I am getting this error when I open the dashboard:

Exception Type: TypeError at /admin/
Exception Value: Object of type ListWithLazyItems is not JSON serializable

Here's the full stack trace of my current dev env (note that I am using Django 3.2, which is still supported by OpenWISP):

Environment:


Request Method: GET
Request URL: http://localhost:8000/admin/

Django Version: 3.2.13
Python Version: 3.8.10
Installed Applications:
['django.contrib.auth',
 'django.contrib.contenttypes',
 'django.contrib.sessions',
 'django.contrib.messages',
 'django.contrib.staticfiles',
 'django.contrib.gis',
 'django.contrib.sites',
 'allauth',
 'allauth.account',
 'allauth.socialaccount',
 'django_extensions',
 'django_filters',
 'openwisp_controller.config',
 'openwisp_controller.connection',
 'openwisp_controller.pki',
 'openwisp_controller.geo',
 'openwisp_users',
 'openwisp_ipam',
 'openwisp_monitoring.monitoring',
 'openwisp_monitoring.device',
 'openwisp_monitoring.check',
 'nested_admin',
 'openwisp_notifications',
 'openwisp_utils.admin_theme',
 'admin_auto_filters',
 'django.contrib.admin',
 'django.forms',
 'sortedm2m',
 'reversion',
 'leaflet',
 'flat_json_widget',
 'rest_framework',
 'rest_framework.authtoken',
 'rest_framework_gis',
 'drf_yasg',
 'channels',
 'import_export',
 'debug_toolbar']
Installed Middleware:
['django.middleware.security.SecurityMiddleware',
 'django.contrib.sessions.middleware.SessionMiddleware',
 'django.middleware.common.CommonMiddleware',
 'django.middleware.csrf.CsrfViewMiddleware',
 'django.contrib.auth.middleware.AuthenticationMiddleware',
 'django.contrib.messages.middleware.MessageMiddleware',
 'django.middleware.clickjacking.XFrameOptionsMiddleware',
 'debug_toolbar.middleware.DebugToolbarMiddleware']


Template error:
In template /home/nemesis/Code/openwisp/openwisp-monitoring/openwisp_monitoring/device/templates/admin/dashboard/device_map.html, error at line 13
   Object of type ListWithLazyItems is not JSON serializable
   3 : {% load static %}
   4 : <script type="text/javascript">
   5 :   window._owGeoMapConfig = {
   6 :     geoJsonUrl: '{{ monitoring_location_geojson_url }}',
   7 :     locationDeviceUrl: '{{ monitoring_device_list_url }}?page_size=5'
   8 :   }
   9 : </script>
   10 : <script type="text/javascript" src={%static 'monitoring/js/lib/netjsongraph.min.js' %}></script>
   11 : <script type="text/javascript" src={%static 'monitoring/js/lib/leaflet.fullscreen.min.js' %}></script>
   12 : <div id='leaflet-config'>
   13 :    {% leaflet_json_config %} 
   14 : </div>
   15 : <div id="device-map-container">
   16 :   <div class="ow-loading-spinner"></div>
   17 : </div>
   18 : 

Traceback (most recent call last):
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47, in inner
    response = get_response(request)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/core/handlers/base.py", line 204, in _get_response
    response = response.render()
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/response.py", line 105, in render
    self.content = self.rendered_content
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/response.py", line 83, in rendered_content
    return template.render(context, self._request)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/backends/django.py", line 61, in render
    return self.template.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 170, in render
    return self._render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/test/utils.py", line 100, in instrumented_test_render
    return self.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/loader_tags.py", line 150, in render
    return compiled_parent._render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/test/utils.py", line 100, in instrumented_test_render
    return self.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/loader_tags.py", line 150, in render
    return compiled_parent._render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/test/utils.py", line 100, in instrumented_test_render
    return self.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/loader_tags.py", line 150, in render
    return compiled_parent._render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/test/utils.py", line 100, in instrumented_test_render
    return self.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/loader_tags.py", line 150, in render
    return compiled_parent._render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/test/utils.py", line 100, in instrumented_test_render
    return self.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/loader_tags.py", line 62, in render
    result = block.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/defaulttags.py", line 315, in render
    return nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/defaulttags.py", line 214, in render
    nodelist.append(node.render_annotated(context))
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/loader_tags.py", line 195, in render
    return template.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 172, in render
    return self._render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/test/utils.py", line 100, in instrumented_test_render
    return self.nodelist.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 938, in render
    bit = node.render_annotated(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/base.py", line 905, in render_annotated
    return self.render(context)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/template/library.py", line 192, in render
    output = self.func(*resolved_args, **resolved_kwargs)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/leaflet/templatetags/leaflet_tags.py", line 124, in leaflet_json_config
    return json.dumps(settings_as_json, cls=DjangoJSONEncoder)
  File "/usr/lib/python3.8/json/__init__.py", line 234, in dumps
    return cls(
  File "/usr/lib/python3.8/json/encoder.py", line 199, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/lib/python3.8/json/encoder.py", line 257, in iterencode
    return _iterencode(o, 0)
  File "/home/nemesis/.virtualenvs/openwisp2/lib/python3.8/site-packages/django/core/serializers/json.py", line 105, in default
    return super().default(o)
  File "/usr/lib/python3.8/json/encoder.py", line 179, in default
    raise TypeError(f'Object of type {o.__class__.__name__} '

Exception Type: TypeError at /admin/
Exception Value: Object of type ListWithLazyItems is not JSON serializable

PS: I get the same issue if I try with Django 4.0.7.

Seems it was a bug with django-leaflet, upgrading it fixed it!

[totallynotvaishnav]

Here is the snippet to create 1000+ devices and locations. I called this from a JS file. Do remove the code once you created the devices and locations.

const $crf_token = $('[name="csrfmiddlewaretoken"]').attr("value");

function getRandomCoords(from, to, fixed) {
  return (Math.random() * (to - from) + from).toFixed(fixed) * 1;
}

for (let i = 0; i < 1005; i++) {
  const mac = "XX:XX:XX:XX:XX:XX".replace(/X/g, function () {
    return "0123456789ABCDEF".charAt(Math.floor(Math.random() * 16));
  });
  const device = {
    name: "device-" + i,
    organization: "<<Organization ID>>",
    mac_address: mac,
    config: {
      backend: "netjsonconfig.OpenWrt",
      templates: ["<<Template ID>>"],
      context: {},
      config: {},
    },
  };

  $.ajax({
    type: "POST",
    url: "http://localhost:8000/api/v1/controller/device/",
    headers: {"X-CSRFToken": $crf_token},
    data: JSON.stringify(device),
    contentType: "application/json",
    dataType: "json",
    success: function (data) {
      const location = {
        location: {
          name: "location-" + i,
          type: "outdoor",
          is_mobile: false,
          geometry: {
            type: "Point",
            coordinates: [
              getRandomCoords(-180, 180, 5),
              getRandomCoords(-180, 180, 5),
            ],
          },
        },
      };
      $.ajax({
        type: "PUT",
        url: `http://localhost:8000/api/v1/controller/device/${data.id}/location/`,
        headers: {"X-CSRFToken": $crf_token},
        data: JSON.stringify(location),
        contentType: "application/json",
        dataType: "json",
        error: function (e) {
          console.log("error");
          console.log(e);
        },
      });
    },
    error: function (e) {
      console.log("error");
      console.log(e);
    },
  });
}

[nemesifier]

Let's rebase, use the latest version of the library, fix conflicts, disable clustering and test again @pandafy.

[pandafy]

@nemesifier I tried different options available in the leaflet libarary. This is the best result I have achieved so far

https://github.com/openwisp/openwisp-monitoring/assets/32094433/65e298cc-ebc3-4a5c-be46-249c916f985c

Leaflet does not support looping (wrapping) the map like Google Maps.

The alternate option is to plot the markers on three world maps (center, left and right), and then setting bounds on the map to not allow panning more than 3 world maps. This is more of a workaround than a solution.