kubernetes-deployment
kubernetes-deployment copied to clipboard
codeformuenster
meine-stadt-transparent kustomization on vsh-cluster
TODO
- [ ] Change importer
- [ ] Nominatim deployment
- [ ] Readiness probes
- [ ] Liveness probes
- [ ] Requests/limits
- [ ] Upgrade minio
- [ ] Upgrade mariadb
- [ ] Upgrade elasticsearch
- [ ] ???
Befehle, um docker-compose ans Laufen zu bekommen für die Gemeinde Ladbergen:
mkdir ladbergen-transparent/log
mkdir ladbergen-transparent/meine-stadt-transparent-static
docker volume create --opt type=none --opt o=bind django_static --opt device=/path/to/ladbergen-transparent/meine-stadt-transparent-static
docker-compose up
docker-compose run --rm django ./manage.py setup
# Fülle Dummydaten und lösche sie danach wieder, falls alles okay ist
docker-compose run --rm django ./manage.py loaddata mainapp/fixtures/initdata.json
docker-compose run --rm django ./manage.py flush
# Importiere echte Daten: https://github.com/meine-stadt-transparent/meine-stadt-transparent/blob/main/docs/Import.md
docker-compose run --rm django ./manage.py import_body https://ladbergen.ratsinfomanagement.net/webservice/oparl/v1.1/body/1
docker-compose run --rm django ./manage.py import_fetch
docker-compose run --rm django ./manage.py import_objects
docker-compose run --rm django ./manage.py import_files
beispielhafte .env
# All available options are documented in docs/Customization.md
DEBUG=False
SITE_NAME=Ladbergen Transparent
REAL_HOST=meine-stadt-transparent.mein-ladbergen.de
SECRET_KEY=280bce8c150d80ff9116ab82bbacf41eb06c1b345720f08cfc76bf7d50125fc2
LANGUAGE_CODE=de-de
TIME_ZONE=Europe/Berlin
DATABASE_URL=mysql://meinestadttransparent:4dee95a458744e86cc0564f11560a7d0d37ee39b065aa7991f2585134e63709d@mariadb/meinestadttransparent?charset=utf8mb4
CSP_FRAME=https://matomo.schuetze.link
ELASTICSEARCH_LANG=german
ELASTICSEARCH_URL=elasticsearch:9200
MINIO_SECRET_KEY=9f17e94b808838520d3d1eb94168b7effff93b76a6b6b7a8205742e401e5e457
MINIO_HOST=minio:9000
# Delete this entry if you don't use the docker container
STATIC_ROOT=/static
beispielhafte docker-compose.yml
version: '3'
services:
django:
image: konstin2/meine-stadt-transparent
container_name: django
volumes:
- django_static:/static
- ./.env:/app/.env
- ./log:/app/log
ports:
- 8000:8000
networks:
- mst_network
mariadb:
image: mariadb:10
container_name: mariadb
volumes:
- mariadb_data:/var/lib/mysql
environment:
- MYSQL_DATABASE=meinestadttransparent
- MYSQL_USER=meinestadttransparent
- MYSQL_PASSWORD=4dee95a458744e86cc0564f11560a7d0d37ee39b065aa7991f2585134e63709d
- MYSQL_RANDOM_ROOT_PASSWORD=true
networks:
- mst_network
command: ['--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
container_name: elasticsearch
environment:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- bootstrap.memory_lock=true
- cluster.routing.allocation.disk.threshold_enabled=false
- discovery.type=single-node
ulimits: # Elasticsearch needs those
memlock:
soft: -1
hard: -1
volumes:
- elasticsearch_data:/usr/share/elasticsearch/data
networks:
- mst_network
minio:
image: minio/minio
container_name: minio
volumes:
- minio_data:/minio_data
environment:
MINIO_ACCESS_KEY: meinestadttransparent
MINIO_SECRET_KEY: 9f17e94b808838520d3d1eb94168b7effff93b76a6b6b7a8205742e401e5e457
ports:
- 9000:9000
networks:
- mst_network
command: server /minio_data
volumes:
django_static:
external: true
mariadb_data:
elasticsearch_data:
minio_data:
networks:
mst_network:
2021-08-06 12:34:35,633 File 24315: Failed to run pdftotext: Command '['pdftotext', '/tmp/tmpezpf50tt', '-']' returned non-zero exit status 1.
Traceback (most recent call last):
File "/app/mainapp/functions/document_parsing.py", line 77, in extract_from_file
completed = subprocess.run(
File "/usr/local/lib/python3.8/subprocess.py", line 516, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['pdftotext', '/tmp/tmpezpf50tt', '-']' returned non-zero exit status 1.
concurrent.futures.process._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib/python3.8/concurrent/futures/process.py", line 239, in _process_worker
r = call_item.fn(*call_item.args, **call_item.kwargs)
File "/usr/local/lib/python3.8/concurrent/futures/process.py", line 198, in _process_chunk
return [fn(*args) for args in chunk]
File "/usr/local/lib/python3.8/concurrent/futures/process.py", line 198, in <listcomp>
return [fn(*args) for args in chunk]
File "/app/importer/importer.py", line 403, in download_and_analyze_file
file.parsed_text, file.page_count = extract_from_file(
File "/app/mainapp/functions/document_parsing.py", line 87, in extract_from_file
page_count = PdfFileReader(
File "/app/.venv/lib/python3.8/site-packages/PyPDF2/pdf.py", line 1084, in __init__
self.read(stream)
File "/app/.venv/lib/python3.8/site-packages/PyPDF2/pdf.py", line 1689, in read
stream.seek(-1, 2)
OSError: [Errno 22] Invalid argument
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/manage.py", line 27, in <module>
main()
File "/app/manage.py", line 23, in main
execute_from_command_line(sys.argv)
File "/app/.venv/lib/python3.8/site-packages/django/core/management/__init__.py", line 401, in execute_from_command_line
utility.execute()
File "/app/.venv/lib/python3.8/site-packages/django/core/management/__init__.py", line 395, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/app/.venv/lib/python3.8/site-packages/django/core/management/base.py", line 330, in run_from_argv
self.execute(*args, **cmd_options)
File "/app/.venv/lib/python3.8/site-packages/django/core/management/base.py", line 371, in execute
output = self.handle(*args, **options)
File "/app/importer/management/commands/import_files.py", line 38, in handle
importer.load_files(
File "/app/importer/importer.py", line 463, in load_files
for succeeded in executor.map(
File "/usr/local/lib/python3.8/concurrent/futures/process.py", line 484, in _chain_from_iterable_of_lists
for element in iterable:
File "/usr/local/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
yield fs.pop().result()
File "/usr/local/lib/python3.8/concurrent/futures/_base.py", line 437, in result
return self.__get_result()
File "/usr/local/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
OSError: [Errno 22] Invalid argument