Rework mime type white list

[Tschuppi81]

Org: Ensure mime type validator on file upload fields in form code

TYPE: Feature LINK: ogc-2738

[linear[bot]]

OGC-2738 Arbitrary File Upload (Medium)

[codecov[bot]]

:x: 2 Tests Failed:

Tests completed	Failed	Passed	Skipped
2309	2	2307	17

View the top 2 failed test(s) by shortest run time

tests/onegov/org/test_views_event.py::test_import_export_events_with_custom_tags

Stack Traces | 3.55s run time

client = <tests.onegov.org.conftest.Client object at 0x7f1fc59f2e50>

    def test_import_export_events_with_custom_tags(client: Client) -> None:
        session = client.app.session()
        for event in session.query(Event):
            session.delete(event)
        transaction.commit()
    
        fs = client.app.filestorage
        assert fs is not None
        data = {
            'event_tags': ['Singing', 'Christmas']
        }
        with fs.open('eventsettings.yml', 'w') as f:
            yaml.dump(data, f)
    
        # Submit and publish an event
        page = client.get('/events').click("Veranstaltung erfassen")
        event_date = date.today() + timedelta(days=1)
        page.form['email'] = "[email protected]"
        page.form['title'] = "Weihnachtssingen"
        page.form['description'] = "Das Govikoner Sinfonieorchester lädt ein."
        page.form['location'] = "Konzertsaal"
        page.form['price'] = "CHF 75.-"
        page.form['organizer'] = "Sinfonieorchester"
        page.form['organizer_email'] = "[email protected]"
        page.form['organizer_phone'] = "+41 41 123 45 67"
        page.form['tags'] = ["Singing", "Christmas"]
        page.form['start_date'] = event_date.isoformat()
        page.form['start_time'] = "18:00"
        page.form['end_time'] = "22:00"
        page.form['repeat'] = 'without'
        page.form.submit().follow().form.submit().follow()
    
        client.login_editor()
    
        page = client.get('.../tickets/ALL/open').click("Annehmen").follow()
        page = page.click("Veranstaltung annehmen").follow()
    
        assert "Weihnachtssingen" in client.get('/events')
        assert "Singing" in client.get('/events')
        assert "Christmas" in client.get('/events')
    
        # Export
        page = client.get('/events').click("Export")
        page.form['file_format'] = 'xlsx'
        page = page.form.submit()
    
        file = Upload(
            'file',
            page.body,
            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
        )
    
        # Import (Dry run)
        page = client.get('/events').click("Import")
        page.form['dry_run'] = True
        page.form['file'] = file
        page = page.form.submit()
>       assert "1 Veranstaltungen werden importiert" in page
E       AssertionError: assert '1 Veranstaltungen werden importiert' in <200 OK text/html body=b'<!DOCTYP...\n\n'/40434>

.../onegov/org/test_views_event.py:864: AssertionError

tests/onegov/org/test_views_event.py::test_import_export_events

Stack Traces | 4.57s run time

client = <tests.onegov.org.conftest.Client object at 0x7f231dfca550>

    def test_import_export_events(client: Client) -> None:
        session = client.app.session()
        for event in session.query(Event):
            session.delete(event)
        transaction.commit()
    
        # Submit and publish an event
        page = client.get('/events').click("Veranstaltung erfassen")
        event_date = date.today() + timedelta(days=1)
        page.form['email'] = "[email protected]"
        page.form['title'] = "Weihnachtssingen"
        page.form['description'] = "Das Govikoner Sinfonieorchester lädt ein."
        page.form['location'] = "Konzertsaal"
        page.form['price'] = "CHF 75.-"
        page.form['organizer'] = "Sinfonieorchester"
        page.form['organizer_email'] = "[email protected]"
        page.form['organizer_phone'] = "+41 41 123 45 67"
        page.form['tags'] = ["Music", "Tradition"]
        page.form['start_date'] = event_date.isoformat()
        page.form['start_time'] = "18:00"
        page.form['end_time'] = "22:00"
        page.form['repeat'] = 'without'
        page.form.submit().follow().form.submit().follow()
    
        client.login_editor()
    
        page = client.get('.../tickets/ALL/open').click("Annehmen").follow()
        page = page.click("Veranstaltung annehmen").follow()
    
        assert "Weihnachtssingen" in client.get('/events')
        assert "Music" in client.get('/events')
        assert "Tradition" in client.get('/events')
    
        # Export
        page = client.get('/events').click("Export")
        page.form['file_format'] = 'xlsx'
        page = page.form.submit()
    
        file = Upload(
            'file',
            page.body,
            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
        )
    
        # Import (Dry run)
        page = client.get('/events').click("Import")
        page.form['dry_run'] = True
        page.form['file'] = file
        page = page.form.submit()
>       assert "1 Veranstaltungen werden importiert" in page
E       AssertionError: assert '1 Veranstaltungen werden importiert' in <200 OK text/html body=b'<!DOCTYP...\n\n'/40434>

.../onegov/org/test_views_event.py:772: AssertionError

To view more test analytics, go to the Test Analytics Dashboard _{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}

[Tschuppi81]

I saw that files types are handled differently for view_upload_file_by_json in handle_file_upload. Basically all file types are allowed. Shall we keep this?

[Tschuppi81]

Should I completely remove type application/octet-stream ? It is mostly used in conjunction with application/zip

[Daverball]

I saw that files types are handled differently for view_upload_file_by_json in handle_file_upload. Basically all file types are allowed. Shall we keep this?

We can make sure to set supported_content_types on GeneralFileCollection. That's the only one that would allow anything to be uploaded currently through those views.

[Daverball]

Should I completely remove type application/octet-stream ? It is mostly used in conjunction with application/zip

It's probably fine to remove it for now. There may however be the rare false positive for any files that cannot be identified correctly by libmagic. Generally pdfs, zips and any other binary file formats can end up as application/octet-stream, it's a generic catch-all content type for binary data if it couldn't be detected as anything else.