Specified key too long - initial "opentaxii-sync-data config/data-configuration.yaml"

[ventz]

Getting during setup (during the opentaxii-sync-data initial step):

# opentaxii-sync-data config/data-configuration.yaml
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 470, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/lib/python3/dist-packages/pymysql/cursors.py", line 165, in execute
    result = self._query(query)
  File "/usr/lib/python3/dist-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 860, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1061, in _read_query_result
    result.read()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1349, in read
    first_packet = self.connection._read_packet()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1018, in _read_packet
    packet.check_error()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 384, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/lib/python3/dist-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
pymysql.err.InternalError: (1071, 'Specified key was too long; max key length is 767 bytes')

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/bin/opentaxii-sync-data", line 7, in <module>
    from opentaxii.cli.persistence import sync_data_configuration
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/cli/__init__.py", line 10, in <module>
    server = TAXIIServer(config)
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/server.py", line 36, in __init__
    server=self, api=initialize_api(config['persistence_api']))
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/utils.py", line 40, in initialize_api
    instance = cls(**params)
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/persistence/sqldb/api.py", line 46, in __init__
    self.db.create_all_tables()
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/sqldb_helper.py", line 60, in create_all_tables
    self.metadata.create_all(bind=self.engine)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/schema.py", line 3934, in create_all
    tables=tables)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1929, in _run_visitor
    conn._run_visitor(visitorcallable, element, **kwargs)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1538, in _run_visitor
    **kwargs).traverse_single(element)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
    return meth(obj, **kw)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 733, in visit_metadata
    _is_metadata_operation=True)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
    return meth(obj, **kw)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 772, in visit_table
    self.traverse_single(index)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
    return meth(obj, **kw)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 791, in visit_index
    self.connection.execute(CreateIndex(index))
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 945, in execute
    return meth(self, multiparams, params)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
    return connection._execute_ddl(self, multiparams, params)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
    compiled
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
    context)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
    exc_info
  File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 186, in reraise
    raise value.with_traceback(tb)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 470, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/lib/python3/dist-packages/pymysql/cursors.py", line 165, in execute
    result = self._query(query)
  File "/usr/lib/python3/dist-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 860, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1061, in _read_query_result
    result.read()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1349, in read
    first_packet = self.connection._read_packet()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1018, in _read_packet
    packet.check_error()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 384, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/lib/python3/dist-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1071, 'Specified key was too long; max key length is 767 bytes') [SQL: 'CREATE UNIQUE INDEX ix_data_collections_name ON data_collections (name)']

Re-trying produces different error, but similar type:

Traceback (most recent call last):
  File "/usr/local/bin/opentaxii-sync-data", line 7, in <module>
    from opentaxii.cli.persistence import sync_data_configuration
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/cli/__init__.py", line 10, in <module>
    server = TAXIIServer(config)
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/server.py", line 39, in __init__
    server=self, api=initialize_api(config['auth_api']))
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/utils.py", line 40, in initialize_api
    instance = cls(**params)
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/auth/sqldb/api.py", line 46, in __init__
    self.db.create_all_tables()
  File "/usr/local/lib/python3.6/dist-packages/opentaxii/sqldb_helper.py", line 60, in create_all_tables
    self.metadata.create_all(bind=self.engine)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/schema.py", line 3934, in create_all
    tables=tables)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1929, in _run_visitor
    conn._run_visitor(visitorcallable, element, **kwargs)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1538, in _run_visitor
    **kwargs).traverse_single(element)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
    return meth(obj, **kw)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 733, in visit_metadata
    _is_metadata_operation=True)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
    return meth(obj, **kw)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 767, in visit_table
    include_foreign_key_constraints=include_foreign_key_constraints
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 945, in execute
    return meth(self, multiparams, params)
  File "/usr/lib/python3/dist-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
    return connection._execute_ddl(self, multiparams, params)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
    compiled
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
    context)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
    exc_info
  File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 186, in reraise
    raise value.with_traceback(tb)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
    context)
  File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 470, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/lib/python3/dist-packages/pymysql/cursors.py", line 165, in execute
    result = self._query(query)
  File "/usr/lib/python3/dist-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 860, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1061, in _read_query_result
    result.read()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1349, in read
    first_packet = self.connection._read_packet()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 1018, in _read_packet
    packet.check_error()
  File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 384, in check_error
    err.raise_mysql_exception(self._data)
  File "/usr/lib/python3/dist-packages/pymysql/err.py", line 107, in raise_mysql_exception
    raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1071, 'Specified key was too long; max key length is 767 bytes') [SQL: '\nCREATE TABLE accounts (\n\tid INTEGER NOT NULL AUTO_INCREMENT, \n\tusername VARCHAR(256), \n\tpassword_hash VARCHAR(256), \n\tis_admin BOOL, \n\t_permissions TEXT NOT NULL, \n\tPRIMARY KEY (id), \n\tUNIQUE (username), \n\tCHECK (is_admin IN (0, 1))\n)\n\n']

The only changes have been as following install -- in config.yaml setting db_connection for each, and specifying the MISP URL and API keys. Also changing the taxii auth username + password.

[GlennHD]

Did you read the readme? =) Specified key was too long

[ventz]

Nod - saw that and tried it (also tried making the passcodes smaller from the start) but same problem. Unfortunately I can’t debug it since I don’t have the env anymore.