LDA Implementation. Loss keeps diverging

[karttikeya]

I was trying to implement LDA in edward by referring to this issue #407 , but am facing issues in training. I have been trying this for a few days now, and have also tried using other forms of inference such as Gibbs, Expectation propagation, MAP, etc.

vocabulary = ["A", "B", "C", "D","E", "F", "G", "H"]
documents = ["AEAEAEAEAE",
             "BFBFBFBFBF",
             "CGCGCGCGCG",
             "DHDHDHDHDH",
             "EAEAEAEAEA",
             "FBFBFBFBFB",
             "GCGCGCGCGC",
             "HDHDHDHDHD",]
K = 4
D = len(documents)
V = len(vocabulary)
N = len(documents[0])
# transform the dataset into a numerical form the model can use
word_to_id = {vocabulary[idx]:idx for idx in range(len(vocabulary))}
id_vocabulary = [word_to_id[word] for word in vocabulary]
id_documents = [[word_to_id[word] for word in doc] for doc in documents]


theta = Dirichlet(tf.zeros([D, K]) +1/K, name="theta")
phi = Dirichlet(tf.zeros([K, V]) + 1/V, name="phi")
doc   = tf.placeholder(tf.int32, N)

z = Categorical(probs = tf.tile(tf.expand_dims(theta,1),[1,N,1]), name="z")

w = Categorical(probs =tf.gather(phi,z), name="w")
qphi = Dirichlet(tf.nn.softplus(tf.Variable(tf.random_normal([K,V]))), name="qphi")

inference = ed.KLqp({phi: qphi} , data={w: tf.constant(id_documents)})

inference.initialize(logdir = "log")
inference.run(n_iter = 1000, logdir = "log/")

My training keeps on diverging most of the time, and the result is not good even when it converges.

Error:

120/1000 [ 12%] ███                            ETA: 18s | Loss: 354.033

---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
   1360     try:
-> 1361       return fn(*args)
   1362     except errors.OpError as e:

E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\client\session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
   1339           return tf_session.TF_Run(session, options, feed_dict, fetch_list,
-> 1340                                    target_list, status, run_metadata)
   1341 

E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\framework\errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
    515             compat.as_text(c_api.TF_Message(self.status.status)),
--> 516             c_api.TF_GetCode(self.status.status))
    517     # Delete the underlying status object from memory otherwise it stays alive

InvalidArgumentError: Nan in summary histogram for: parameter/Variable/0_1
	 [[Node: parameter/Variable/0_1 = HistogramSummary[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](parameter/Variable/0_1/tag, Variable/read)]]

During handling of the above exception, another exception occurred:

InvalidArgumentError                      Traceback (most recent call last)
<ipython-input-4-14c3203a3b26> in <module>()
      1 inference.initialize(logdir = "log")
----> 2 inference.run(n_iter = 1000, logdir = "log/")
      3 # l = []
      4 
      5 # init = tf.global_variables_initializer()

E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\inference.py in run(self, variables, use_coordinator, *args, **kwargs)
    144 
    145     for _ in range(self.n_iter):
--> 146       info_dict = self.update()
    147       self.print_progress(info_dict)
    148 

E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\variational_inference.py in update(self, feed_dict)
    159     if self.logging and self.n_print != 0:
    160       if t == 1 or t % self.n_print == 0:
--> 161         summary = sess.run(self.summarize, feed_dict)
    162         self.train_writer.add_summary(summary, t)
    163 

E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
    903     try:
    904       result = self._run(None, fetches, feed_dict, options_ptr,
--> 905                          run_metadata_ptr)
    906       if run_metadata:
    907         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1135     if final_fetches or final_targets or (handle and feed_dict_tensor):
   1136       results = self._do_run(handle, final_targets, final_fetches,
-> 1137                              feed_dict_tensor, options, run_metadata)
   1138     else:
   1139       results = []

E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1353     if handle is None:
   1354       return self._do_call(_run_fn, self._session, feeds, fetches, targets,
-> 1355                            options, run_metadata)
   1356     else:
   1357       return self._do_call(_prun_fn, self._session, handle, feeds, fetches)

E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
   1372         except KeyError:
   1373           pass
-> 1374       raise type(e)(node_def, op, message)
   1375 
   1376   def _extend_graph(self):

InvalidArgumentError: Nan in summary histogram for: parameter/Variable/0_1
	 [[Node: parameter/Variable/0_1 = HistogramSummary[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](parameter/Variable/0_1/tag, Variable/read)]]

Caused by op 'parameter/Variable/0_1', defined at:
  File "E:\anaconda\envs\pmi\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "E:\anaconda\envs\pmi\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
    app.launch_new_instance()
  File "E:\anaconda\envs\pmi\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
    app.start()
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\kernelapp.py", line 486, in start
    self.io_loop.start()
  File "E:\anaconda\envs\pmi\lib\site-packages\tornado\platform\asyncio.py", line 112, in start
    self.asyncio_loop.run_forever()
  File "E:\anaconda\envs\pmi\lib\asyncio\base_events.py", line 421, in run_forever
    self._run_once()
  File "E:\anaconda\envs\pmi\lib\asyncio\base_events.py", line 1425, in _run_once
    handle._run()
  File "E:\anaconda\envs\pmi\lib\asyncio\events.py", line 127, in _run
    self._callback(*self._args)
  File "E:\anaconda\envs\pmi\lib\site-packages\tornado\ioloop.py", line 760, in _run_callback
    ret = callback()
  File "E:\anaconda\envs\pmi\lib\site-packages\tornado\stack_context.py", line 276, in null_wrapper
    return fn(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\zmq\eventloop\zmqstream.py", line 536, in <lambda>
    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))
  File "E:\anaconda\envs\pmi\lib\site-packages\zmq\eventloop\zmqstream.py", line 450, in _handle_events
    self._handle_recv()
  File "E:\anaconda\envs\pmi\lib\site-packages\zmq\eventloop\zmqstream.py", line 480, in _handle_recv
    self._run_callback(callback, msg)
  File "E:\anaconda\envs\pmi\lib\site-packages\zmq\eventloop\zmqstream.py", line 432, in _run_callback
    callback(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\tornado\stack_context.py", line 276, in null_wrapper
    return fn(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "E:\anaconda\envs\pmi\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\IPython\core\interactiveshell.py", line 2728, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "E:\anaconda\envs\pmi\lib\site-packages\IPython\core\interactiveshell.py", line 2856, in run_ast_nodes
    if self.run_code(code, result):
  File "E:\anaconda\envs\pmi\lib\site-packages\IPython\core\interactiveshell.py", line 2910, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-4-14c3203a3b26>", line 2, in <module>
    inference.run(n_iter = 1000, logdir = "log/")
  File "E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\inference.py", line 125, in run
    self.initialize(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\klqp.py", line 110, in initialize
    return super(KLqp, self).initialize(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\variational_inference.py", line 51, in initialize
    super(VariationalInference, self).initialize(*args, **kwargs)
  File "E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\inference.py", line 274, in initialize
    self._set_log_variables(log_vars)
  File "E:\anaconda\envs\pmi\lib\site-packages\edward\inferences\inference.py", line 376, in _set_log_variables
    var, collections=[self._summary_key])
  File "E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\summary\summary.py", line 193, in histogram
    tag=tag, values=values, name=scope)
  File "E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\ops\gen_logging_ops.py", line 215, in _histogram_summary
    "HistogramSummary", tag=tag, values=values, name=name)
  File "E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\framework\ops.py", line 3271, in create_op
    op_def=op_def)
  File "E:\anaconda\envs\pmi\lib\site-packages\tensorflow\python\framework\ops.py", line 1650, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): Nan in summary histogram for: parameter/Variable/0_1
	 [[Node: parameter/Variable/0_1 = HistogramSummary[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](parameter/Variable/0_1/tag, Variable/read)]]