Problems when trying to use Llama-2 models for function calling

[nmm001001]

When I run the following command:

generator = Generator.hf(functions, "meta-llama/Llama-2-7b-chat-hf")

I get the following error:

---

AttributeError Traceback (most recent call last) Cell In[11], line 1 ----> 1 function_call = generator.generate("What is the weather like today in Delaware?") 2 print(function_call)

File /local_llm_function_calling/generator.py:189, in Generator.generate(self, prompt, function_call, max_length, max_new_tokens, suffix) 174 """Generate the function call 175 176 Args: (...) 186 FunctionCall: The generated function call 187 """ 188 function_name = self.choose_function(prompt, function_call, suffix) --> 189 arguments = self.generate_arguments( 190 prompt, function_name, max_new_tokens, max_length 191 ) 192 return {"name": function_name, "parameters": arguments}

File /local_llm_function_calling/generator.py:157, in Generator.generate_arguments(self, prompt, function_call, max_length, max_new_tokens) 147 prefix = self.prompter.prompt(prompt, self.functions, function_call) 148 constraint = JsonSchemaConstraint( 149 [ 150 function (...) 155 ] # type: ignore 156 ) --> 157 generated = self.constrainer.generate( 158 prefix, 159 constraint, 160 max_length, 161 max_new_tokens, 162 ) 163 validated = constraint.validate(generated) 164 return generated[: validated.end_index] if validated.end_index else generated

File /local_llm_function_calling/constrainer.py:221, in Constrainer.generate(self, prefix, constraint, max_len, max_new_tokens) 219 generation = self.model.start_generation(prefix) 220 for _ in range(max_new_tokens) if max_new_tokens else count(): --> 221 if self.advance_generation(generation, constraint, max_len): 222 break 223 return generation.get_generated()

File /local_llm_function_calling/constrainer.py:191, in Constrainer.advance_generation(self, generation, constraint, max_len) 173 def advance_generation( 174 self, 175 generation: Generation, 176 constraint: Callable[[str], tuple[bool, bool]], 177 max_len: int | None = None, 178 ) -> bool: 179 """Advance the generation by one token 180 181 Args: (...) 189 bool: Whether the generation is complete 190 """ --> 191 done, length = self.gen_next_token(generation, constraint) 192 if done: 193 return True

File /local_llm_function_calling/constrainer.py:163, in Constrainer.gen_next_token(self, generation, constraint) 161 except SequenceTooLongError: 162 return (True, 0) --> 163 for token in sorted_tokens: 164 generated = generation.get_generated(token) 165 fit = constraint(generated)

File /local_llm_function_calling/model/huggingface.py:63, in HuggingfaceGeneration.get_sorted_tokens(self) 54 def get_sorted_tokens(self) -> Iterator[int]: 55 """Get the tokens sorted by probability 56 57 Raises: (...) 61 The next of the most likely tokens 62 """ ---> 63 if self.inputs.shape[1] >= self.model.config.n_positions: 64 raise SequenceTooLongError() 65 gen_tokens = self.model.generate( 66 input_ids=self.inputs, 67 output_scores=True, (...) 70 pad_token_id=self.tokenizer.eos_token_id, 71 )

File /function_calling_env/lib/python3.11/site-packages/transformers/configuration_utils.py:262, in PretrainedConfig.getattribute(self, key) 260 if key != "attribute_map" and key in super().getattribute("attribute_map"): 261 key = super().getattribute("attribute_map")[key] --> 262 return super().getattribute(key)

AttributeError: 'LlamaConfig' object has no attribute 'n_positions'

Has anyone been able to successfully run this with Llama-2 models? If so did you run into this problem and how did you fix it?