langflow Custom component error variable not found

Bug Description

I am trying to create a custom vectorstore component. To get started with, I copied code of existing vectorstore from file elasticsearch.py to new file & change it as below (only removed some functions which I do not plan to use)

from typing import Any

from langchain.schema import Document
from langchain_elasticsearch import ElasticsearchStore
from loguru import logger
from elasticsearch import Elasticsearch

from langflow.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
from langflow.io import (
    DataInput,
    DropdownInput,
    FloatInput,
    HandleInput,
    IntInput,
    MultilineInput,
    SecretStrInput,
    StrInput,
)
from langflow.schema import Data


class ElasticsearchCustomVectorStoreComponent(LCVectorStoreComponent):
    """Elasticsearch custom vector store with with advanced, customizable search capabilities."""

    display_name: str = "Elasticsearch Custom"
    description: str = "Elasticsearch Custom Vector Store with with advanced, customizable search capabilities."
    documentation = "https://python.langchain.com/docs/integrations/vectorstores/elasticsearch"
    name = "ElasticsearchCustom"
    icon = "ElasticsearchStore"

    inputs = [
        StrInput(
            name="elasticsearch_url",
            display_name="Elasticsearch URL",
            value="http://localhost:9200",
            info="URL for self-managed Elasticsearch deployments (e.g., http://localhost:9200). "
            "Do not use with Elastic Cloud deployments, use Elastic Cloud ID instead.",
        ),
        SecretStrInput(
            name="cloud_id",
            display_name="Elastic Cloud ID",
            value="",
            info="Use this for Elastic Cloud deployments. Do not use together with 'Elasticsearch URL'.",
        ),
        StrInput(
            name="index_name",
            display_name="Index Name",
            value="langflow",
            info="The index name where the vectors will be stored in Elasticsearch cluster.",
        ),
        MultilineInput(
            name="search_input",
            display_name="Search Input",
            info="Enter a search query. Leave empty to retrieve all documents.",
        ),
        StrInput(
            name="username",
            display_name="Username",
            value="",
            advanced=False,
            info=(
                "Elasticsearch username (e.g., 'elastic'). "
                "Required for both local and Elastic Cloud setups unless API keys are used."
            ),
        ),
        SecretStrInput(
            name="password",
            display_name="Password",
            value="",
            advanced=False,
            info=(
                "Elasticsearch password for the specified user. "
                "Required for both local and Elastic Cloud setups unless API keys are used."
            ),
        ),
        HandleInput(
            name="embedding",
            display_name="Embedding",
            input_types=["Embeddings"],
        ),
        DropdownInput(
            name="search_type",
            display_name="Search Type",
            options=["similarity", "mmr"],
            value="similarity",
            advanced=True,
        ),
        IntInput(
            name="number_of_results",
            display_name="Number of Results",
            info="Number of results to return.",
            advanced=True,
            value=4,
        ),
        FloatInput(
            name="search_score_threshold",
            display_name="Search Score Threshold",
            info="Minimum similarity score threshold for search results.",
            value=0.0,
            advanced=True,
        ),
        SecretStrInput(
            name="api_key",
            display_name="Elastic API Key",
            value="",
            advanced=True,
            info="API Key for Elastic Cloud authentication. If used, 'username' and 'password' are not required.",
        ),
    ]

    @check_cached_vector_store
    def build_vector_store(self) -> ElasticsearchStore:
        print("inside build_vector_store")
        print("%"*55)
        """Builds the Elasticsearch Vector Store object."""
        if self.cloud_id and self.elasticsearch_url:
            msg = (
                "Both 'cloud_id' and 'elasticsearch_url' provided. "
                "Please use only one based on your deployment (Cloud or Local)."
            )
            raise ValueError(msg)

        es_params = {
            "index_name": self.index_name,
            "embedding": self.embedding,
            "es_user": self.username or None,
            "es_password": self.password or None,
        }

        if self.cloud_id:
            es_params["es_cloud_id"] = self.cloud_id
        else:
            es_params["es_url"] = self.elasticsearch_url

        if self.api_key:
            es_params["api_key"] = self.api_key
            
        elastic_client = Elasticsearch(hosts=[es_params["es_url"]], basic_auth=(es_params["es_user"], es_params["es_password"]), verify_certs=False)
        elasticsearch = ElasticsearchStore(index_name = es_params["index_name"], embedding=es_params["embedding"],es_connection = elastic_client) #(**es_params)

        return elasticsearch

    def search(self, query: str | None = None) -> list[dict[str, Any]]:
        """Search for similar documents in the vector store or retrieve all documents if no query is provided."""
        vector_store = self.build_vector_store()
        search_kwargs = {
            "k": self.number_of_results,
            "score_threshold": self.search_score_threshold,
        }

        if query:
            search_type = self.search_type.lower()
            if search_type not in {"similarity", "mmr"}:
                msg = f"Invalid search type: {self.search_type}"
                logger.error(msg)
                raise ValueError(msg)
            try:
                if search_type == "similarity":
                    print("%"*55)
                    results = vector_store.similarity_search_with_score(query, **search_kwargs)
                    print(results)
                    print("%"*55)
                elif search_type == "mmr":
                    results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
            except Exception as e:
                msg = (
                    "Error occurred while querying the Elasticsearch VectorStore,"
                    " there is no Data into the VectorStore."
                )
                logger.exception(msg)
                raise ValueError(msg) from e
            return [
                {"page_content": doc.page_content, "metadata": doc.metadata, "score": score} for doc, score in results
            ]
        results = self.get_all_documents(vector_store, **search_kwargs)
        return [{"page_content": doc.page_content, "metadata": doc.metadata, "score": score} for doc, score in results]

    def get_all_documents(self, vector_store: ElasticsearchStore, **kwargs) -> list[tuple[Document, float]]:
        """Retrieve all documents from the vector store."""
        client = vector_store.client
        index_name = self.index_name

        query = {
            "query": {"match_all": {}},
            "size": kwargs.get("k", self.number_of_results),
        }

        response = client.search(index=index_name, body=query)

        results = []
        for hit in response["hits"]["hits"]:
            doc = Document(
                page_content=hit["_source"].get("text", ""),
                metadata=hit["_source"].get("metadata", {}),
            )
            score = hit["_score"]
            results.append((doc, score))

        return results

    def search_documents(self) -> list[Data]:
        """Search for documents in the vector store based on the search input.

        If no search input is provided, retrieve all documents.
        """
        results = self.search(self.search_input)
        retrieved_data = [
            Data(
                text=result["page_content"],
                file_path=result["metadata"].get("file_path", ""),
            )
            for result in results
        ]
        self.status = retrieved_data
        return retrieved_data

    def get_retriever_kwargs(self):
        """Get the keyword arguments for the retriever."""
        return {
            "search_type": self.search_type.lower(),
            "search_kwargs": {
                "k": self.number_of_results,
                "score_threshold": self.search_score_threshold,
            },
        }

The component appears in UI and I am able to add it to the flow. However while building, below part of the file custom_component.py gives error as " variable not found"

  def get_variable(name: str, field: str):
            if hasattr(self, "_user_id") and not self.user_id:
                msg = f"User id is not set for {self.__class__.__name__}"
                raise ValueError(msg)
            variable_service = get_variable_service()  # Get service instance
            # Retrieve and decrypt the variable by name for the current user
            with session_scope() as session:
                user_id = self.user_id or ""
                return variable_service.get_variable(user_id=user_id, name=name, field=field, session=session)
        return get_variable

What am I missing? Also, confirmed that all required input properties are properly set via UI while using the component.

I tried printing the arguments being passed to variable_service.get_variable and found that "name" is empty.

Can someone please help?

Reproduction

Copy contents of langflow/components/vectorstores/elasticsearch.py to a new file
Add references of the file in init.py so that the new vectorstore is visible to use on frontend UI
Try using the component

Expected behavior

Component should work successfully

Who can help?

@italojohnny

Operating System

Windows 10

Langflow Version

latest

Python Version

3.12

Screenshot

No response

Flow File

document rag.json

Dec 03 '24 10:12 nirvitarka

Hello @nirvitarka! I'm here to help you with any bugs, questions, or contributions you might have.

To resolve the "variable not found" error in your custom Elasticsearch vector store component, you need to ensure that the "name" argument passed to the variable_service.get_variable function is not empty. This error occurs because the "name" is empty when the function is called.

Here's how you can address this issue:

Check for Empty Name: Before calling variable_service.get_variable, ensure that the "name" is not empty. You can add a check to raise an error if the "name" is empty, preventing the function from being called with an invalid argument.

Modify the get_variable Function: Update the get_variable function to include a check for an empty "name" argument. Here's an example of how you can modify the function:

def get_variable(name: str, field: str):
    if hasattr(self, "_user_id") and not self.user_id:
        msg = f"User id is not set for {self.__class__.__name__}"
        raise ValueError(msg)
    if not name:
        raise ValueError("The 'name' argument cannot be empty.")
    variable_service = get_variable_service()  # Get service instance
    # Retrieve and decrypt the variable by name for the current user
    with session_scope() as session:
        user_id = self.user_id or ""
        return variable_service.get_variable(user_id=user_id, name=name, field=field, session=session)
return get_variable

By adding this check, you ensure that the function will not proceed with an empty "name", which should help prevent the "variable not found" error ^[1].

To continue talking to Dosu, mention @dosu.

Dec 03 '24 10:12 dosubot[bot]

Just now tried the existing elasticsearch vectorstore and that too gives same error

Dec 03 '24 12:12 nirvitarka

I'm encountering the same error when attempting to build mem0 memory. I believe the problem occurs because both the component and the main code use the same name for the variable called "user_id". When the component is called with self.user_id, this ends up generating the conflict.

Dec 03 '24 14:12 moiraphill

I'm facing the same issue on Weaviate vector store component, if you try to edit anything in the Weaviate component even if you add a new line, you'll encouter this error when you try to run a search on this component.

Error building Component Weaviate: 

 variable not found.

Traceback (most recent call last):
  File "/home/osema/projects/PycharmProjects/lf/.venv/lib/python3.11/site-packages/langflow/graph/vertex/base.py", line 709, in _build_results
    result = await initialize.loading.get_instance_results(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/osema/projects/PycharmProjects/lf/.venv/lib/python3.11/site-packages/langflow/interface/initialize/loading.py", line 60, in get_instance_results
    custom_params = update_params_with_load_from_db_fields(
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/osema/projects/PycharmProjects/lf/.venv/lib/python3.11/site-packages/langflow/interface/initialize/loading.py", line 118, in update_params_with_load_from_db_fields
    key = custom_component.variables(params[field], field)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/osema/projects/PycharmProjects/lf/.venv/lib/python3.11/site-packages/langflow/custom/custom_component/custom_component.py", line 432, in get_variable
    return variable_service.get_variable(user_id=user_id, name=name, field=field, session=session)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/osema/projects/PycharmProjects/lf/.venv/lib/python3.11/site-packages/langflow/services/variable/service.py", line 69, in get_variable
    raise ValueError(msg)
ValueError:  variable not found.

Dec 17 '24 19:12 touatiosema

Hi Guys!

Workaround!!!

The solution I found was to save the component with any name, delete the edited one and add the new adjusted component. This way it works correctly with the modifications made!

Jan 22 '25 17:01 tornis

Hi, @nirvitarka. I'm Dosu, and I'm helping the langflow team manage their backlog. I'm marking this issue as stale.

Issue Summary:

You encountered a "variable not found" error while creating a custom vector store component.
The error was due to a missing variable, potentially related to the "user_id" variable.
I suggested ensuring the "name" argument in the get_variable function is not empty.
@tornis provided a workaround by renaming and re-adding the component to avoid conflicts.

Next Steps:

Please confirm if this issue is still relevant to the latest version of the langflow repository by commenting here.
If there is no response, the issue will be automatically closed in 7 days.

Thank you for your understanding and contribution!

Apr 23 '25 18:04 dosubot[bot]