Scrapegraph-ai
Scrapegraph-ai copied to clipboard
Published
20 hours ago •
ScrapeGraphAI
ScrapeGraphAI
TypeError: cannot pickle '_thread.RLock' object
Describe the bug When running the following code:
from langchain_openai import AzureChatOpenAI, AzureOpenAIEmbeddings
import os
from scrapegraphai.graphs import SmartScraperGraph, SearchGraph, SpeechGraph, SmartScraperMultiGraph
os.environ["AZURE_OPENAI_ENDPOINT"] = ""
os.environ["AZURE_OPENAI_API_KEY"] = ""
llm_model_instance = AzureChatOpenAI(
azure_deployment="",
openai_api_version="2024-02-15-preview",
temperature=0
)
embedder_model_instance = AzureOpenAIEmbeddings(
azure_deployment="",
openai_api_version="2024-02-15-preview",
)
graph_config = {
"llm": {
"model_instance": llm_model_instance
},
"embeddings": {
"model_instance": embedder_model_instance
}
}
smart_scraper_graph = SmartScraperMultiGraph(
prompt="List me all the projects with their descriptions",
# also accepts a string with the already downloaded HTML code
source=["https://perinim.github.io/projects"],
config=graph_config
)
result = smart_scraper_graph.run()
print(result)
I receive:
File "C:\Users\EESPOSG8D\Sviluppo\Python\ScrapeGraphAI\test.py", line 28, in <module>
smart_scraper_graph = SmartScraperMultiGraph(
File "C:\Users\EESPOSG8D\Sviluppo\Python\venv\lib\site-packages\scrapegraphai\graphs\smart_scraper_multi_graph.py", line 52, in __init__
self.copy_config = deepcopy(config)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 271, in _reconstruct
state = deepcopy(state, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 271, in _reconstruct
state = deepcopy(state, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 271, in _reconstruct
state = deepcopy(state, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 271, in _reconstruct
state = deepcopy(state, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 271, in _reconstruct
state = deepcopy(state, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 271, in _reconstruct
state = deepcopy(state, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 146, in deepcopy
y = copier(x, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 231, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "C:\Users\EESPOSG8D\AppData\Local\Programs\Python\Python310\lib\copy.py", line 161, in deepcopy
rv = reductor(4)
TypeError: cannot pickle '_thread.RLock' object
I am using Python 3.10.9
This TypeError also occurs in the ScriptCreatorMultiGraph. Did you find a solution? @VinciGit00 @PeriniM
We might need the same fixes in SearchGraph as well.
scrapegraphai/graphs/search_graph.py
I am getting the same error in SearchGraph . TypeError: cannot pickle '_thread.RLock' object
