py3-pinterest
py3-pinterest copied to clipboard
get_user_pins not returning all pins even with batching
When I called get_user_pins on users with large numbers of pins (>5k) it seems to get the first 10+ pages and then stops short. This is the function I'm using.
def get_pins(username, page_size=250, max_pages=None):
pages_processed = 0
user = pinterest.get_user_overview(username)
total_num_pins = user.get("pin_count")
total_pages = (
total_num_pins + page_size - 1
) // page_size # Ensuring all pins are counted even if the last page is not full
pbar = tqdm(
total=total_pages, desc="Pages Processed", position=1, leave=False
) # Set up the main progress bar for pages
# Fetch the initial batch of pins
pin_batch = pinterest.get_user_pins(username, page_size=page_size, reset_bookmark=True)
while pin_batch: # Continue until an empty batch is returned
pins = {} # Initialize or reset the pins dictionary for the new batch
for pin in pin_batch:
try:
pin_id = int(pin.get("id"))
pins[pin_id] = pin
except Exception as e:
continue # Skip any pins that cause errors
pages_processed += 1
pbar.update(1) # Update the progress bar for each batch processed
# Fetch the next batch of pins
pin_batch = pinterest.get_user_pins(username, page_size=page_size)
if (max_pages is not None) and (pages_processed > max_pages):
break
pbar.close() # Close the progress bar when no more pins are returned
return pin_batch