import streamlit as st
import os
import tempfile
import gc
import base64
import time
import yaml
from tqdm import tqdm
from brightdata_scrapper import *
from dotenv import load_dotenv
load_dotenv()
from crewai import Agent, Crew, Process, Task, LLM
from crewai_tools import FileReadTool
docs_tool = FileReadTool()
bright_data_api_key = os.getenv("BRIGHT_DATA_API_KEY")
@st.cache_resource
def load_llm():
llm = LLM(model="gpt-4o", api_key=os.getenv("OPENAI_API_KEY"))
# llm = LLM(
# model="ollama/llama3.2",
# base_url="http://localhost:11434"
# )
return llm
# ===========================
# Define Agents & Tasks
# ===========================
def create_agents_and_tasks():
"""Creates a Crew for analysis of the channel scrapped output"""
with open("config.yaml", 'r') as file:
config = yaml.safe_load(file)
analysis_agent = Agent(
role=config["agents"][0]["role"],
goal=config["agents"][0]["goal"],
backstory=config["agents"][0]["backstory"],
verbose=True,
tools=[docs_tool],
llm=load_llm()
)
response_synthesizer_agent = Agent(
role=config["agents"][1]["role"],
goal=config["agents"][1]["goal"],
backstory=config["agents"][1]["backstory"],
verbose=True,
llm=load_llm()
)
analysis_task = Task(
description=config["tasks"][0]["description"],
expected_output=config["tasks"][0]["expected_output"],
agent=analysis_agent
)
response_task = Task(
description=config["tasks"][1]["description"],
expected_output=config["tasks"][1]["expected_output"],
agent=response_synthesizer_agent
)
crew = Crew(
agents=[analysis_agent, response_synthesizer_agent],
tasks=[analysis_task, response_task],
process=Process.sequential,
verbose=True
)
return crew
# ===========================
# Streamlit Setup
# ===========================
st.markdown("""
# YouTube Trend Analysis powered by
&
""".format(base64.b64encode(open("assets/crewai.png", "rb").read()).decode(), base64.b64encode(open("assets/brightdata.png", "rb").read()).decode()), unsafe_allow_html=True)
if "messages" not in st.session_state:
st.session_state.messages = [] # Chat history
if "response" not in st.session_state:
st.session_state.response = None
if "crew" not in st.session_state:
st.session_state.crew = None # Store the Crew object
def reset_chat():
st.session_state.messages = []
gc.collect()
def start_analysis():
# Create a status container
with st.spinner('Scraping videos... This may take a moment.'):
status_container = st.empty()
status_container.info("Extracting videos from the channels...")
channel_snapshot_id = trigger_scraping_channels(bright_data_api_key, st.session_state.youtube_channels, 10, st.session_state.start_date, st.session_state.end_date, "Latest", "")
status = get_progress(bright_data_api_key, channel_snapshot_id['snapshot_id'])
while status['status'] != "ready":
status_container.info(f"Current status: {status['status']}")
time.sleep(10)
status = get_progress(bright_data_api_key, channel_snapshot_id['snapshot_id'])
if status['status'] == "failed":
status_container.error(f"Scraping failed: {status}")
return
if status['status'] == "ready":
status_container.success("Scraping completed successfully!")
# Show a list of YouTube vidoes here in a scrollable container
channel_scrapped_output = get_output(bright_data_api_key, status['snapshot_id'], format="json")
st.markdown("## YouTube Videos Extracted")
# Create a container for the carousel
carousel_container = st.container()
# Calculate number of videos per row (adjust as needed)
videos_per_row = 3
with carousel_container:
# Calculate number of rows needed
num_videos = len(channel_scrapped_output[0])
num_rows = (num_videos + videos_per_row - 1) // videos_per_row
for row in range(num_rows):
# Create columns for each row
cols = st.columns(videos_per_row)
# Fill each column with a video
for col_idx in range(videos_per_row):
video_idx = row * videos_per_row + col_idx
# Check if we still have videos to display
if video_idx < num_videos:
with cols[col_idx]:
st.video(channel_scrapped_output[0][video_idx]['url'])
status_container.info("Processing transcripts...")
st.session_state.all_files = []
# Calculate transcripts
for i in tqdm(range(len(channel_scrapped_output[0]))):
# save transcript to file
youtube_video_id = channel_scrapped_output[0][i]['shortcode']
file = "transcripts/" + youtube_video_id + ".txt"
st.session_state.all_files.append(file)
with open(file, "w") as f:
for j in range(len(channel_scrapped_output[0][i]['formatted_transcript'])):
text = channel_scrapped_output[0][i]['formatted_transcript'][j]['text']
start_time = channel_scrapped_output[0][i]['formatted_transcript'][j]['start_time']
end_time = channel_scrapped_output[0][i]['formatted_transcript'][j]['end_time']
f.write(f"({start_time:.2f}-{end_time:.2f}): {text}\n")
f.close()
st.session_state.channel_scrapped_output = channel_scrapped_output
status_container.success("Scraping complete! We shall now analyze the videos and report trends...")
else:
status_container.error(f"Scraping failed with status: {status}")
if status['status'] == "ready":
status_container = st.empty()
with st.spinner('The agent is analyzing the videos... This may take a moment.'):
# create crew
st.session_state.crew = create_agents_and_tasks()
st.session_state.response = st.session_state.crew.kickoff(inputs={"file_paths": ", ".join(st.session_state.all_files)})
# ===========================
# Sidebar
# ===========================
with st.sidebar:
st.header("YouTube Channels")
# Initialize the channels list in session state if it doesn't exist
if "youtube_channels" not in st.session_state:
st.session_state.youtube_channels = [""] # Start with one empty field
# Function to add new channel field
def add_channel_field():
st.session_state.youtube_channels.append("")
# Create input fields for each channel
for i, channel in enumerate(st.session_state.youtube_channels):
col1, col2 = st.columns([6, 1])
with col1:
st.session_state.youtube_channels[i] = st.text_input(
"Channel URL",
value=channel,
key=f"channel_{i}",
label_visibility="collapsed"
)
# Show remove button for all except the first field
with col2:
if i > 0:
if st.button("❌", key=f"remove_{i}"):
st.session_state.youtube_channels.pop(i)
st.rerun()
# Add channel button
st.button("Add Channel ➕", on_click=add_channel_field)
st.divider()
st.subheader("Date Range")
col1, col2 = st.columns(2)
with col1:
start_date = st.date_input("Start Date")
st.session_state.start_date = start_date
# store date as string
st.session_state.start_date = start_date.strftime("%Y-%m-%d")
with col2:
end_date = st.date_input("End Date")
st.session_state.end_date = end_date
st.session_state.end_date = end_date.strftime("%Y-%m-%d")
st.divider()
st.button("Start Analysis 🚀", type="primary", on_click=start_analysis)
# st.button("Clear Chat", on_click=reset_chat)
# ===========================
# Main Chat Interface
# ===========================
# Main content area
if st.session_state.response:
with st.spinner('Generating content... This may take a moment.'):
try:
result = st.session_state.response
st.markdown("### Generated Analysis")
st.markdown(result)
# Add download button
st.download_button(
label="Download Content",
data=result.raw,
file_name=f"youtube_trend_analysis.md",
mime="text/markdown"
)
except Exception as e:
st.error(f"An error occurred: {str(e)}")
# Footer
st.markdown("---")
st.markdown("Built with CrewAI, Bright Data and Streamlit")