# URL-basierte Volltext-Anreicherung mit Goose3
# Anforderungen: pip install beautifulsoup4 streamlit requests goose3
# Start: steamlit run app.py

import streamlit as st
import json
import os
import datetime
import time
from goose3 import Goose

# List of file extensions to skip (e.g. audio and video formats)
SKIPPED_EXTENSIONS = ['.mp4', '.mp3', '.avi', '.mpeg', '.mov', '.wmv', '.flv']

# Function to scrape a webpage using Goose3 and return the extracted information
def scrape_page(url, follow_redirect=True):
    goose = Goose()  # Initialize Goose3 with default settings
    try:
        if isinstance(url, list):  # Check if URL is in list format and convert to string
            url = url[0]
        
        # Skip URLs with certain extensions
        if any(url.lower().endswith(ext) for ext in SKIPPED_EXTENSIONS):
            return "skipped", None

        article = goose.extract(url=url)
        
        # Extract the relevant information
        full_text = article.cleaned_text  # Cleaned full text
        title = article.title  # Extracted title
        summary = article.meta_description[:500] if article.meta_description else full_text[:500]  # Meta description as summary or first 500 chars
        keywords = article.meta_keywords  # Extracted meta keywords
        top_image = article.top_image.src if article.top_image else None  # URL of the top image

        # Fallback: If full text is missing, use meta description
        if not full_text:
            full_text = article.meta_description or "No full text available for this page."

        return {
            'title': title,
            'full_text': full_text,
            'summary': summary,
            'keywords': keywords,
            'top_image': top_image,
            'url': url
        }

    except Exception as e:
        st.error(f"Error scraping {url}: {e}")
        return None

# Function to process the JSON file and enrich it with scraped data
def process_json(data, url_field, timeout, save_folder, follow_redirect=True, skip_media_files=True, crawl_all=True, num_records=10):
    try:
        total_records = len(data)
        records_to_process = total_records if crawl_all else min(num_records, total_records)

        st.write(f"Processing {records_to_process} out of {total_records} records...")

        enriched_data = []
        for i, record in enumerate(data[:records_to_process], 1):
            # Display the progress message for processing records
            st.info(f"Processing record {i}/{records_to_process}", icon="ℹ️")

            # Extract URL using the selected field
            try:
                url = eval(f"record{url_field}")
                # If URL is in list format, convert it to a string (take the first URL)
                if isinstance(url, list):
                    url = url[0]

            except Exception as e:
                st.warning(f"No valid URL found for record {i}/{records_to_process}. Skipping... (Error: {e})", icon="⚠️")
                continue

            if url:
                # Skip media files if the option is selected
                if skip_media_files and any(url.lower().endswith(ext) for ext in SKIPPED_EXTENSIONS):
                    st.warning(f"Skipping media file URL #{i}: {url}", icon="⚠️")
                    continue

                # Display the message for scraping the current URL
                st.success(f"Scraping URL #{i}: {url}", icon="🟢")
                scraped_data = scrape_page(url, follow_redirect=follow_redirect)

                if scraped_data != "skipped" and scraped_data:
                    # Only display the summary preview message without additional status info
                    if scraped_data['summary']:
                        st.success(scraped_data['summary'][:250] + "..." if len(scraped_data['summary']) > 250 else scraped_data['summary'])
                    
                    # Add scraped data to the record
                    record['additional_data'] = {
                        'title': scraped_data['title'],
                        'full_text': scraped_data['full_text'],
                        'summary': scraped_data['summary'],
                        'keywords': scraped_data['keywords'],
                        'top_image': scraped_data['top_image'],
                        'final_url': scraped_data['url']
                    }

                enriched_data.append(record)

            # Introduce delay for the given timeout
            time.sleep(timeout)

        # Save the enriched JSON file
        timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
        enriched_file_name = os.path.join(save_folder, f"data_enriched_{timestamp}.json")

        with open(enriched_file_name, 'w', encoding='utf-8') as f:
            json.dump(enriched_data, f, ensure_ascii=False, indent=4)

        st.success(f"Enriched data saved as {enriched_file_name}", icon="🟢")

    except Exception as e:
        st.error(f"Error processing JSON file: {e}")

# Function to extract available fields from JSON structure
def extract_fields(data):
    field_set = set()

    # Recursive function to explore the JSON structure
    def recurse_json(obj, parent_key=''):
        if isinstance(obj, dict):
            for key, value in obj.items():
                new_key = f"{parent_key}['{key}']" if parent_key else f"['{key}']"
                field_set.add(new_key)
                recurse_json(value, new_key)
        elif isinstance(obj, list):
            for item in obj:
                recurse_json(item, parent_key)

    recurse_json(data)
    return sorted(list(field_set))

# Streamlit UI
st.title('JSON Web Scraper and Enricher using Goose3')

# Upload JSON file
uploaded_file = st.file_uploader("Choose a JSON file", type="json")

if uploaded_file:
    try:
        # Load the JSON data
        data = json.load(uploaded_file)

        # Extract all field paths from the JSON structure
        available_fields = extract_fields(data)

        # Allow the user to choose a URL field
        url_field = st.selectbox("Select the URL field", available_fields, index=available_fields.index("['properties']['ccm:wwwurl']"))

        # Other options for processing
        timeout = st.number_input("Enter delay between requests (seconds)", min_value=0, value=0)
        save_folder = st.text_input("Folder to save enriched JSON", value=".")

        follow_redirects = st.checkbox("Follow redirects", value=True)

        # Option to skip media files (audio/video)
        skip_media_files = st.checkbox("Skip media files (e.g., .mp4, .mp3, .avi)", value=True)

        # Option to process all records or only a limited number
        crawl_all = st.checkbox("Crawl all records", value=True)
        num_records = st.number_input("Number of records to process", min_value=1, value=10, disabled=crawl_all)

        if st.button("Start Processing"):
            process_json(data, url_field, timeout, save_folder, follow_redirects, skip_media_files, crawl_all, num_records)

    except Exception as e:
        st.error(f"Error loading JSON file: {e}")