Why there are no descriptions? about engineering-blogs HOT 1 OPEN

import requests
from bs4 import BeautifulSoup
from transformers import pipeline

# Initialize a summarization pipeline
summarizer = pipeline("summarization")

def crawl_and_summarize(url):
    # Crawl the webpage
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')

    # Extract main content, could be more specific based on site structure
    text = soup.get_text()

    # Summarize the text
    summary = summarizer(text, max_length=130, min_length=30, do_sample=False)
    return summary[0]['summary_text']

def read_urls_from_file(file_path):
    with open(file_path, 'r') as file:
        return [line.strip() for line in file if line.strip()]

# File containing URLs, one per line
file_path = 'urls.txt'

# Read URLs from the file
urls = read_urls_from_file(file_path)

# Crawl and summarize each URL
for url in urls:
    try:
        summary = crawl_and_summarize(url)
        print(f"URL: {url}\nSummary: {summary}\n")
    except Exception as e:
        print(f"Error processing {url}: {e}")