impoved request handeling for ISBN
This commit is contained in:
+60
-27
@@ -26,7 +26,6 @@ from jinja2 import TemplateNotFound
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import requests
|
|
||||||
|
|
||||||
# Ensure imports work regardless of whether gunicorn starts in /app or /app/Web.
|
# Ensure imports work regardless of whether gunicorn starts in /app or /app/Web.
|
||||||
_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
@@ -9544,13 +9543,10 @@ def _fetch_from_open_library(clean_isbn):
|
|||||||
def _fetch_from_isbn_de(clean_isbn):
|
def _fetch_from_isbn_de(clean_isbn):
|
||||||
"""
|
"""
|
||||||
Source 4: isbn.de (Web Scraping für deutsche Schulbücher)
|
Source 4: isbn.de (Web Scraping für deutsche Schulbücher)
|
||||||
Sehr gute Trefferquote für Klett, Cornelsen, Westermann etc.
|
Robuste Version mit Fallbacks für Meta-Tags und Tabellendaten.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Die URL leitet meist automatisch auf den richtigen Buch-Slug weiter
|
|
||||||
url = f"https://www.isbn.de/buch/{clean_isbn}"
|
url = f"https://www.isbn.de/buch/{clean_isbn}"
|
||||||
|
|
||||||
# Ein User-Agent ist wichtig, da Webseiten simple Python-Scripte oft blockieren
|
|
||||||
headers = {
|
headers = {
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||||
}
|
}
|
||||||
@@ -9562,40 +9558,77 @@ def _fetch_from_isbn_de(clean_isbn):
|
|||||||
|
|
||||||
soup = BeautifulSoup(response.text, 'html.parser')
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
|
|
||||||
# Prüfen, ob wirklich ein Buch gefunden wurde
|
# 1. Titel prüfen
|
||||||
# (isbn.de wirft manchmal keinen 404, sondern zeigt eine Suchseite ohne Treffer)
|
|
||||||
title_elem = soup.find('h1')
|
title_elem = soup.find('h1')
|
||||||
if not title_elem or "nicht gefunden" in title_elem.text.lower() or "Suche" in title_elem.text:
|
if not title_elem or "nicht gefunden" in title_elem.text.lower() or "Suche" in title_elem.text:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
title = title_elem.text.strip()
|
title = title_elem.text.strip()
|
||||||
|
|
||||||
# isbn.de nutzt oft Schema.org Tags (itemprop), was das Auslesen sehr sicher macht
|
# --- Hilfsfunktion für unstrukturierte HTML-Tabellen ---
|
||||||
authors = "Unknown Author"
|
def get_detail_by_keyword(keywords):
|
||||||
author_elem = soup.find(itemprop="author")
|
"""Sucht nach Schlagwörtern (z.B. 'Verlag') und gibt den angrenzenden Wert zurück."""
|
||||||
if author_elem:
|
for tag in soup.find_all(['th', 'td', 'strong', 'b', 'span']):
|
||||||
authors = author_elem.text.strip()
|
text = tag.text.strip().lower()
|
||||||
|
if any(kw in text for kw in keywords):
|
||||||
|
# Wenn es eine Tabellenzelle (th/td) ist, nimm das nächste Geschwister-Element
|
||||||
|
if tag.name in ['th', 'td']:
|
||||||
|
sibling = tag.find_next_sibling('td')
|
||||||
|
if sibling:
|
||||||
|
return sibling.text.strip()
|
||||||
|
# Wenn es ein Label in einem Listen- oder Absatz-Element ist
|
||||||
|
parent = tag.parent
|
||||||
|
if parent and parent.name in ['li', 'p', 'div']:
|
||||||
|
return parent.text.replace(tag.text, '').strip()
|
||||||
|
return None
|
||||||
|
|
||||||
publisher = "Unknown Publisher"
|
# --- Hilfsfunktion für Schema.org (itemprop) ---
|
||||||
publisher_elem = soup.find(itemprop="publisher")
|
def get_itemprop(prop_name):
|
||||||
if publisher_elem:
|
elem = soup.find(attrs={"itemprop": prop_name})
|
||||||
publisher = publisher_elem.text.strip()
|
if elem:
|
||||||
|
return elem.get('content') or elem.text.strip()
|
||||||
|
return None
|
||||||
|
|
||||||
pub_date = "Unknown Date"
|
authors = get_itemprop("author")
|
||||||
date_elem = soup.find(itemprop="datePublished")
|
if not authors:
|
||||||
if date_elem:
|
authors = get_detail_by_keyword(['autor', 'herausgeber', 'von:'])
|
||||||
pub_date = date_elem.text.strip()
|
authors = authors if authors else "Unknown Author"
|
||||||
|
|
||||||
description = "Keine Beschreibung verfügbar"
|
publisher = get_itemprop("publisher")
|
||||||
desc_elem = soup.find(itemprop="description")
|
if not publisher:
|
||||||
if desc_elem:
|
publisher = get_detail_by_keyword(['verlag'])
|
||||||
description = desc_elem.text.strip()
|
publisher = publisher if publisher else "Unknown Publisher"
|
||||||
|
|
||||||
|
pub_date = get_itemprop("datePublished")
|
||||||
|
if not pub_date:
|
||||||
|
pub_date = get_detail_by_keyword(['erscheinungsjahr', 'erschienen', 'datum'])
|
||||||
|
pub_date = pub_date if pub_date else "Unknown Date"
|
||||||
|
|
||||||
|
page_count = get_itemprop("numberOfPages")
|
||||||
|
if not page_count:
|
||||||
|
page_count = get_detail_by_keyword(['seiten', 'umfang'])
|
||||||
|
|
||||||
|
if page_count:
|
||||||
|
match = re.search(r'\d+', page_count)
|
||||||
|
page_count = match.group(0) if match else "Unknown"
|
||||||
|
else:
|
||||||
|
page_count = "Unknown"
|
||||||
|
|
||||||
|
description = get_itemprop("description")
|
||||||
|
if not description:
|
||||||
|
for class_name in ['description', 'zusammenfassung', 'klappentext', 'buch-beschreibung']:
|
||||||
|
fallback_desc = soup.find('div', class_=re.compile(class_name, re.IGNORECASE))
|
||||||
|
if fallback_desc:
|
||||||
|
description = fallback_desc.text.strip()
|
||||||
|
break
|
||||||
|
description = description if description else "Keine Beschreibung verfügbar"
|
||||||
|
|
||||||
thumbnail = ""
|
thumbnail = ""
|
||||||
img_elem = soup.find('img', itemprop="image")
|
img_elem = soup.find('img', itemprop="image")
|
||||||
|
if not img_elem:
|
||||||
|
img_elem = soup.find('img', class_=lambda c: c and 'cover' in str(c).lower())
|
||||||
|
|
||||||
if img_elem and 'src' in img_elem.attrs:
|
if img_elem and 'src' in img_elem.attrs:
|
||||||
thumbnail = img_elem['src']
|
thumbnail = img_elem['src']
|
||||||
# Falls der Link relativ ist (z.B. /cover/...)
|
|
||||||
if thumbnail.startswith('/'):
|
if thumbnail.startswith('/'):
|
||||||
thumbnail = "https://www.isbn.de" + thumbnail
|
thumbnail = "https://www.isbn.de" + thumbnail
|
||||||
|
|
||||||
@@ -9605,7 +9638,7 @@ def _fetch_from_isbn_de(clean_isbn):
|
|||||||
"publisher": publisher,
|
"publisher": publisher,
|
||||||
"publishedDate": pub_date,
|
"publishedDate": pub_date,
|
||||||
"description": description,
|
"description": description,
|
||||||
"pageCount": "Unknown", # Seitenanzahl ist oft nicht standardisiert hinterlegt
|
"pageCount": page_count,
|
||||||
"price": None,
|
"price": None,
|
||||||
"thumbnail": thumbnail,
|
"thumbnail": thumbnail,
|
||||||
"source": "isbn-de"
|
"source": "isbn-de"
|
||||||
|
|||||||
Reference in New Issue
Block a user