Compare commits

...

8 Commits

2 changed files with 306 additions and 228 deletions
+98 -42
View File
@@ -417,8 +417,6 @@ def _is_csrf_exempt_request():
@app.before_request
def _enforce_csrf_protection():
if request.endpoint == 'upload_csv_batch':
return None
if _is_csrf_exempt_request():
_get_csrf_token()
return None
@@ -11865,27 +11863,33 @@ def batch_upload_page():
return render_template('upload_batch.html')
from flask_wtf.csrf import CSRFProtect
csrf = CSRFProtect(app)
@app.route('/upload_csv_batch', methods=['POST'])
@csrf.exempt
def upload_csv_batch():
"""
Route for batch adding new items to the inventory via CSV.
Handles CSV parsing, bulk image upload (conversion to WebP), GridFS storage,
and groups identical items based on their Name.
Handles CSV parsing, bulk image upload with deduplication (SHA-256 hash matching),
GridFS storage, code generation, location syncing, and grouped item creation.
"""
import pandas as pd
import ast
#if 'username' not in session:
# return jsonify({'success': False, 'message': 'Nicht angemeldet'}), 401
import hashlib
username = session['username']
# permissions = _get_current_user_permissions() ... (anpassen wie in Original)
# if not _action_access_allowed(permissions, 'can_insert'):
# return jsonify({'success': False, 'message': 'Einfüge-Rechte erforderlich'}), 403
username = session.get('username', 'System')
def generate_unique_batch_code(base_code, position):
"""
Generiert einen eindeutigen Code für einen Artikel innerhalb einer Serie (Batch).
:param base_code: Der Code des ersten Artikels in der Gruppe (String oder None).
:param position: Die Position des aktuellen Artikels in der Gruppe (Integer).
:return: Ein eindeutiger Code als String.
"""
if base_code:
return f"{base_code}-{position}"
else:
random_prefix = str(uuid.uuid4())[:6].upper()
return f"BATCH-{random_prefix}-{position}"
fs = get_gridfs()
upload_session_id = str(uuid.uuid4())[:8]
@@ -11900,7 +11904,7 @@ def upload_csv_batch():
# 2. CSV Einlesen und Validieren
try:
df = pd.read_csv(csv_file)
df = pd.read_csv(csv_file, sep=',')
except Exception as e:
app.logger.error(f"[Upload {upload_session_id}] Fehler beim Lesen der CSV: {str(e)}")
return jsonify({"success": False, "message": f"Fehler beim Lesen der CSV: {str(e)}"}), 400
@@ -11908,10 +11912,11 @@ def upload_csv_batch():
if 'Name' not in df.columns:
return jsonify({"success": False, "message": "Die CSV muss zwingend eine 'Name' Spalte enthalten."}), 400
# 3. Bilder verarbeiten, nach WebP konvertieren und in GridFS speichern
# Mapping: Original-Dateiname (ohne Pfad/Erweiterung) -> GridFS Filename (.webp)
image_mapping = {}
# 3. Bilder verarbeiten & Duplikate im selben Durchlauf filtern (Hash-Matching)
image_mapping = {} # Original-Dateiname (ohne Ext) -> GridFS Filename (.webp)
processed_hashes = {} # SHA-256 Hash -> GridFS Filename (.webp)
processed_count = 0
dedup_count = 0
error_count = 0
for index, image in enumerate(uploaded_images):
@@ -11929,6 +11934,18 @@ def upload_csv_batch():
error_count += 1
continue
# SHA-256 Hash des Bildinhalts zur Erkennung identischer Bilder
img_hash = hashlib.sha256(image_bytes).hexdigest()
if img_hash in processed_hashes:
# Bild ist identisch zu einem bereits verarbeiteten Bild im selben Batch
existing_filename = processed_hashes[img_hash]
image_mapping[base_name_no_ext] = existing_filename
dedup_count += 1
app.logger.info(f"{image_log_prefix} Duplikat erkannt ({original_secure_name}). Wiederverwendung von: {existing_filename}")
continue
# Neues Bild verarbeiten und nach WebP konvertieren
optimized_io = io.BytesIO()
with Image.open(io.BytesIO(image_bytes)) as img:
if img.mode not in ('RGB', 'RGBA'):
@@ -11945,7 +11962,7 @@ def upload_csv_batch():
optimized_io.seek(0)
new_filename = f"{uuid.uuid4().hex}_{int(time.time())}.webp"
# Speichern in GridFS analog zu upload_item
# In GridFS speichern
file_id = fs.put(
optimized_io,
filename=new_filename,
@@ -11957,7 +11974,8 @@ def upload_csv_batch():
}
)
# Im Mapping speichern (damit wir sie später der CSV zuordnen können)
# In Hash-Tabelle und Mapping sichern
processed_hashes[img_hash] = new_filename
image_mapping[base_name_no_ext] = new_filename
processed_count += 1
@@ -11965,10 +11983,14 @@ def upload_csv_batch():
app.logger.error(f"{image_log_prefix} Processing failed: {str(e)}")
error_count += 1
# 4. Items gruppieren (Analog zu series_group_id aus upload_item)
df['Name'] = df['Name'].fillna('Unbenannt').astype(str)
# 4. Predefined Locations laden
try:
predefined_locations = it.get_predefined_locations()
except Exception:
predefined_locations = []
# Optional: Fülle NaN Werte in der CSV mit sinnvollen Defaults für die Datenbank
# 5. Dataframe bereinigen & gruppieren
df['Name'] = df['Name'].fillna('Unbenannt').astype(str)
df = df.fillna({
'Ort': 'Unbekannt',
'Beschreibung': '',
@@ -11985,9 +12007,22 @@ def upload_csv_batch():
series_group_id = str(uuid.uuid4()) if item_count > 1 else None
parent_item_id = None
# Basis-Code für automatisierte Seriencodes ermitteln
first_row_code = str(group.iloc[0].get('Code_4', '')).strip()
base_code = first_row_code if first_row_code else None
for position, (index, row) in enumerate(group.iterrows(), start=1):
# Bilder aus der CSV-Zeile extrahieren und über das image_mapping mappen
# Ort automatisch zu predefined_locations hinzufügen, falls neu
ort_val = str(row['Ort']).strip()
if ort_val and ort_val not in predefined_locations:
try:
it.add_predefined_location(ort_val)
predefined_locations.append(ort_val)
except Exception as e:
app.logger.warning(f"Ort {ort_val} konnte nicht hinzugefügt werden: {e}")
# Bilder für diesen Artikel zuordnen
item_image_filenames = []
if 'Images' in row and pd.notna(row['Images']):
try:
@@ -12002,11 +12037,20 @@ def upload_csv_batch():
except (ValueError, SyntaxError):
pass
# --- NEU: BILDER-REFERENZEN PRO ARTIKEL DEDUPLIZIEREN ---
# Falls die CSV z.B. ['bild1.jpg', 'bild1.jpg'] enthält, filtern wir das hier heraus,
# damit die GridFS-Datei nicht doppelt als Referenz gespeichert wird.
unique_image_filenames = []
for img in item_image_filenames:
if img not in unique_image_filenames:
unique_image_filenames.append(img)
# --------------------------------------------------------
def parse_filter_col(col_data):
try:
res = ast.literal_eval(str(col_data))
return res if isinstance(res, list) else []
except:
except Exception:
return []
filter_upload = parse_filter_col(row.get('Filter', '[]'))
@@ -12015,28 +12059,37 @@ def upload_csv_batch():
reservierbar = bool(row.get('Reservierbar', False))
# DB Insert Funktion aufrufen (mit korrigierten, positionellen Parametern)
# Code_4 Behandlung: Falls in CSV definiert nutzen, sonst Batch-Code erzeugen
row_code = str(row.get('Code_4', '')).strip()
if row_code:
unique_code = row_code
elif 'generate_unique_batch_code' in globals():
unique_code = generate_unique_batch_code(base_code, position)
else:
unique_code = None
# DB Insert (exakt abgestimmt auf die 10 positionellen Argumente)
item_id = it.add_item(
str(row['Name']),
str(row['Ort']),
str(row['Beschreibung']),
item_image_filenames,
filter_upload,
filter_upload2,
filter_upload3,
str(row['Anschaffungsjahr']) if row['Anschaffungsjahr'] else None,
str(row['Anschaffungskosten']) if row['Anschaffungskosten'] else None,
str(row['Code_4']) if row['Code_4'] else None,
str(row['Name']), # 1. Name
ort_val, # 2. Ort
str(row['Beschreibung']), # 3. Beschreibung
unique_image_filenames, # 4. Image Filenames (GridFS) -> HIER GEÄNDERT
filter_upload, # 5. Filter 1
filter_upload2, # 6. Filter 2
filter_upload3, # 7. Filter 3
str(row['Anschaffungsjahr']) if row['Anschaffungsjahr'] else None, # 8. Jahr
str(row['Anschaffungskosten']) if row['Anschaffungskosten'] else None, # 9. Kosten
unique_code, # 10. Unique Code / Code_4
reservierbar=reservierbar,
series_group_id=series_group_id,
series_count=item_count,
series_position=position,
is_grouped_sub_item=(position > 1),
parent_item_id=parent_item_id,
isbn='',
item_type='other',
library_category='',
is_library=False
isbn=str(row.get('ISBN', '')),
item_type=str(row.get('Item_Type', 'other')),
library_category=str(row.get('Library_Category', '')),
is_library=bool(row.get('Is_Library', False))
)
if item_id:
@@ -12047,12 +12100,15 @@ def upload_csv_batch():
app.logger.error(f"Fehler beim Erstellen von Item: {row['Name']} (Index {index})")
app.logger.info(
f"Batch Upload abgeschlossen: {len(created_item_ids)} Items erstellt. {processed_count} Bilder verarbeitet.")
f"Batch Upload abgeschlossen: {len(created_item_ids)} Items erstellt. "
f"{processed_count} neue Bilder hochgeladen, {dedup_count} Bild-Duplikate zusammengeführt."
)
return jsonify({
"success": True,
"message": f"Upload erfolgreich. {len(created_item_ids)} Items importiert und {processed_count} Bilder konvertiert.",
"message": f"Upload erfolgreich. {len(created_item_ids)} Items importiert. {processed_count} neue Bilder gespeichert ({dedup_count} Duplikate zusammengeführt).",
"created_count": len(created_item_ids),
"images_processed": processed_count,
"images_deduplicated": dedup_count,
"images_failed": error_count
}), 200
+208 -186
View File
@@ -3,215 +3,237 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Batch Upload - CSV & Bilder</title>
<title>Batch Upload</title>
<!-- CSRF-Token für JavaScript bereitstellen -->
<meta name="csrf-token" content="{{ session.get('_csrf_token', '') }}">
<style>
:root {
--primary-color: #4a90e2;
--background-color: #f4f7f6;
--text-color: #333;
--border-radius: 8px;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background-color: var(--background-color);
color: var(--text-color);
display: flex;
justify-content: center;
align-items: center;
min-height: 100vh;
margin: 0;
}
.upload-container {
background: white;
padding: 2rem;
border-radius: var(--border-radius);
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
width: 100%;
max-width: 500px;
}
h2 {
margin-top: 0;
color: var(--primary-color);
text-align: center;
}
.form-group {
margin-bottom: 1.5rem;
}
label {
display: block;
font-weight: 600;
margin-bottom: 0.5rem;
}
input[type="file"] {
display: block;
width: 100%;
padding: 0.5rem;
border: 1px dashed #ccc;
border-radius: var(--border-radius);
background: #fafafa;
cursor: pointer;
}
.btn-submit {
width: 100%;
padding: 0.75rem;
background-color: var(--primary-color);
color: white;
border: none;
border-radius: var(--border-radius);
font-size: 1rem;
font-weight: 600;
cursor: pointer;
transition: background-color 0.3s ease;
}
.btn-submit:hover {
background-color: #357abd;
}
.btn-submit:disabled {
background-color: #a0c4e8;
cursor: not-allowed;
}
/* Status & Feedback Messages */
#status-message {
margin-top: 1rem;
padding: 1rem;
border-radius: var(--border-radius);
display: none;
text-align: center;
}
.success {
background-color: #d4edda;
color: #155724;
border: 1px solid #c3e6cb;
}
.error {
background-color: #f8d7da;
color: #721c24;
border: 1px solid #f5c6cb;
}
.loading {
background-color: #e2e3e5;
color: #383d41;
border: 1px solid #d6d8db;
}
.spinner {
display: inline-block;
width: 1.5rem;
height: 1.5rem;
border: 3px solid rgba(0,0,0,0.1);
border-radius: 50%;
border-top-color: var(--primary-color);
animation: spin 1s ease-in-out infinite;
vertical-align: middle;
margin-right: 0.5rem;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
body { font-family: sans-serif; padding: 20px; background: #f4f7f6; }
.upload-container { background: white; padding: 20px; border-radius: 8px; max-width: 500px; margin: 0 auto; }
.form-group { margin-bottom: 15px; }
label { display: block; font-weight: bold; margin-bottom: 5px; }
input[type="file"] { width: 100%; padding: 8px; box-sizing: border-box; }
.btn-submit { width: 100%; padding: 10px; background: #4a90e2; color: white; border: none; border-radius: 4px; font-weight: bold; cursor: pointer; }
.btn-submit:disabled { background: #ccc; }
#uploadProgress { margin-top: 20px; display: none; }
progress { width: 100%; height: 20px; }
#logList { background: #fafafa; border: 1px solid #eee; padding: 10px; max-height: 150px; overflow-y: auto; font-size: 0.85em; list-style: none; }
</style>
</head>
<body>
<div class="upload-container">
<h2>Inventar Batch Upload</h2>
<div class="upload-container">
<h2>Inventar Batch Upload</h2>
<form id="uploadForm">
<div class="form-group">
<label for="csv_file">1. items.csv Datei auswählen</label>
<!-- Akzeptiert nur CSV Dateien -->
<input type="file" id="csv_file" name="csv_file" accept=".csv" required>
</div>
<form id="batchUploadForm">
<div class="form-group">
<label for="csv_file">1. items.csv auswählen</label>
<input type="file" id="csv_file" name="csv_file" accept=".csv" required>
</div>
<div class="form-group">
<label for="images">2. Bilder auswählen</label>
<!-- multiple erlaubt das Auswählen mehrerer Bilder gleichzeitig -->
<input type="file" id="images" name="images" accept="image/*" multiple required>
<small style="color: #666; display: block; margin-top: 5px;">Du kannst mehrere Bilder markieren (Strg/Cmd gedrückt halten).</small>
</div>
<div class="form-group">
<label for="images">2. Bilder auswählen</label>
<input type="file" id="images" name="images" accept="image/*" multiple required>
</div>
<button type="submit" id="submitBtn" class="btn-submit">Daten hochladen</button>
</form>
<button type="submit" id="uploadBtn" class="btn-submit">Daten hochladen</button>
</form>
<div id="status-message"></div>
<div id="uploadProgress">
<div id="progressText">Bereite Upload vor...</div>
<progress id="progressBar" value="0" max="100"></progress>
<ul id="logList"></ul>
</div>
</div>
<script>
document.getElementById('uploadForm').addEventListener('submit', async function(e) {
e.preventDefault();
// Robuster CSV-Parser, der Kommas innerhalb von Anführungszeichen ignoriert
function parseCSVLine(text) {
const result = [];
let cur = '';
let inQuotes = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (c === '"' || c === "'") {
inQuotes = !inQuotes;
} else if (c === ',' && !inQuotes) {
result.push(cur.trim());
cur = '';
} else {
cur += c;
}
}
result.push(cur.trim());
return result;
}
const form = e.target;
const submitBtn = document.getElementById('submitBtn');
const statusDiv = document.getElementById('status-message');
// Extrahiert Bildnamen aus Einträgen wie "['bild1.jpg', 'bild2.jpg']" oder "bild1.jpg"
function extractImageNames(cellValue) {
if (!cellValue) return [];
let raw = cellValue.replace(/^["']|["']$/g, '').trim();
if (raw.startsWith('[') && raw.endsWith(']')) {
try {
const jsonValid = raw.replace(/'/g, '"');
return JSON.parse(jsonValid);
} catch (e) {
const matches = raw.match(/['"]([^'"]+)['"]/g);
if (matches) return matches.map(m => m.replace(/['"]/g, ''));
}
}
return raw ? [raw] : [];
}
submitBtn.disabled = true;
submitBtn.innerText = 'Wird verarbeitet...';
statusDiv.className = 'loading';
statusDiv.style.display = 'block';
statusDiv.innerHTML = '<div class="spinner"></div> Lade Dateien hoch und verarbeite Bilder... Bitte warten.';
document.getElementById('batchUploadForm').addEventListener('submit', async function(e) {
e.preventDefault();
const formData = new FormData(form);
const csvInput = document.getElementById('csv_file');
const imageInput = document.getElementById('images');
const uploadBtn = document.getElementById('uploadBtn');
const progressContainer = document.getElementById('uploadProgress');
const progressBar = document.getElementById('progressBar');
const progressText = document.getElementById('progressText');
const logList = document.getElementById('logList');
const fetchOptions = {
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');
if (!csvInput.files.length) {
alert("Bitte wähle eine CSV-Datei aus.");
return;
}
uploadBtn.disabled = true;
progressContainer.style.display = 'block';
logList.innerHTML = '';
const log = (msg) => {
const li = document.createElement('li');
li.textContent = msg;
logList.appendChild(li);
logList.scrollTop = logList.scrollHeight;
};
const csvFile = csvInput.files[0];
const allImages = Array.from(imageInput.files);
// Map für schnellen Dateinamen-Vergleich (ohne Pfadangabe)
const imageMap = new Map();
allImages.forEach(file => {
imageMap.set(file.name.toLowerCase(), file);
});
const BATCH_SIZE = 50;
try {
const csvText = await csvFile.text();
let rows = csvText.split(/\r?\n/).filter(r => r.trim().length > 0);
if (rows.length <= 1) {
throw new Error("CSV-Datei ist leer oder enthält nur Kopfzeilen.");
}
const headerRow = rows[0];
const dataRows = rows.slice(1);
// 1. Client-Deduplizierung
const uniqueRowsSet = new Set();
const uniqueDataRows = [];
let duplicateCount = 0;
for (const row of dataRows) {
if (uniqueRowsSet.has(row)) {
duplicateCount++;
} else {
uniqueRowsSet.add(row);
uniqueDataRows.push(row);
}
}
log(`${uniqueDataRows.length} eindeutige Einträge. ${duplicateCount} Duplikate entfernt.`);
// Spalten-Index von "Images" ermitteln
const headers = parseCSVLine(headerRow).map(h => h.replace(/['"]/g, '').trim());
const imagesColIndex = headers.findIndex(h => h.toLowerCase() === 'images');
// 2. In Batches aufteilen
const batches = [];
for (let i = 0; i < uniqueDataRows.length; i += BATCH_SIZE) {
batches.push(uniqueDataRows.slice(i, i + BATCH_SIZE));
}
progressBar.max = batches.length;
progressBar.value = 0;
// 3. Sequenzieller Upload
for (let b = 0; b < batches.length; b++) {
const batchRows = batches[b];
progressText.textContent = `Lade Batch ${b + 1} von ${batches.length} hoch...`;
// Zuordnung der benötigten Bilder für diesen Batch
const requiredImagesForBatch = new Set();
if (imagesColIndex !== -1) {
batchRows.forEach(rowStr => {
const cols = parseCSVLine(rowStr);
if (cols[imagesColIndex]) {
const imgNames = extractImageNames(cols[imagesColIndex]);
imgNames.forEach(name => {
const fileMatch = imageMap.get(name.toLowerCase());
if (fileMatch) {
requiredImagesForBatch.add(fileMatch);
}
});
}
});
}
// CSV für diesen Batch erstellen
const batchCsvText = [headerRow, ...batchRows].join('\n');
const batchCsvBlob = new Blob([batchCsvText], { type: 'text/csv' });
const formData = new FormData();
formData.append('csv_file', batchCsvBlob, `batch_${b + 1}.csv`);
// CSRF-Token im Form-Field übergeben
if (csrfToken) {
formData.append('csrf_token', csrfToken);
}
// Zugehörige Bilder anfügen
requiredImagesForBatch.forEach(imgFile => {
formData.append('images', imgFile);
});
log(`Batch ${b + 1}: ${batchRows.length} Items & ${requiredImagesForBatch.size} zugehörige Bilder.`);
// Request mit CSRF-Header ausführen
const response = await fetch('/upload_csv_batch', {
method: 'POST',
body: formData,
credentials: 'include',
headers: {
'X-Requested-With': 'XMLHttpRequest'
'X-CSRFToken': csrfToken || ''
}
};
});
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');
const result = await response.json();
if (csrfToken) {
fetchOptions.headers = {
'X-CSRFToken': csrfToken
};
if (!response.ok || !result.success) {
throw new Error(result.message || `Server-Fehler ${response.status}`);
}
try {
const response = await fetch('/upload_csv_batch', fetchOptions);
log(`Batch ${b + 1} abgeschlossen: ${result.message}`);
progressBar.value = b + 1;
}
// Antwort einmalig als Text auslesen, um sowohl JSON als auch HTML-Fehler abzufangen
const responseText = await response.text();
progressText.textContent = "Upload erfolgreich beendet!";
uploadBtn.disabled = false;
let result;
try {
result = JSON.parse(responseText);
} catch (jsonError) {
console.error("Server hat kein JSON gesendet. Antwort war:", responseText);
throw new Error("Der Server hat einen HTML-Fehler zurückgegeben (z.B. Nginx 413 Entity Too Large oder Server-Crash). Siehe F12 Konsole.");
}
if (response.ok && result.success) {
statusDiv.className = 'success';
statusDiv.innerHTML = `<strong>Erfolg!</strong><br>${result.message}`;
form.reset();
} else {
statusDiv.className = 'error';
statusDiv.innerHTML = `<strong>Fehler:</strong> ${result.message || 'Ein unbekannter Fehler ist aufgetreten.'}`;
}
} catch (error) {
statusDiv.className = 'error';
statusDiv.innerHTML = `<strong>Fehler:</strong> ${error.message}`;
console.error('Upload Error:', error);
} finally {
submitBtn.disabled = false;
submitBtn.innerText = 'Daten hochladen';
}
});
</script>
} catch (err) {
alert("Upload abgebrochen: " + err.message);
log("Fehler: " + err.message);
uploadBtn.disabled = false;
}
});
</script>
</body>
</html>