Compare commits

...

9 Commits

2 changed files with 278 additions and 225 deletions
+57 -40
View File
@@ -417,8 +417,6 @@ def _is_csrf_exempt_request():
@app.before_request
def _enforce_csrf_protection():
if request.endpoint == 'upload_csv_batch':
return None
if _is_csrf_exempt_request():
_get_csrf_token()
return None
@@ -11865,12 +11863,25 @@ def batch_upload_page():
return render_template('upload_batch.html')
from flask_wtf.csrf import CSRFProtect
csrf = CSRFProtect(app)
def clean_db_field(val):
"""Bereinigt Werte, die fälschlicherweise als String-Listen aus der CSV kommen."""
import pandas as pd
if not val or pd.isna(val):
return None
val_str = str(val).strip()
# Erkennt und entpackt String-Listen wie "['100465']" oder '["100465"]'
if (val_str.startswith("['") and val_str.endswith("']")) or (val_str.startswith('["') and val_str.endswith('"]')):
inner = val_str[2:-2].strip()
return inner if inner and inner != "''" and inner != '""' else None
if val_str in ("[]", "['']", '[""]', "nan", "None"):
return None
return val_str
@app.route('/upload_csv_batch', methods=['POST'])
@csrf.exempt
def upload_csv_batch():
"""
Route for batch adding new items to the inventory via CSV.
@@ -11883,6 +11894,13 @@ def upload_csv_batch():
username = session.get('username', 'System')
def generate_unique_batch_code(base_code, position):
if base_code:
return f"{base_code}-{position}"
else:
random_prefix = str(uuid.uuid4())[:6].upper()
return f"BATCH-{random_prefix}-{position}"
fs = get_gridfs()
upload_session_id = str(uuid.uuid4())[:8]
app.logger.info(f"Starting CSV Batch upload session {upload_session_id} - User: {username}")
@@ -11896,7 +11914,7 @@ def upload_csv_batch():
# 2. CSV Einlesen und Validieren
try:
df = pd.read_csv(csv_file)
df = pd.read_csv(csv_file, sep=',')
except Exception as e:
app.logger.error(f"[Upload {upload_session_id}] Fehler beim Lesen der CSV: {str(e)}")
return jsonify({"success": False, "message": f"Fehler beim Lesen der CSV: {str(e)}"}), 400
@@ -11905,8 +11923,8 @@ def upload_csv_batch():
return jsonify({"success": False, "message": "Die CSV muss zwingend eine 'Name' Spalte enthalten."}), 400
# 3. Bilder verarbeiten & Duplikate im selben Durchlauf filtern (Hash-Matching)
image_mapping = {} # Original-Dateiname (ohne Ext) -> GridFS Filename (.webp)
processed_hashes = {} # SHA-256 Hash -> GridFS Filename (.webp)
image_mapping = {}
processed_hashes = {}
processed_count = 0
dedup_count = 0
error_count = 0
@@ -11926,18 +11944,14 @@ def upload_csv_batch():
error_count += 1
continue
# SHA-256 Hash des Bildinhalts zur Erkennung identischer Bilder
img_hash = hashlib.sha256(image_bytes).hexdigest()
if img_hash in processed_hashes:
# Bild ist identisch zu einem bereits verarbeiteten Bild im selben Batch
existing_filename = processed_hashes[img_hash]
image_mapping[base_name_no_ext] = existing_filename
dedup_count += 1
app.logger.info(f"{image_log_prefix} Duplikat erkannt ({original_secure_name}). Wiederverwendung von: {existing_filename}")
continue
# Neues Bild verarbeiten und nach WebP konvertieren
optimized_io = io.BytesIO()
with Image.open(io.BytesIO(image_bytes)) as img:
if img.mode not in ('RGB', 'RGBA'):
@@ -11954,8 +11968,7 @@ def upload_csv_batch():
optimized_io.seek(0)
new_filename = f"{uuid.uuid4().hex}_{int(time.time())}.webp"
# In GridFS speichern
file_id = fs.put(
fs.put(
optimized_io,
filename=new_filename,
content_type='image/webp',
@@ -11966,7 +11979,6 @@ def upload_csv_batch():
}
)
# In Hash-Tabelle und Mapping sichern
processed_hashes[img_hash] = new_filename
image_mapping[base_name_no_ext] = new_filename
processed_count += 1
@@ -11981,8 +11993,8 @@ def upload_csv_batch():
except Exception:
predefined_locations = []
# 5. Dataframe bereinigen & gruppieren
df['Name'] = df['Name'].fillna('Unbenannt').astype(str)
# 5. Dataframe bereinigen
df['Name'] = df['Name'].fillna('Unbenannt').astype(str).str.strip()
df = df.fillna({
'Ort': 'Unbekannt',
'Beschreibung': '',
@@ -11991,21 +12003,28 @@ def upload_csv_batch():
'Anschaffungskosten': ''
})
created_item_ids = []
grouped_items = df.groupby('Name')
# --- WICHTIG: Gruppierung über einen normalisierten Schlüssel ermöglichen ---
# Erstellt eine unsichtbare Hilfsspalte, die Leerzeichen/Groß-Kleinschreibung ignoriert,
# damit identische Artikel-Typen sauber als Serie erkannt werden.
df['GroupKey'] = df['Name'].str.lower()
for name, group in grouped_items:
created_item_ids = []
grouped_items = df.groupby('GroupKey')
for group_key, group in grouped_items:
item_count = len(group)
series_group_id = str(uuid.uuid4()) if item_count > 1 else None
parent_item_id = None
# Originalen Namen des ersten Elements der Gruppe übernehmen
actual_group_name = group.iloc[0]['Name']
# Basis-Code für automatisierte Seriencodes ermitteln
first_row_code = str(group.iloc[0].get('Code_4', '')).strip()
first_row_code = clean_db_field(group.iloc[0].get('Code_4', ''))
base_code = first_row_code if first_row_code else None
for position, (index, row) in enumerate(group.iterrows(), start=1):
# Ort automatisch zu predefined_locations hinzufügen, falls neu
ort_val = str(row['Ort']).strip()
if ort_val and ort_val not in predefined_locations:
try:
@@ -12014,7 +12033,7 @@ def upload_csv_batch():
except Exception as e:
app.logger.warning(f"Ort {ort_val} konnte nicht hinzugefügt werden: {e}")
# Bilder für diesen Artikel zuordnen
# Bilder zuordnen und pro Artikel deduplizieren
item_image_filenames = []
if 'Images' in row and pd.notna(row['Images']):
try:
@@ -12024,11 +12043,14 @@ def upload_csv_batch():
base_img_name = os.path.splitext(img_name)[0]
if base_img_name in image_mapping:
item_image_filenames.append(image_mapping[base_img_name])
else:
app.logger.warning(f"Bild {img_name} in CSV definiert, aber nicht hochgeladen.")
except (ValueError, SyntaxError):
pass
unique_image_filenames = []
for img in item_image_filenames:
if img not in unique_image_filenames:
unique_image_filenames.append(img)
def parse_filter_col(col_data):
try:
res = ast.literal_eval(str(col_data))
@@ -12042,26 +12064,26 @@ def upload_csv_batch():
reservierbar = bool(row.get('Reservierbar', False))
# Code_4 Behandlung: Falls in CSV definiert nutzen, sonst Batch-Code erzeugen
row_code = str(row.get('Code_4', '')).strip()
# Code_4 Behandlung
row_code = clean_db_field(row.get('Code_4', ''))
if row_code:
unique_code = row_code
elif 'generate_unique_batch_code' in globals():
elif item_count > 1:
unique_code = generate_unique_batch_code(base_code, position)
else:
unique_code = None
# DB Insert (exakt abgestimmt auf die 10 positionellen Argumente)
# DB Insert
item_id = it.add_item(
str(row['Name']), # 1. Name
str(actual_group_name), # 1. Name
ort_val, # 2. Ort
str(row['Beschreibung']), # 3. Beschreibung
item_image_filenames, # 4. Image Filenames (GridFS)
unique_image_filenames, # 4. Image Filenames (GridFS)
filter_upload, # 5. Filter 1
filter_upload2, # 6. Filter 2
filter_upload3, # 7. Filter 3
str(row['Anschaffungsjahr']) if row['Anschaffungsjahr'] else None, # 8. Jahr
str(row['Anschaffungskosten']) if row['Anschaffungskosten'] else None, # 9. Kosten
clean_db_field(row.get('Anschaffungsjahr')), # 8. Jahr
clean_db_field(row.get('Anschaffungskosten')),# 9. Kosten
unique_code, # 10. Unique Code / Code_4
reservierbar=reservierbar,
series_group_id=series_group_id,
@@ -12080,16 +12102,11 @@ def upload_csv_batch():
if position == 1:
parent_item_id = str(item_id)
else:
app.logger.error(f"Fehler beim Erstellen von Item: {row['Name']} (Index {index})")
app.logger.info(
f"Batch Upload abgeschlossen: {len(created_item_ids)} Items erstellt. "
f"{processed_count} neue Bilder hochgeladen, {dedup_count} Bild-Duplikate zusammengeführt."
)
app.logger.error(f"Fehler beim Erstellen von Item: {actual_group_name} (Index {index})")
return jsonify({
"success": True,
"message": f"Upload erfolgreich. {len(created_item_ids)} Items importiert. {processed_count} neue Bilder gespeichert ({dedup_count} Duplikate zusammengeführt).",
"message": f"Upload erfolgreich. {len(created_item_ids)} Items importiert.",
"created_count": len(created_item_ids),
"images_processed": processed_count,
"images_deduplicated": dedup_count,
+221 -185
View File
@@ -3,215 +3,251 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Batch Upload - CSV & Bilder</title>
<title>Batch Upload</title>
<!-- CSRF-Token für JavaScript bereitstellen -->
<meta name="csrf-token" content="{{ session.get('_csrf_token', '') }}">
<style>
:root {
--primary-color: #4a90e2;
--background-color: #f4f7f6;
--text-color: #333;
--border-radius: 8px;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background-color: var(--background-color);
color: var(--text-color);
display: flex;
justify-content: center;
align-items: center;
min-height: 100vh;
margin: 0;
}
.upload-container {
background: white;
padding: 2rem;
border-radius: var(--border-radius);
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
width: 100%;
max-width: 500px;
}
h2 {
margin-top: 0;
color: var(--primary-color);
text-align: center;
}
.form-group {
margin-bottom: 1.5rem;
}
label {
display: block;
font-weight: 600;
margin-bottom: 0.5rem;
}
input[type="file"] {
display: block;
width: 100%;
padding: 0.5rem;
border: 1px dashed #ccc;
border-radius: var(--border-radius);
background: #fafafa;
cursor: pointer;
}
.btn-submit {
width: 100%;
padding: 0.75rem;
background-color: var(--primary-color);
color: white;
border: none;
border-radius: var(--border-radius);
font-size: 1rem;
font-weight: 600;
cursor: pointer;
transition: background-color 0.3s ease;
}
.btn-submit:hover {
background-color: #357abd;
}
.btn-submit:disabled {
background-color: #a0c4e8;
cursor: not-allowed;
}
/* Status & Feedback Messages */
#status-message {
margin-top: 1rem;
padding: 1rem;
border-radius: var(--border-radius);
display: none;
text-align: center;
}
.success {
background-color: #d4edda;
color: #155724;
border: 1px solid #c3e6cb;
}
.error {
background-color: #f8d7da;
color: #721c24;
border: 1px solid #f5c6cb;
}
.loading {
background-color: #e2e3e5;
color: #383d41;
border: 1px solid #d6d8db;
}
.spinner {
display: inline-block;
width: 1.5rem;
height: 1.5rem;
border: 3px solid rgba(0,0,0,0.1);
border-radius: 50%;
border-top-color: var(--primary-color);
animation: spin 1s ease-in-out infinite;
vertical-align: middle;
margin-right: 0.5rem;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
body { font-family: sans-serif; padding: 20px; background: #f4f7f6; }
.upload-container { background: white; padding: 20px; border-radius: 8px; max-width: 500px; margin: 0 auto; }
.form-group { margin-bottom: 15px; }
label { display: block; font-weight: bold; margin-bottom: 5px; }
input[type="file"] { width: 100%; padding: 8px; box-sizing: border-box; }
.btn-submit { width: 100%; padding: 10px; background: #4a90e2; color: white; border: none; border-radius: 4px; font-weight: bold; cursor: pointer; }
.btn-submit:disabled { background: #ccc; }
#uploadProgress { margin-top: 20px; display: none; }
progress { width: 100%; height: 20px; }
#logList { background: #fafafa; border: 1px solid #eee; padding: 10px; max-height: 150px; overflow-y: auto; font-size: 0.85em; list-style: none; }
</style>
</head>
<body>
<div class="upload-container">
<h2>Inventar Batch Upload</h2>
<div class="upload-container">
<h2>Inventar Batch Upload</h2>
<form id="uploadForm">
<div class="form-group">
<label for="csv_file">1. items.csv Datei auswählen</label>
<!-- Akzeptiert nur CSV Dateien -->
<input type="file" id="csv_file" name="csv_file" accept=".csv" required>
</div>
<form id="batchUploadForm">
<div class="form-group">
<label for="csv_file">1. items.csv auswählen</label>
<input type="file" id="csv_file" name="csv_file" accept=".csv" required>
</div>
<div class="form-group">
<label for="images">2. Bilder auswählen</label>
<!-- multiple erlaubt das Auswählen mehrerer Bilder gleichzeitig -->
<input type="file" id="images" name="images" accept="image/*" multiple required>
<small style="color: #666; display: block; margin-top: 5px;">Du kannst mehrere Bilder markieren (Strg/Cmd gedrückt halten).</small>
</div>
<div class="form-group">
<label for="images">2. Bilder auswählen</label>
<input type="file" id="images" name="images" accept="image/*" multiple required>
</div>
<button type="submit" id="submitBtn" class="btn-submit">Daten hochladen</button>
</form>
<button type="submit" id="uploadBtn" class="btn-submit">Daten hochladen</button>
</form>
<div id="status-message"></div>
<div id="uploadProgress">
<div id="progressText">Bereite Upload vor...</div>
<progress id="progressBar" value="0" max="100"></progress>
<ul id="logList"></ul>
</div>
</div>
<script>
document.getElementById('uploadForm').addEventListener('submit', async function(e) {
e.preventDefault();
// 1. Ein echter CSV-Parser, der Zeilenumbrüche und Kommas in Texten korrekt ignoriert
function parseCSV(csvString) {
const rows = [];
let currentRow = [];
let currentCell = '';
let insideQuotes = false;
const form = e.target;
const submitBtn = document.getElementById('submitBtn');
const statusDiv = document.getElementById('status-message');
for (let i = 0; i < csvString.length; i++) {
const char = csvString[i];
const nextChar = csvString[i + 1];
submitBtn.disabled = true;
submitBtn.innerText = 'Wird verarbeitet...';
statusDiv.className = 'loading';
statusDiv.style.display = 'block';
statusDiv.innerHTML = '<div class="spinner"></div> Lade Dateien hoch und verarbeite Bilder... Bitte warten.';
if (char === '"' && insideQuotes && nextChar === '"') {
currentCell += '"';
i++; // Escaped Quotes ("") überspringen
} else if (char === '"') {
insideQuotes = !insideQuotes;
} else if (char === ',' && !insideQuotes) {
currentRow.push(currentCell);
currentCell = '';
} else if ((char === '\n' || char === '\r') && !insideQuotes) {
if (char === '\r' && nextChar === '\n') i++; // Windows Umbrüche überspringen
currentRow.push(currentCell);
// Leere Zeilen ignorieren
if (currentRow.length > 1 || currentRow[0] !== '') {
rows.push(currentRow);
}
currentRow = [];
currentCell = '';
} else {
currentCell += char;
}
}
if (currentCell !== '' || currentRow.length > 0) {
currentRow.push(currentCell);
if (currentRow.length > 1 || currentRow[0] !== '') {
rows.push(currentRow);
}
}
return rows;
}
const formData = new FormData(form);
// 2. Baut das Array wieder sicher zu einer sauberen CSV-Zeile für das Backend zusammen
function rowToCSV(rowArray) {
return rowArray.map(cell => {
if (cell === null || cell === undefined) return '';
let cellStr = String(cell);
// Wenn kritische Zeichen drin sind, sauber in Anführungszeichen verpacken
if (cellStr.includes(',') || cellStr.includes('"') || cellStr.includes('\n') || cellStr.includes('\r')) {
return '"' + cellStr.replace(/"/g, '""') + '"';
}
return cellStr;
}).join(',');
}
const fetchOptions = {
method: 'POST',
body: formData,
credentials: 'include',
headers: {
'X-Requested-With': 'XMLHttpRequest'
}
};
// 3. Bildnamen extrahieren (bleibt wie es war)
function extractImageNames(cellValue) {
if (!cellValue) return [];
let raw = String(cellValue).replace(/^["']|["']$/g, '').trim();
if (raw.startsWith('[') && raw.endsWith(']')) {
try {
const jsonValid = raw.replace(/'/g, '"');
return JSON.parse(jsonValid);
} catch (e) {
const matches = raw.match(/['"]([^'"]+)['"]/g);
if (matches) return matches.map(m => m.replace(/['"]/g, ''));
}
}
return raw ? [raw] : [];
}
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');
document.getElementById('batchUploadForm').addEventListener('submit', async function(e) {
e.preventDefault();
const csvInput = document.getElementById('csv_file');
const imageInput = document.getElementById('images');
const uploadBtn = document.getElementById('uploadBtn');
const progressContainer = document.getElementById('uploadProgress');
const progressBar = document.getElementById('progressBar');
const progressText = document.getElementById('progressText');
const logList = document.getElementById('logList');
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');
if (!csvInput.files.length) {
alert("Bitte wähle eine CSV-Datei aus.");
return;
}
uploadBtn.disabled = true;
progressContainer.style.display = 'block';
logList.innerHTML = '';
const log = (msg) => {
const li = document.createElement('li');
li.textContent = msg;
logList.appendChild(li);
logList.scrollTop = logList.scrollHeight;
};
const csvFile = csvInput.files[0];
const allImages = Array.from(imageInput.files);
const imageMap = new Map();
allImages.forEach(file => {
imageMap.set(file.name.toLowerCase(), file);
});
const BATCH_SIZE = 50;
try {
const csvText = await csvFile.text();
// Hier rufen wir jetzt unseren sicheren Parser auf!
const allRows = parseCSV(csvText);
if (allRows.length <= 1) {
throw new Error("CSV-Datei ist leer oder enthält nur Kopfzeilen.");
}
const headerRow = allRows[0];
const dataRows = allRows.slice(1);
log(`${dataRows.length} Einträge gefunden. Bereite Batches vor...`);
// Spalten-Index von "Images" sicher ermitteln
const imagesColIndex = headerRow.findIndex(h => h.toLowerCase().trim() === 'images');
// In Batches aufteilen
const batches = [];
for (let i = 0; i < dataRows.length; i += BATCH_SIZE) {
batches.push(dataRows.slice(i, i + BATCH_SIZE));
}
progressBar.max = batches.length;
progressBar.value = 0;
for (let b = 0; b < batches.length; b++) {
const batchRows = batches[b];
progressText.textContent = `Lade Batch ${b + 1} von ${batches.length} hoch...`;
const requiredImagesForBatch = new Set();
if (imagesColIndex !== -1) {
batchRows.forEach(rowCols => {
if (rowCols[imagesColIndex]) {
const imgNames = extractImageNames(rowCols[imagesColIndex]);
imgNames.forEach(name => {
const fileMatch = imageMap.get(name.toLowerCase());
if (fileMatch) {
requiredImagesForBatch.add(fileMatch);
}
});
}
});
}
// Sicherer Zusammenbau des CSV-Batches
const batchCsvArray = [headerRow, ...batchRows];
const batchCsvText = batchCsvArray.map(rowToCSV).join('\n');
const batchCsvBlob = new Blob([batchCsvText], { type: 'text/csv' });
const formData = new FormData();
formData.append('csv_file', batchCsvBlob, `batch_${b + 1}.csv`);
if (csrfToken) {
fetchOptions.headers = {
'X-CSRFToken': csrfToken
};
formData.append('csrf_token', csrfToken);
}
try {
const response = await fetch('/upload_csv_batch', fetchOptions);
requiredImagesForBatch.forEach(imgFile => {
formData.append('images', imgFile);
});
// Antwort einmalig als Text auslesen, um sowohl JSON als auch HTML-Fehler abzufangen
const responseText = await response.text();
log(`Batch ${b + 1}: ${batchRows.length} Items & ${requiredImagesForBatch.size} zugehörige Bilder.`);
let result;
try {
result = JSON.parse(responseText);
} catch (jsonError) {
console.error("Server hat kein JSON gesendet. Antwort war:", responseText);
throw new Error("Der Server hat einen HTML-Fehler zurückgegeben (z.B. Nginx 413 Entity Too Large oder Server-Crash). Siehe F12 Konsole.");
const response = await fetch('/upload_csv_batch', {
method: 'POST',
body: formData,
headers: {
'X-CSRFToken': csrfToken || ''
}
});
if (response.ok && result.success) {
statusDiv.className = 'success';
statusDiv.innerHTML = `<strong>Erfolg!</strong><br>${result.message}`;
form.reset();
} else {
statusDiv.className = 'error';
statusDiv.innerHTML = `<strong>Fehler:</strong> ${result.message || 'Ein unbekannter Fehler ist aufgetreten.'}`;
}
const result = await response.json();
} catch (error) {
statusDiv.className = 'error';
statusDiv.innerHTML = `<strong>Fehler:</strong> ${error.message}`;
console.error('Upload Error:', error);
} finally {
submitBtn.disabled = false;
submitBtn.innerText = 'Daten hochladen';
if (!response.ok || !result.success) {
throw new Error(result.message || `Server-Fehler ${response.status}`);
}
});
</script>
log(`Batch ${b + 1} abgeschlossen.`);
progressBar.value = b + 1;
}
progressText.textContent = "Upload erfolgreich beendet! Keine verschobenen Spalten mehr.";
uploadBtn.disabled = false;
} catch (err) {
alert("Upload abgebrochen: " + err.message);
log("Fehler: " + err.message);
uploadBtn.disabled = false;
}
});
</script>
</body>
</html>