fix of the json upload format
This commit is contained in:
@@ -48,30 +48,64 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
// Robuster CSV-Parser, der Kommas innerhalb von Anführungszeichen ignoriert
|
// 1. Ein echter CSV-Parser, der Zeilenumbrüche und Kommas in Texten korrekt ignoriert
|
||||||
function parseCSVLine(text) {
|
function parseCSV(csvString) {
|
||||||
const result = [];
|
const rows = [];
|
||||||
let cur = '';
|
let currentRow = [];
|
||||||
let inQuotes = false;
|
let currentCell = '';
|
||||||
for (let i = 0; i < text.length; i++) {
|
let insideQuotes = false;
|
||||||
const c = text[i];
|
|
||||||
if (c === '"' || c === "'") {
|
for (let i = 0; i < csvString.length; i++) {
|
||||||
inQuotes = !inQuotes;
|
const char = csvString[i];
|
||||||
} else if (c === ',' && !inQuotes) {
|
const nextChar = csvString[i + 1];
|
||||||
result.push(cur.trim());
|
|
||||||
cur = '';
|
if (char === '"' && insideQuotes && nextChar === '"') {
|
||||||
|
currentCell += '"';
|
||||||
|
i++; // Escaped Quotes ("") überspringen
|
||||||
|
} else if (char === '"') {
|
||||||
|
insideQuotes = !insideQuotes;
|
||||||
|
} else if (char === ',' && !insideQuotes) {
|
||||||
|
currentRow.push(currentCell);
|
||||||
|
currentCell = '';
|
||||||
|
} else if ((char === '\n' || char === '\r') && !insideQuotes) {
|
||||||
|
if (char === '\r' && nextChar === '\n') i++; // Windows Umbrüche überspringen
|
||||||
|
currentRow.push(currentCell);
|
||||||
|
// Leere Zeilen ignorieren
|
||||||
|
if (currentRow.length > 1 || currentRow[0] !== '') {
|
||||||
|
rows.push(currentRow);
|
||||||
|
}
|
||||||
|
currentRow = [];
|
||||||
|
currentCell = '';
|
||||||
} else {
|
} else {
|
||||||
cur += c;
|
currentCell += char;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result.push(cur.trim());
|
if (currentCell !== '' || currentRow.length > 0) {
|
||||||
return result;
|
currentRow.push(currentCell);
|
||||||
|
if (currentRow.length > 1 || currentRow[0] !== '') {
|
||||||
|
rows.push(currentRow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extrahiert Bildnamen aus Einträgen wie "['bild1.jpg', 'bild2.jpg']" oder "bild1.jpg"
|
// 2. Baut das Array wieder sicher zu einer sauberen CSV-Zeile für das Backend zusammen
|
||||||
|
function rowToCSV(rowArray) {
|
||||||
|
return rowArray.map(cell => {
|
||||||
|
if (cell === null || cell === undefined) return '';
|
||||||
|
let cellStr = String(cell);
|
||||||
|
// Wenn kritische Zeichen drin sind, sauber in Anführungszeichen verpacken
|
||||||
|
if (cellStr.includes(',') || cellStr.includes('"') || cellStr.includes('\n') || cellStr.includes('\r')) {
|
||||||
|
return '"' + cellStr.replace(/"/g, '""') + '"';
|
||||||
|
}
|
||||||
|
return cellStr;
|
||||||
|
}).join(',');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Bildnamen extrahieren (bleibt wie es war)
|
||||||
function extractImageNames(cellValue) {
|
function extractImageNames(cellValue) {
|
||||||
if (!cellValue) return [];
|
if (!cellValue) return [];
|
||||||
let raw = cellValue.replace(/^["']|["']$/g, '').trim();
|
let raw = String(cellValue).replace(/^["']|["']$/g, '').trim();
|
||||||
if (raw.startsWith('[') && raw.endsWith(']')) {
|
if (raw.startsWith('[') && raw.endsWith(']')) {
|
||||||
try {
|
try {
|
||||||
const jsonValid = raw.replace(/'/g, '"');
|
const jsonValid = raw.replace(/'/g, '"');
|
||||||
@@ -116,7 +150,6 @@ document.getElementById('batchUploadForm').addEventListener('submit', async func
|
|||||||
const csvFile = csvInput.files[0];
|
const csvFile = csvInput.files[0];
|
||||||
const allImages = Array.from(imageInput.files);
|
const allImages = Array.from(imageInput.files);
|
||||||
|
|
||||||
// Map für schnellen Dateinamen-Vergleich (ohne Pfadangabe)
|
|
||||||
const imageMap = new Map();
|
const imageMap = new Map();
|
||||||
allImages.forEach(file => {
|
allImages.forEach(file => {
|
||||||
imageMap.set(file.name.toLowerCase(), file);
|
imageMap.set(file.name.toLowerCase(), file);
|
||||||
@@ -126,57 +159,40 @@ document.getElementById('batchUploadForm').addEventListener('submit', async func
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const csvText = await csvFile.text();
|
const csvText = await csvFile.text();
|
||||||
let rows = csvText.split(/\r?\n/).filter(r => r.trim().length > 0);
|
// Hier rufen wir jetzt unseren sicheren Parser auf!
|
||||||
|
const allRows = parseCSV(csvText);
|
||||||
|
|
||||||
if (rows.length <= 1) {
|
if (allRows.length <= 1) {
|
||||||
throw new Error("CSV-Datei ist leer oder enthält nur Kopfzeilen.");
|
throw new Error("CSV-Datei ist leer oder enthält nur Kopfzeilen.");
|
||||||
}
|
}
|
||||||
|
|
||||||
const headerRow = rows[0];
|
const headerRow = allRows[0];
|
||||||
const dataRows = rows.slice(1);
|
const dataRows = allRows.slice(1);
|
||||||
|
|
||||||
// 1. Client-Deduplizierung
|
log(`${dataRows.length} Einträge gefunden. Bereite Batches vor...`);
|
||||||
const uniqueRowsSet = new Set();
|
|
||||||
const uniqueDataRows = [];
|
|
||||||
let duplicateCount = 0;
|
|
||||||
|
|
||||||
for (const row of dataRows) {
|
// Spalten-Index von "Images" sicher ermitteln
|
||||||
if (uniqueRowsSet.has(row)) {
|
const imagesColIndex = headerRow.findIndex(h => h.toLowerCase().trim() === 'images');
|
||||||
duplicateCount++;
|
|
||||||
} else {
|
|
||||||
uniqueRowsSet.add(row);
|
|
||||||
uniqueDataRows.push(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log(`${uniqueDataRows.length} eindeutige Einträge. ${duplicateCount} Duplikate entfernt.`);
|
// In Batches aufteilen
|
||||||
|
|
||||||
// Spalten-Index von "Images" ermitteln
|
|
||||||
const headers = parseCSVLine(headerRow).map(h => h.replace(/['"]/g, '').trim());
|
|
||||||
const imagesColIndex = headers.findIndex(h => h.toLowerCase() === 'images');
|
|
||||||
|
|
||||||
// 2. In Batches aufteilen
|
|
||||||
const batches = [];
|
const batches = [];
|
||||||
for (let i = 0; i < uniqueDataRows.length; i += BATCH_SIZE) {
|
for (let i = 0; i < dataRows.length; i += BATCH_SIZE) {
|
||||||
batches.push(uniqueDataRows.slice(i, i + BATCH_SIZE));
|
batches.push(dataRows.slice(i, i + BATCH_SIZE));
|
||||||
}
|
}
|
||||||
|
|
||||||
progressBar.max = batches.length;
|
progressBar.max = batches.length;
|
||||||
progressBar.value = 0;
|
progressBar.value = 0;
|
||||||
|
|
||||||
// 3. Sequenzieller Upload
|
|
||||||
for (let b = 0; b < batches.length; b++) {
|
for (let b = 0; b < batches.length; b++) {
|
||||||
const batchRows = batches[b];
|
const batchRows = batches[b];
|
||||||
progressText.textContent = `Lade Batch ${b + 1} von ${batches.length} hoch...`;
|
progressText.textContent = `Lade Batch ${b + 1} von ${batches.length} hoch...`;
|
||||||
|
|
||||||
// Zuordnung der benötigten Bilder für diesen Batch
|
|
||||||
const requiredImagesForBatch = new Set();
|
const requiredImagesForBatch = new Set();
|
||||||
|
|
||||||
if (imagesColIndex !== -1) {
|
if (imagesColIndex !== -1) {
|
||||||
batchRows.forEach(rowStr => {
|
batchRows.forEach(rowCols => {
|
||||||
const cols = parseCSVLine(rowStr);
|
if (rowCols[imagesColIndex]) {
|
||||||
if (cols[imagesColIndex]) {
|
const imgNames = extractImageNames(rowCols[imagesColIndex]);
|
||||||
const imgNames = extractImageNames(cols[imagesColIndex]);
|
|
||||||
imgNames.forEach(name => {
|
imgNames.forEach(name => {
|
||||||
const fileMatch = imageMap.get(name.toLowerCase());
|
const fileMatch = imageMap.get(name.toLowerCase());
|
||||||
if (fileMatch) {
|
if (fileMatch) {
|
||||||
@@ -187,26 +203,24 @@ document.getElementById('batchUploadForm').addEventListener('submit', async func
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// CSV für diesen Batch erstellen
|
// Sicherer Zusammenbau des CSV-Batches
|
||||||
const batchCsvText = [headerRow, ...batchRows].join('\n');
|
const batchCsvArray = [headerRow, ...batchRows];
|
||||||
|
const batchCsvText = batchCsvArray.map(rowToCSV).join('\n');
|
||||||
const batchCsvBlob = new Blob([batchCsvText], { type: 'text/csv' });
|
const batchCsvBlob = new Blob([batchCsvText], { type: 'text/csv' });
|
||||||
|
|
||||||
const formData = new FormData();
|
const formData = new FormData();
|
||||||
formData.append('csv_file', batchCsvBlob, `batch_${b + 1}.csv`);
|
formData.append('csv_file', batchCsvBlob, `batch_${b + 1}.csv`);
|
||||||
|
|
||||||
// CSRF-Token im Form-Field übergeben
|
|
||||||
if (csrfToken) {
|
if (csrfToken) {
|
||||||
formData.append('csrf_token', csrfToken);
|
formData.append('csrf_token', csrfToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Zugehörige Bilder anfügen
|
|
||||||
requiredImagesForBatch.forEach(imgFile => {
|
requiredImagesForBatch.forEach(imgFile => {
|
||||||
formData.append('images', imgFile);
|
formData.append('images', imgFile);
|
||||||
});
|
});
|
||||||
|
|
||||||
log(`Batch ${b + 1}: ${batchRows.length} Items & ${requiredImagesForBatch.size} zugehörige Bilder.`);
|
log(`Batch ${b + 1}: ${batchRows.length} Items & ${requiredImagesForBatch.size} zugehörige Bilder.`);
|
||||||
|
|
||||||
// Request mit CSRF-Header ausführen
|
|
||||||
const response = await fetch('/upload_csv_batch', {
|
const response = await fetch('/upload_csv_batch', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: formData,
|
body: formData,
|
||||||
@@ -221,11 +235,11 @@ document.getElementById('batchUploadForm').addEventListener('submit', async func
|
|||||||
throw new Error(result.message || `Server-Fehler ${response.status}`);
|
throw new Error(result.message || `Server-Fehler ${response.status}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
log(`Batch ${b + 1} abgeschlossen: ${result.message}`);
|
log(`Batch ${b + 1} abgeschlossen.`);
|
||||||
progressBar.value = b + 1;
|
progressBar.value = b + 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
progressText.textContent = "Upload erfolgreich beendet!";
|
progressText.textContent = "Upload erfolgreich beendet! Keine verschobenen Spalten mehr.";
|
||||||
uploadBtn.disabled = false;
|
uploadBtn.disabled = false;
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
Reference in New Issue
Block a user