const path = require("path"); const fs = require("fs"); const dotenv = require("dotenv"); dotenv.config(); // dotenv.config({ path: ".env.local" }); const { Shift, Publisher, PrismaClient } = require("@prisma/client"); const CON = require("./const"); const common = require("./common"); const data = require("./data"); //const { fi } = require("date-fns/locale"); //Works with nextjs, but fails with nodejs // for nodejs //const api = require("./pages/api/index"); exports.GenerateExcel = async function (req, res) { const prisma = common.getPrismaClient(); const year = req.params.year; const month = parseInt(req.params.month) - 1; const fromDate = new Date(year, month, 1); // month is 0 based // to last day of the month. special case december const toDate = new Date(year, month + 1, 0); // month is 0 based // toDate.setMonth(fromDate.getMonth() + 1); //get all shiifts for the month var shifts = await prisma.shift.findMany({ where: { startTime: { gte: fromDate, lt: toDate, }, }, include: { cartEvent: { include: { location: true, }, }, publishers: true, }, }); var filePath = path.join(CON.contentPath, "График КОЛИЧКИ.xlsx"); const bеgin = new Date(); //----------------- exit "График КОЛИЧКИ.xlsx" with exceljs ---------------- const ExcelJS = require("exceljs"); const xjswb = new ExcelJS.Workbook(); if (req.params.process == "1") { try { xjswb.xlsx .readFile(filePath) .then(function () { try { var worksheet = xjswb.getWorksheet(13); //get row 2 with all the styles var weekHeader = worksheet.getRow(2); var newWorksheet = xjswb.addWorksheet( CON.monthNamesBG[month] ); newWorksheet.name = CON.monthNamesBG[month].toUpperCase(); //copy each row from the template with all the styles worksheet.eachRow( { includeEmpty: true }, function (row, rowNumber) { var newRow = newWorksheet.getRow(rowNumber); newRow.height = row.height; row.eachCell( { includeEmpty: true }, function (cell, colNumber) { var newCell = newRow.getCell(colNumber); newCell.value = cell.value; newCell.font = cell.font; newCell.alignment = cell.alignment; newCell.border = cell.border; newCell.fill = cell.fill; newCell.numberFormat = cell.numberFormat; newCell.protection = cell.protection; } ); } ); // worksheet.eachRow({ includeEmpty: true }, function (row, rowNumber) { // row.eachCell({ includeEmpty: true }, function (cell, colNumber) { // newWorksheet.getCell(rowNumber, colNumber).value = cell.value; // }); // }); for (let i = start.row; i <= end.row; i++) { const leftBorderCell = worksheet.getCell(i, start.col); //hide original sheet worksheet.state = "hidden"; //save file xjswb.xlsx.writeFile( path.join( contentPath, `График КОЛИЧКИ ${year}-${month + 1}.xlsx` ) ); //send the file to the client res.setHeader( "Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ); res.setHeader( "Content-Disposition", "attachment; filename=" + encodeURI(`График КОЛИЧКИ ${year}-${month + 1}.xlsx`) ); xjswb.xlsx.write(res); } } catch (err) { console.log(err); res.end( "[" + new Date().toLocaleString() + "] (" + (new Date() - bеgin) + "ms) " + err ); } }) .then(function () { console.log("done"); //show cyrillic text in response res.setHeader("Content-Type", "text/html; charset=utf-8"); res.end( "Генериране на График КОЛИЧКИ${year}-${month}.xlsx завършено успешно за " + (new Date() - bеgin) + "ms" ); }); } catch (err) { console.log(err); res.end(err.message); } } //----------------- exit "График КОЛИЧКИ.xlsx" with xlsx-style ---------------- if (req.params.process == "2") { const XLSX = require('xlsx'); const wb = XLSX.utils.book_new(); wb.Props = { Title: "График КОЛИЧКИ", Subject: "График КОЛИЧКИ", }; wb.SheetNames.push("График КОЛИЧКИ"); const ws_data = [ [1, 2, 3], [true, false, null, "sheetjs"], ["foo", "bar", new Date("2014-02-19T14:30Z"), "0.3"], ["baz", null, "qux"] ]; const ws = XLSX.utils.aoa_to_sheet(ws_data); wb.Sheets["График КОЛИЧКИ"] = ws; const xlsxstyle = require("xlsx-style"); try { const workbook = xlsxstyle.readFile(filePath); const sheetNames = workbook.SheetNames; //find the sheet with the name "Зима" in sheetNames const sheetName = sheetNames.find((name) => name === "Зима"); // Get the data of "Sheet1" const data = xlsxstyle.utils.sheet_to_json(workbook.Sheets[sheetNames[2]]); var worksheet = workbook.Sheets[sheetName]; //copy worksheet to new workbook //add new worksheet to new workbook with month name // var newWorksheet = wb.addWorksheet(CON.monthNamesBG[month]); var rows = xlsxstyle.utils.sheet_to_row_object_array(worksheet, { header: 1, }); XLSX.utils.book_append_sheet(wb, worksheet, "_" + CON.monthNamesBG[month]); //save file XLSX.writeFile(wb, path.join(CON.contentPath, `_График КОЛИЧКИ ${year}-${month + 1}.xlsx`)); //save file xlsxstyle.writeFile( newWorkbook, path.join( contentPath, `График КОЛИЧКИ ${year}-${month + 1}.xlsx` ) ); //send the file to the client res.setHeader( "Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ); res.setHeader( "Content-Disposition", "attachment; filename=" + encodeURI(`График КОЛИЧКИ ${year}-${month + 1}.xlsx`) ); xlsxstyle.writeFile(newWorkbook, res); res.setHeader("Content-Type", "text/html; charset=utf-8"); res.end( "Генериране на График КОЛИЧКИ${year}-${month}.xlsx завършено успешно за " + (new Date() - bеgin) + " ms!" ); } catch (err) { console.log(err); res.end("[" + new Date().toLocaleString() + "] " + err); } } //----------------- exit "График КОЛИЧКИ.xlsx" with node-excel-export ---------------- if (req.params.process == "3") { // // uses xlsx-style in the background // // https://github.com/protobi/js-xlsx#cell-styles // // https://www.npmjs.com/package/node-excel-export // var excel = require('node-excel-export'); // var rows = [[ // { value: "EXTREMELY LONG TITLE 1", bold: 1, autoWidth: true }, // { value: "TITLE2" }, // { value: "TITLE3" } // ]]; // var styles = { // headerHilight: { // fill: { // fgColor: { // rgb: 'FFE36600' // } // }, // font: { // color: { // rgb: 'FFFFFFFF' // }, // sz: 10, // bold: true, // // underline: true // } // }, // cellOdd: { // fill: { // fgColor: { // rgb: 'FFF8F8F7' // } // } // } // }; // const heading = [ // [{value: 'b1', style: styles.headerHilight}, // {value: 'd1', style: styles.headerHilight}, // {value: 'e1', style: styles.headerHilight}], // ['b2', 'd2', 'e2'] // <-- It can be only values // ]; // var specification = { // "shiftTime": { // displayName: 'Смяна', // headerStyle: styles.headerHilight, // cellStyle: styles.cellOdd, // width: 60 // }, // "publisherName": { // "displayName": 'ПЛИСКА ПОНЕДЕЛНИК', // "headerStyle": styles.headerHilight, // "width": 250 // }, // "Col2": { // "displayName": 'СТАДИОН СРЯДА', // "headerStyle": styles.headerHilight, // "width": 215 // }, // "Col3": { // displayName: 'УНИВЕРСИТЕТ ЧЕТВЪРТЪК', // headerStyle: styles.headerHilight, // width: 150 // } // } // var report = excel.buildExport( // [{ // name: `График КОЛИЧКИ ${year}-${month + 1}.xlsx`, // specification: specification, // heading: heading, // <- Raw heading array (optional) // data: rows // }]); // //save file to disk // fs.writeFile(path.join(contentPath, `График КОЛИЧКИ ${year}-${month + 1}.xlsx`), report, 'binary', function (err) { }); // res.setHeader('Content-Type', 'text/html; charset=utf-8'); // res.end("Генериране на График КОЛИЧКИ${year}-${month}.xlsx завършено успешно за " + (new Date() - bеgin) + " ms!"); // //send the file to the client // console.log("excel genarated in " + (new Date() - bеgin) + "ms"); // // res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); // // res.setHeader("Content-Disposition", "attachment; filename=" + encodeURI(`График КОЛИЧКИ ${year}-${month + 1}.xlsx`)); // // res.end(report); } } exports.ImportFromExcel = function (req, res) { } exports.processEvents = async function (events, year, monthNumber, progressCallback, createAvailabilities) { const prisma = common.getPrismaClient(); const d = new Date(year, monthNumber - 1);//month is 0 based in js const monthDatesInfo = common.getMonthDatesInfo(d); //CAL.GetMonthDatesInfo(date); try { await prisma.shift.deleteMany({ where: { startTime: { gte: monthDatesInfo.firstMonday, lt: monthDatesInfo.lastSunday, }, }, }); } catch (e) { console.log(e); } var shifts = await prisma.shift.findMany({ where: { isActive: true, startTime: { gte: monthDatesInfo.firstMonday, lt: monthDatesInfo.lastSunday, }, } }); var locations = await prisma.location.findMany({ where: { isActive: true, } }); var cartEvents = await prisma.cartEvent.findMany({ where: { isActive: true, } }); var publishers = await prisma.publisher.findMany({ where: { isActive: true, }, include: { availabilities: { where: { isActive: true, }, }, assignments: { include: { shift: true, }, }, }, }); const totalEvents = events.length; for (let i = 0; i < totalEvents; i++) { const event = events[i]; const progress = (i / totalEvents) * 100; if (progress > 1) { progressCallback(progress); } try { const date = new Date(event.date); let startStr, endStr; if (event.time) { startStr = event.time.split("-")[0].trim(); endStr = event.time.split("-")[1].trim(); } else { //get the start event time and event.shiftNr and calculate start and end based on that const shift = shifts.find((s) => s.nr === event.shiftNr); if (!shift) { console.warn(`Could not find shift with nr '${event.shiftNr}'`); continue; } startStr = shift.startTime; endStr = shift.endTime; } let st = new Date(event.date); st.setHours(startStr.split(":")[0]); st.setMinutes(startStr.split(":")[1]); const start = st; st = new Date(event.date); st.setHours(endStr.split(":")[0]); st.setMinutes(endStr.split(":")[1]); const end = st var location = locations.find((l) => l.name.toLowerCase().includes(event.placeOfEvent.toLowerCase()) ); if (!location) { console.warn(`Could not find location with name '${event.placeOfEvent}'`); //await prisma.location.create({ data: { name: event.placeOfEvent } }); continue; } var dayofWeek = common.getDayOfWeekNameEnEnumForDate(date); const cartEvent = cartEvents.find( (ce) => ce.locationId === location.id && ce.dayofweek === dayofWeek ); if (!cartEvent) { console.warn(`Could not find cart event for date '${date}' and location '${event.placeOfEvent}'`); continue; } let shift = shifts.find((s) => s.cartEventId === cartEvent.id && new Date(s.startTime).getTime() === new Date(start).getTime() ); // get only hh:mm from the date let isTransportRequired = event.shiftNr == 1 || end.toLocaleTimeString().substring(0, 5) == cartEvent.endTime.toLocaleTimeString().substring(0, 5); if (!shift) { //if shiftnr = 1, notes = "Докарва" + event.transport //if shiftnr = 8, notes = "Взема" + event.transport let note = isTransportRequired ? event.transport : ""; // "Докарва количка от Люлин/Прибира количка в Люлин" const shiftEntity = await prisma.shift.create({ data: { name: event.dayOfWeek + " " + event.dayOfMonth + ", " + start.toLocaleTimeString() + " - " + end.toLocaleTimeString(), startTime: start, endTime: end, notes: note, requiresTransport: isTransportRequired, cartEvent: { connect: { id: cartEvent.id, }, }, }, }); shift = shiftEntity; console.log(`Created shift with ID ${shiftEntity.id} for cart event with ID ${cartEvent.id} on ${date} from ${start.toLocaleTimeString()} to ${end.toLocaleTimeString()}`); } for (const nameOrFamily of event.names) { for (const name of common.separateFamilyMemberNames2(nameOrFamily)) { var publisher = null const pubs = await data.findPublisher(name, null, "id,email,firstName,lastName", true); publisher = pubs[0]; if (!publisher) { const fuzzyPublisher = common.fuzzySearch(publishers, name); if (fuzzyPublisher) { console.log( `Found publisher '${fuzzyPublisher.firstName} ${fuzzyPublisher.lastName}' through fuzzy search for '${name}'` ); publisher = fuzzyPublisher; } else { console.warn(`NO publisher found! Could not find publisher with name '${name}'. Creating new publisher from known info.`); //continue; try { let firstname = name.substring(0, name.lastIndexOf(" ")).trim(); let lastname = name.substring(name.lastIndexOf(" ") + 1).trim(); // Remove the last letter if it is "а" or "и" // if (lastname.endsWith('а') || lastname.endsWith('и')) { if (lastname.endsWith('и')) { lastname = lastname.slice(0, -1); } //if any name is empty, skip this publisher if (firstname == "" || lastname == "") { console.warn(`NO publisher found! Could not find publisher with name '${name}', but we need both first and last name. Skipping this publisher.`); continue; } //var name = names[i].trim(); // //cut last letter of name if it is "a" or "и" (bulgarian feminine ending) // if (name.endsWith("a") || name.endsWith("и")) { // name = name.substring(0, name.length - 1); // } var manualPub = { email: name.toLowerCase().replace(/ /g, "."), // + "@gmail.com" firstName: firstname, lastName: lastname, isActive: true, isImported: true, // role: "EXTERNAL", }; publisher = await prisma.publisher.create({ data: manualPub }); console.log(`Created publisher with ID ${publisher.id} for name '${name}'`); // create availability with the same date as the event. //ToDo: add parameter to control if we want to create availability for each event. can be done whe we import previous shifts. // if (createAvailabilities) { // const dayofWeek = common.getDayOfWeekNameEnEnumForDate(date); // const availability = await prisma.availability.create({ // data: { // publisherId: publisher.id, // dayofweek: dayofWeek, // startTime: startTime, // endTime: endTime, // name: `от график, ${publisher.firstName} ${publisher.lastName}`, // isFromPreviousAssignment: true, // isActive: true, // }, // }); // console.log(`Created WEEKLY availability with ID ${availability.id} for date '${date.toDateString()}' and publisher '${publisher.firstName} ${publisher.lastName}'`); // } const personResponse = await axiosInstance.post("/publishers", manualPub); // let personId = personResponse.data.id; } catch (e) { console.error(`shiftCache: error adding MANUAL publisher to the system (${manualPub.email} ${manualPub.firstName} ${manualPub.lastName}): ` + e); } } } if (location != null && publisher != null && shift != null) { let isWithTransport = false; if (isTransportRequired) { const pubInitials = publisher.firstName[0] + publisher.lastName[0]; // get cotent after last - or long dash-`-` and remove spaces, trim dots and make lowercase let transportInitials = event.transport.split("-").pop().replace(/[\s.]/g, "").toUpperCase(); isWithTransport = transportInitials.includes(pubInitials); } const assignment = await prisma.assignment.create({ data: { //publisherId: publisher.id, // shiftId: shift.id, publisher: { connect: { id: publisher.id, }, }, shift: { connect: { id: shift.id, }, }, isWithTransport: isWithTransport, }, }); //ToDo: fix findPublisherAvailability and creation of availabilities // check if there is an availability for this publisher on this date, and if not, create one //ToDo: check if that works // const availability = await data.findPublisherAvailability(publisher.id, start); // if (!availability && createAvailabilities) { // const dayofWeek = common.getDayOfWeekNameEnEnumForDate(date); // const availability = await prisma.availability.create({ // data: { // publisherId: publisher.id, // //date: date, // dayofweek: dayofWeek, // //weekOfMonth: common.getWeekOfMonth(date), // startTime: start, // endTime: end, // name: `от предишен график, ${publisher.firstName} ${publisher.lastName}`, // isFromPreviousAssignment: true, // isWithTransportIn: isWithTransport && event.shiftNr == 1, // isWithTransportOut: isWithTransport && event.shiftNr > 1, // }, // }); // console.log(`Created SYSTEM availability with ID ${availability.id} for date '${date.toDateString()}' and publisher '${publisher.firstName} ${publisher.lastName}'`); // } console.log(`Created assignment with ID ${assignment.id} for date '${date.toDateString()}' and location '${event.placeOfEvent}'. publisher: ${publisher.firstName} ${publisher.lastName}}`); } } } } catch (e) { console.log(e); } } console.log("Done"); } //We used GPT to generate the file so far - day by day, by copy-pasting it in the chat. See PROMPTS.md exports.ReadDocxFileForMonth = async function (filePath, buffer, month, year, progressCallback, createAvailabilities) { try { const JSZip = require("jszip"); const zip = new JSZip(); //if filepath is not null read it. otherwise use buffer if (filePath != null) { buffer = await fs.readFileSync(filePath); } const zipFile = await zip.loadAsync(buffer); const documentXml = await zipFile.file("word/document.xml").async("string"); const xml2js = require("xml2js"); const parser = new xml2js.Parser({ explicitArray: false, ignoreAttrs: true, }); const json = await parser.parseStringPromise(documentXml); //const tableData = parsedXml['w:document']['w:body']['w:tbl']['w:tr'][1]['w:tc']['w:p']['w:r']['w:t']; // addParentReferences(json); // const xmlJs = require('xml-js'); // const jsonstring = xmlJs.xml2json(json, { compact: true }); const cleanedJsonObj = common.jsonRemoveEmptyNodes(json); //let filename = `График source ${year}-${month}.json`; //fs.writeFileSync("./content/temp/" + filename, JSON.stringify(cleanedJsonObj)); //initial json source for previous shifts const extractedData = extractData(cleanedJsonObj, month, year); //console.log(extractedData); //modify the file // try { // let filename = `График ${year}-${month}.json`; // fs.writeFileSync("./content/temp/" + filename, JSON.stringify(extractedData)); // } catch (e) { // console.log(e); // } await exports.processEvents(extractedData, year, month, progressCallback, createAvailabilities); } catch (err) { console.log(err); } }; const weekNames = [ "Понеделник", "Вторник", "Сряда", "Четвъртък", "Петък", "Събота", "Неделя", ]; function findWeekNameNodes(obj, path = []) { let result = []; if (Array.isArray(obj)) { for (let i = 0; i < obj.length; i++) { const newPath = path.slice(); newPath.push(i); result = result.concat(findWeekNameNodes(obj[i], newPath)); } } else if (typeof obj === "object") { for (const key in obj) { const newPath = path.slice(); newPath.push(key); result = result.concat(findWeekNameNodes(obj[key], newPath)); } } else if (typeof obj === "string") { const matches = obj.match(/(\S+) (\d+)/); if (matches && weekNames.includes(matches[1])) { result.push({ weekName: matches[1], dayOfMonth: matches[2], path }); } } return result; } function findShifts(node) { if (node === null || typeof node !== "object") return null; if (node.hasOwnProperty("w:tbl")) { return node["w:tbl"]; } else { return findShifts(node._parent); } } function extractData(parsedJson, month, year) { const weekNameNodes = findWeekNameNodes(parsedJson); const data = []; let lastDay = 0; let monthOverflow = false; for (const node of weekNameNodes) { const { weekName, dayOfMonth, path } = node; let currentNode = parsedJson; let baseNode = null; let parentNode = null; const dom = parseInt(dayOfMonth); if (lastDay > dom) { monthOverflow = true; } lastDay = dom; let date = new Date(year, month - (monthOverflow ? 0 : 1), dom); for (const key of path) { if (currentNode[key] && currentNode[key]["w:tc"]) { parentNode = currentNode; baseNode = currentNode[key]; } currentNode = currentNode[key]; } // for (const key of path) { // parentNode = currentNode; // currentNode = currentNode[key]; // } console.log("Processing " + weekName + " " + dayOfMonth + " " + CON.monthNamesBG[date.getMonth()]); const dailyData = extractDataForDay(parentNode, weekName, date); dailyData.forEach((item) => { if (!data.some((existingItem) => existingItem.date === item.date && existingItem.shiftNr === item.shiftNr && existingItem.dayOfMonth === item.dayOfMonth )) { data.push(item); } }); } return data; } function extractDataForDay(weeknameNode, weekName, date) { let result = []; weekName = weekName.split(" ")[0]; let placeOfEvent = weeknameNode[0]["w:tc"]["w:p"][1]["w:r"]["w:t"] ?? weeknameNode[0]["w:tc"]["w:p"][1]["w:r"]["0"]["w:t"]; let names = []; let shiftNr = 0; let tbl = weeknameNode; for (const trKey in tbl) { try { const weekNameNodes = findWeekNameNodes(tbl[trKey]); if (weekNameNodes.length > 0) { if (names && names.length > 0) { shiftNr = 0; } dayOfMonth = date.getDate(); //tbl[trKey]["w:tc"]["w:p"][0]["w:r"]["w:t"].match(/(\d+)/)[1]; weekName = weekName; // tbl[trKey]["w:tc"]["w:p"][0]["w:r"]["w:t"].match(/(\S+) (\d+)/)[1]; placeOfEvent = placeOfEvent.trim();//tbl[trKey]["w:tc"]["w:p"][1]["w:r"]["w:t"]; continue; } const tr = tbl[trKey]; console.log("Processin row: " + JSON.stringify(tr)); let time = tr["w:tc"]?.[1]?.["w:p"]?.["w:r"]?.["w:t"] ?? tr["w:tc"]?.[1]?.["w:p"]?.[0]?.["w:r"]?.["w:t"]; let transport = tr["w:tc"]?.[3]?.["w:p"]?.["w:r"]?.[1]?.["w:t"]; let namesPath = ["w:tc", 2, "w:p"]; try { names = [getTextContent(safelyAccess(tr, namesPath))].join("").trim(); } catch (e) { console.log("try to parse names:" + names + "; " + e + " " + JSON.stringify(tr["w:tc"] + " " + trKey) + e.stack); } //if starts with "Докарва" or empty - try the first cell instead of second if (names.startsWith("Докарва") || names.startsWith("Прибира ") || names === "") { transport = names; time = getTextContent(safelyAccess(tr, ["w:tc", 0, "w:p"])); namesPath = ["w:tc", 1, "w:p"]; try { names = [getTextContent(safelyAccess(tr, namesPath))].join("").trim(); } catch (e) { console.log("try to parse names:" + names + "; " + e + " " + JSON.stringify(tr["w:tc"] + " " + trKey) + e.stack); } } names = names.split(",").map((name) => name.trim()).filter((name) => name !== ""); shiftNr++; result.push({ date, dayOfWeek: weekName, dayOfMonth: date.getDate(), placeOfEvent, shiftNr, time, names, transport, }); } catch (e) { console.log("failed extracting data from node " + trKey + ": " + e + ": " + e.stack); } } return result; } function safelyAccess(obj, path) { return path.reduce((acc, key) => (acc && key in acc) ? acc[key] : undefined, obj); } const getTextContent = (obj) => { let textContent = ''; const traverse = (node) => { if (typeof node === 'string') { textContent += node; } else if (Array.isArray(node)) { node.forEach((child) => traverse(child)); } else if (typeof node === 'object') { Object.values(node).forEach((child) => traverse(child)); } }; traverse(obj); return textContent; }; // ImportSchedule("./content/sources/march.json", 3); // function GenerateFlatJsonFile() { let data = JSON.parse(fs.readFileSync("./content/sources/test.json", "utf8")); let data_flat = transformJsonToFlat(data, 3); fs.writeFileSync( "./content/sources/march_flat.json", JSON.stringify(data_flat) ); } function transformJsonToFlat(inputJson, month) { const output = []; let dayNr = 0; inputJson.events.forEach((event) => { const date = new Date(2023, month - 1, event.dayOfMonth + 1); dayNr++; let shiftNr = 1; event.shifts.forEach((shift) => { const shiftNames = shift.names.split(",").map((name) => name.trim()); // if (shift.transport !== null) { // shiftNames.push(shift.transport); // } output.push({ date: common.getISODateOnly(date), dayOfWeek: event.dayOfWeek, dayNr, shiftNr: shiftNr++, time: shift.time, names: shiftNames, transport: shift.transport, }); }); }); return output; }