From 7d386e3c597154307f1cb9d271d5e79615044253 Mon Sep 17 00:00:00 2001 From: mz7-wiki Date: Fri, 5 Dec 2025 01:42:23 -0800 Subject: [PATCH 1/3] Optimize spiHelperArchiveCase (one-click archiving) Problem: The current implementation of spiHelperArchiveCase is very inefficient. It goes section by section, blanking and archiving each one, spending many seconds on each section. If a particular SPI case has, say, five closed reports that need to be archived, that means we have to generate no less than 10 edits to archive the case and spend the better part of a minute waiting for the script to finish. When SPI is backlogged, these delays quickly pile up. Solution: Instead of going section by section, we can more optimally go through and do a "one-click archive" by grabbing the case page text only once, figuring out which lines need to be archived and which lines should be kept, and then submitting just two edits: one to archive and one to blank the archived sections. Doing this is tremendously more efficient: instead of waiting a minute to archive an SPI case that has many sections, the processing time is now down to just a couple seconds or so. --- spihelper.js | 140 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 91 insertions(+), 49 deletions(-) diff --git a/spihelper.js b/spihelper.js index b7ed7a3..df114ba 100644 --- a/spihelper.js +++ b/spihelper.js @@ -250,6 +250,8 @@ const spiHelperAdminTemplates = [ const spiHelperCaseStatusRegex = /{{\s*SPI case status\s*\|?\s*(\S*?)\s*}}/i // Regex to match closed case statuses (close or closed) const spiHelperCaseClosedRegex = /^closed?$/i +// Regex to match the closed case statuses with the full template string (used for optimized one-click archiving) +const spiHelperCaseStatusClosedRegex = /{{\s*SPI case status\s*\|?\s*closed?\s*}}/i const spiHelperClerkStatusRegex = /{{(CURequest|awaitingadmin|clerk ?request|(?:self|requestand|cu)?endorse|inprogress|decline(?:-ip)?|moreinfo|relisted|onhold)}}/i @@ -1819,61 +1821,101 @@ async function spiHelperPostMergeCleanup (originalText) { */ async function spiHelperArchiveCase () { 'use strict' - let i = 0 - let previousRev = 0 - while (i < spiHelperCaseSections.length) { - const sectionId = spiHelperCaseSections[i].index - const sectionText = await spiHelperGetPageText(spiHelperPageName, false, - sectionId) - - const currentRev = await spiHelperGetPageRev(spiHelperPageName) - if (previousRev === currentRev && currentRev !== 0) { - // Our previous archive hasn't gone through yet, wait a bit and retry - await new Promise((resolve) => { - setTimeout(resolve, 100) - }) + // Get the entire page text + let pageText = await spiHelperGetPageText(spiHelperPageName, true) + + // Remove all {{SPI case status|close(d)}} templates from the page text + pageText.replace(spiHelperCaseStatusClosedRegex, '') + + let textToArchiveLines = [] + let textToKeepLines = [] + let sectionLines = [] + + // Now we will iterate line by line through pageText and decide if the line + // should be appended to sectionsToArchiveLines, which we will then append to the archive. + let inClosedSection = false + for (const line of pageText.split('\n')) { + if (spiHelperSectionRegex.test(line)) { + // We found the start of an SPI case section + if (inClosedSection) { + textToArchiveLines.push(sectionLines.join('\n')) + } else { + textToKeepLines.push(sectionLines.join('\n')) + } - // Re-grab the case sections list since the page may have updated - spiHelperCaseSections = await spiHelperGetInvestigationSectionIDs() - continue + // reset sectionLines to hold the current section's text + sectionLines = [] + + // start by assuming that this section is closed + inClosedSection = true } - previousRev = await spiHelperGetPageRev(spiHelperPageName) - i++ - const result = spiHelperCaseStatusRegex.exec(sectionText) - if (result === null) { - // Bail out - can't find the case status template in this section - continue + + const case_status = line.match(spiHelperCaseStatusRegex) + if (case_status && !spiHelperCaseClosedRegex.test(case_status[1])) { + // this section is not in fact closed + inClosedSection = false } - if (spiHelperCaseClosedRegex.test(result[1])) { - // A running concern with the SPI archives is whether they exceed the post-expand - // include size. Calculate what percent of that size the archive will be if we - // add the current page to it - if >1, we need to archive the archive - const postExpandPercent = - (await spiHelperGetPostExpandSize(spiHelperPageName, sectionId) + - await spiHelperGetPostExpandSize(spiHelperGetArchiveName())) / - spiHelperGetMaxPostExpandSize() - if (postExpandPercent >= 1) { - // We'd overflow the archive, so move it and then archive the current page - // Find the first empty archive page - let archiveId = 1 - while (await spiHelperGetPageText(spiHelperGetArchiveName() + '/' + archiveId, false) !== '') { - archiveId++ - } - const newArchiveName = spiHelperGetArchiveName() + '/' + archiveId - await spiHelperMovePage(spiHelperGetArchiveName(), newArchiveName, 'Moving archive to avoid exceeding post expand size limit', false, false) - await spiHelperEditPage(spiHelperGetArchiveName(), '', 'Removing redirect', false, 'nochange') - } - // Need an await here - if we have multiple sections archiving we don't want - // to stomp on each other - await spiHelperArchiveCaseSection(sectionId) - // need to re-fetch caseSections since the section numbering probably just changed, - // also reset our index - i = 0 - spiHelperCaseSections = await spiHelperGetInvestigationSectionIDs() + + sectionLines.push(line) + } + + if (inClosedSection) { + textToArchiveLines.push(sectionLines.join('\n')) + } else { + textToKeepLines.push(sectionLines.join('\n')) + } + + const textToArchive = textToArchiveLines.join('\n') + const textToKeep = textToKeepLines.join('\n') + + if (!textToArchive) { + // Nothing to archive + console.log("spihelper: nothing to archive here.") + return + } + + // A running concern with the SPI archives is whether they exceed the post-expand + // include size. Calculate what percent of that size the archive will be if we + // add the current page to it - if >1, we need to archive the archive + const postExpandPercent = + (await spiHelperGetPostExpandSize(spiHelperPageName) + + await spiHelperGetPostExpandSize(spiHelperGetArchiveName())) / + spiHelperGetMaxPostExpandSize() + if (postExpandPercent >= 1) { + // We'd overflow the archive, so move it and then archive the current page + // Find the first empty archive page + let archiveId = 1 + while (await spiHelperGetPageText(spiHelperGetArchiveName() + '/' + archiveId, false) !== '') { + archiveId++ } + const newArchiveName = spiHelperGetArchiveName() + '/' + archiveId + await spiHelperMovePage(spiHelperGetArchiveName(), newArchiveName, 'Moving archive to avoid exceeding post expand size limit', false, false) + await spiHelperEditPage(spiHelperGetArchiveName(), '', 'Removing redirect', false, 'nochange') } -} + // Update the archive + let archivetext = await spiHelperGetPageText(spiHelperGetArchiveName(), true) + if (!archivetext) { + archivetext = '__TOC__\n{{SPIarchive notice|1=' + spiHelperCaseName + '}}\n{{SPIpriorcases}}' + } else { + archivetext = archivetext.replace(/\s*{{SPIpriorcases}}/gi, '\n{{SPIpriorcases}}') // fmt fix whenever needed. + } + archivetext += '\n' + textToArchive + const archiveSuccess = await spiHelperEditPage(spiHelperGetArchiveName(), archivetext, + 'Archiving closed cases from [[' + spiHelperGetInterwikiPrefix() + spiHelperPageName + ']]', + false, spiHelperSettings.watchArchive, spiHelperSettings.watchArchiveExpiry) + + if (!archiveSuccess) { + $statusLine.addClass('spihelper-errortext').append('b').text('Failed to update archive, not removing section from case page') + return + } + + // Update case page to blank the sections we archived + await spiHelperEditPage(spiHelperPageName, textToKeep, 'Archiving closed cases to [[' + spiHelperGetInterwikiPrefix() + spiHelperGetArchiveName() + ']]', + false, spiHelperSettings.watchCase, spiHelperSettings.watchCaseExpiry, spiHelperStartingRevID, null) + // Update to the latest revision ID + spiHelperStartingRevID = await spiHelperGetPageRev(spiHelperPageName) +} /** * Archive a specific section of a case * From b5ae5286b1aaa34bcc75ccd973b874e9eaab3516 Mon Sep 17 00:00:00 2001 From: mz7-wiki Date: Fri, 5 Dec 2025 02:04:22 -0800 Subject: [PATCH 2/3] fix code comment in spiHelperArchiveCase --- spihelper.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spihelper.js b/spihelper.js index df114ba..3cc286a 100644 --- a/spihelper.js +++ b/spihelper.js @@ -1832,7 +1832,8 @@ async function spiHelperArchiveCase () { let sectionLines = [] // Now we will iterate line by line through pageText and decide if the line - // should be appended to sectionsToArchiveLines, which we will then append to the archive. + // should be appended to textToArchiveLines, which we will then append to the archive, + // or textToKeepLines, which will remain on the case page. let inClosedSection = false for (const line of pageText.split('\n')) { if (spiHelperSectionRegex.test(line)) { From ad7cefaf317d8ac52cad710aa03521297896d8d9 Mon Sep 17 00:00:00 2001 From: mz7-wiki Date: Sat, 6 Dec 2025 15:09:22 -0800 Subject: [PATCH 3/3] fix removal of {{SPI case status|close}} during one-click archiving Had a bug in my optimization which prevents the {{SPI case status|close}} string from being removed when archiving a case. I didn't notice this at first because there is some logic in the template which hides it from view when it is on an archive page. --- spihelper.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spihelper.js b/spihelper.js index 3cc286a..3a9508f 100644 --- a/spihelper.js +++ b/spihelper.js @@ -251,7 +251,7 @@ const spiHelperCaseStatusRegex = /{{\s*SPI case status\s*\|?\s*(\S*?)\s*}}/i // Regex to match closed case statuses (close or closed) const spiHelperCaseClosedRegex = /^closed?$/i // Regex to match the closed case statuses with the full template string (used for optimized one-click archiving) -const spiHelperCaseStatusClosedRegex = /{{\s*SPI case status\s*\|?\s*closed?\s*}}/i +const spiHelperCaseStatusClosedRegex = /{{\s*SPI case status\s*\|?\s*closed?\s*}}/gi const spiHelperClerkStatusRegex = /{{(CURequest|awaitingadmin|clerk ?request|(?:self|requestand|cu)?endorse|inprogress|decline(?:-ip)?|moreinfo|relisted|onhold)}}/i @@ -1825,7 +1825,7 @@ async function spiHelperArchiveCase () { let pageText = await spiHelperGetPageText(spiHelperPageName, true) // Remove all {{SPI case status|close(d)}} templates from the page text - pageText.replace(spiHelperCaseStatusClosedRegex, '') + pageText = pageText.replaceAll(spiHelperCaseStatusClosedRegex, '') let textToArchiveLines = [] let textToKeepLines = [] @@ -1903,7 +1903,7 @@ async function spiHelperArchiveCase () { } archivetext += '\n' + textToArchive const archiveSuccess = await spiHelperEditPage(spiHelperGetArchiveName(), archivetext, - 'Archiving closed cases from [[' + spiHelperGetInterwikiPrefix() + spiHelperPageName + ']]', + 'Archiving closed section(s) from [[' + spiHelperGetInterwikiPrefix() + spiHelperPageName + ']]', false, spiHelperSettings.watchArchive, spiHelperSettings.watchArchiveExpiry) if (!archiveSuccess) { @@ -1912,7 +1912,7 @@ async function spiHelperArchiveCase () { } // Update case page to blank the sections we archived - await spiHelperEditPage(spiHelperPageName, textToKeep, 'Archiving closed cases to [[' + spiHelperGetInterwikiPrefix() + spiHelperGetArchiveName() + ']]', + await spiHelperEditPage(spiHelperPageName, textToKeep, 'Archiving closed section(s) to [[' + spiHelperGetInterwikiPrefix() + spiHelperGetArchiveName() + ']]', false, spiHelperSettings.watchCase, spiHelperSettings.watchCaseExpiry, spiHelperStartingRevID, null) // Update to the latest revision ID spiHelperStartingRevID = await spiHelperGetPageRev(spiHelperPageName)