From 902f85484c7b869535ac93e1d0ecd88a717e31a5 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Wed, 23 Jul 2025 12:11:13 -0500 Subject: [PATCH 01/12] Update build.sbt to read TortoiseVersion version from ENV with fallback --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 91feaf2d5..eb3ee022c 100644 --- a/build.sbt +++ b/build.sbt @@ -13,7 +13,7 @@ import scala.sys.process.{ Process, ProcessLogger } name := "Galapagos" version := "1.0-SNAPSHOT" -val tortoiseVersion = "1.0-2f7bb74" +val tortoiseVersion = sys.env.getOrElse("TORTOISE_VERSION", "1.0-2f7bb74") resolvers ++= Seq( "tortoise" at "https://dl.cloudsmith.io/public/netlogo/tortoise/maven/" From 382164e06f7ea2cb6058baa48f533196f2525a12 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Wed, 23 Jul 2025 16:08:36 -0500 Subject: [PATCH 02/12] Extensions: Add `extension` keyword --- app/assets/javascripts/keywords.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/javascripts/keywords.coffee b/app/assets/javascripts/keywords.coffee index c2cff6da8..64e0549db 100644 --- a/app/assets/javascripts/keywords.coffee +++ b/app/assets/javascripts/keywords.coffee @@ -10,6 +10,7 @@ directives = [ 'DIRECTED-LINK-BREED', 'UNDIRECTED-LINK-BREED', 'EXTENSIONS', + 'EXTENSION', '__INCLUDES' ] From 0adcf2167347e6ee9eec7c204a03d9c6cccc9028 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Wed, 23 Jul 2025 16:10:44 -0500 Subject: [PATCH 03/12] Extensions: Add NLWExtensionManager extension loader to Galapagos --- .../beak/nlw-extensions-manager.coffee | 94 +++++++++++++++++++ app/assets/javascripts/beak/tortoise.coffee | 21 +++++ 2 files changed, 115 insertions(+) create mode 100644 app/assets/javascripts/beak/nlw-extensions-manager.coffee diff --git a/app/assets/javascripts/beak/nlw-extensions-manager.coffee b/app/assets/javascripts/beak/nlw-extensions-manager.coffee new file mode 100644 index 000000000..ffb21fb4d --- /dev/null +++ b/app/assets/javascripts/beak/nlw-extensions-manager.coffee @@ -0,0 +1,94 @@ + + + +# This is a singleton class for managing NetLogo Web (NLW) extensions. +# There is a few, unfortunately, global objects that we have to depend on: +# 1. Extensions. -- Managed by Tortoise Engine +# 2. nlwExtensionsManager -- Managed by Galapagos +class NLWExtensionManager + @instance: null + constructor: (compiler) -> + if NLWExtensionManager.instance? + return NLWExtensionManager.instance + + NLWExtensionManager.instance = this + window.nlwExtensionManager = NLWExtensionManager.instance + + @compiler = compiler + @urlRepo = {} + + loadURLExtensions: (source) -> + urlRepo = @urlRepo + extensions = @compiler.listExtensions(source) + url_extensions = Object.fromEntries (await Promise.all(extensions + .filter((ext) -> ext.url != null) + .map((ext) -> + {name, url} = ext + baseName = NLWExtensionManager.getBaseNameFromURL(url) + primURL = NLWExtensionManager.getPrimitiveJSONSrc(url) + if urlRepo[baseName]? + # If the extension is already loaded, just return it + return [baseName, urlRepo[baseName]] + # We want to get a lazy loader for the extension, + # and fetch the primitives JSON file before we + # trigger the recompilation. + return Promise.resolve().then(() -> + prims = await NLWExtensionManager.fetchPrimitives(primURL) + NLWExtensionManager.confirmNamesMatch(prims, name) + return [baseName, { + getExtension: () -> import(url), + prims + }] + ) + ) + )) + + Object.assign(@urlRepo, url_extensions) + NLWExtensionManager.updateGlobalExtensionsObject(url_extensions) + console.log("Loaded URL extensions:", url_extensions) + url_extensions + + # Helpers + @updateGlobalExtensionsObject: (url_extensions) -> + # Update the global Extensions object with the new URL extensions + if not window.Extensions? + window.Extensions = {} + for _, ext of url_extensions + key = ext.prims.name + if window.Extensions[key]? + console.warn("Extension '#{key}' already exists in global Extensions object.") + window.Extensions[key] = ext + console.log("NLW Extensions updated in global object:", window.Extensions) + + @confirmNamesMatch: (primitives, name) -> + # Check if the primitives JSON file name matches the extension name + if primitives?.name.toLowerCase() isnt name.toLowerCase() + console.warn("Primitives JSON file name '#{primitives.name.toLowerCase()}' does not match extension import name '#{name.toLowerCase()}'") + + @fetchPrimitives: (primURL) -> + try + response = await fetch(primURL) + if response.ok + return await response.json() + else + throw new Error("Failed to fetch primitives from #{primURL}: HTTP #{response.status}") + catch ex + console.error("Error fetching primitives from #{primURL}: #{ex.message}") + return null + + @getBaseNameFromURL: (url) -> + # Remove the file extension from the URL + # to get the base name. By convention, the + # primitives JSON file is named after the extension + # base name, with a .json extension. + # e.g. "my-extension.js" becomes "my-extension.json" + url.split('.').slice(0, -1).join('.') + + @getPrimitiveJSONSrc: (url) -> + # Get the base name from the URL and append '.json' + baseName = NLWExtensionManager.getBaseNameFromURL(url) + return "#{baseName}.json" + + +# Exports +export default NLWExtensionManager \ No newline at end of file diff --git a/app/assets/javascripts/beak/tortoise.coffee b/app/assets/javascripts/beak/tortoise.coffee index 318e7b05e..45e89f16e 100644 --- a/app/assets/javascripts/beak/tortoise.coffee +++ b/app/assets/javascripts/beak/tortoise.coffee @@ -2,6 +2,7 @@ import SessionLite from "./session-lite.js" import { DiskSource, NewSource, UrlSource, ScriptSource } from "./nlogo-source.js" import { toNetLogoWebMarkdown, nlogoToSections, sectionsToNlogo } from "./tortoise-utils.js" import { createNotifier, listenerEvents } from "../notifications/listener-events.js" +import NLWExtensionManager from "./nlw-extensions-manager.js" # (String|DomElement, BrowserCompiler, Array[Rewriter], Array[Listener], ModelResult, # Boolean, String, String, NlogoSource, Boolean) => SessionLite @@ -28,6 +29,7 @@ newSession = (container, compiler, rewriters, listeners, modelResult, ) session + # (() => Unit) => Unit startLoading = (process) -> document.querySelector("#loading-overlay").style.display = "" @@ -88,6 +90,22 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, getWorkInProgress, callback, rewriters, listeners, extraWidgets = []) -> compiler = new BrowserCompiler() + extensionManager = new NLWExtensionManager(compiler) + + parser = new DOMParser(); + xmlDoc = parser.parseFromString(nlogoxSource.nlogo, "text/xml"); + errorNode = xmlDoc.querySelector("parsererror") + if errorNode + throw new Error("Invalid Nlogo XML: " + errorNode.textContent) + + codeElement = xmlDoc.querySelector("code") + codeText = codeElement.innerHTML + code = if not codeText.startsWith("".length)) + + await extensionManager.loadURLExtensions(code) notifyListeners = createNotifier(listenerEvents, listeners) @@ -202,6 +220,9 @@ fromNlogoSync = (nlogoSource, container, locale, isUndoReversion, getWorkInProgress, callback, rewriters, listeners, extraWidgets = []) -> compiler = new BrowserCompiler() + extensionManager = new NLWExtensionManager(compiler) + + await extensionManager.loadURLExtensions(nlogoSource) notifyListeners = createNotifier(listenerEvents, listeners) From 01c84afd4c143fb73d999e93645dd0475b0669cc Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Wed, 23 Jul 2025 16:14:32 -0500 Subject: [PATCH 04/12] Extensions: Rename NLWExtensionManager to NLWExtensionsLoader --- ...er.coffee => nlw-extensions-loader.coffee} | 27 +++++++++---------- app/assets/javascripts/beak/tortoise.coffee | 10 +++---- 2 files changed, 18 insertions(+), 19 deletions(-) rename app/assets/javascripts/beak/{nlw-extensions-manager.coffee => nlw-extensions-loader.coffee} (80%) diff --git a/app/assets/javascripts/beak/nlw-extensions-manager.coffee b/app/assets/javascripts/beak/nlw-extensions-loader.coffee similarity index 80% rename from app/assets/javascripts/beak/nlw-extensions-manager.coffee rename to app/assets/javascripts/beak/nlw-extensions-loader.coffee index ffb21fb4d..13f61c6db 100644 --- a/app/assets/javascripts/beak/nlw-extensions-manager.coffee +++ b/app/assets/javascripts/beak/nlw-extensions-loader.coffee @@ -4,15 +4,15 @@ # This is a singleton class for managing NetLogo Web (NLW) extensions. # There is a few, unfortunately, global objects that we have to depend on: # 1. Extensions. -- Managed by Tortoise Engine -# 2. nlwExtensionsManager -- Managed by Galapagos -class NLWExtensionManager +# 2. nlwExtensionsLoader -- Managed by Galapagos +class NLWExtensionsLoader @instance: null constructor: (compiler) -> - if NLWExtensionManager.instance? - return NLWExtensionManager.instance + if NLWExtensionsLoader.instance? + return NLWExtensionsLoader.instance - NLWExtensionManager.instance = this - window.nlwExtensionManager = NLWExtensionManager.instance + NLWExtensionsLoader.instance = this + window.NLWExtensionsLoader = NLWExtensionsLoader.instance @compiler = compiler @urlRepo = {} @@ -24,8 +24,8 @@ class NLWExtensionManager .filter((ext) -> ext.url != null) .map((ext) -> {name, url} = ext - baseName = NLWExtensionManager.getBaseNameFromURL(url) - primURL = NLWExtensionManager.getPrimitiveJSONSrc(url) + baseName = NLWExtensionsLoader.getBaseNameFromURL(url) + primURL = NLWExtensionsLoader.getPrimitiveJSONSrc(url) if urlRepo[baseName]? # If the extension is already loaded, just return it return [baseName, urlRepo[baseName]] @@ -33,8 +33,8 @@ class NLWExtensionManager # and fetch the primitives JSON file before we # trigger the recompilation. return Promise.resolve().then(() -> - prims = await NLWExtensionManager.fetchPrimitives(primURL) - NLWExtensionManager.confirmNamesMatch(prims, name) + prims = await NLWExtensionsLoader.fetchPrimitives(primURL) + NLWExtensionsLoader.confirmNamesMatch(prims, name) return [baseName, { getExtension: () -> import(url), prims @@ -44,8 +44,7 @@ class NLWExtensionManager )) Object.assign(@urlRepo, url_extensions) - NLWExtensionManager.updateGlobalExtensionsObject(url_extensions) - console.log("Loaded URL extensions:", url_extensions) + NLWExtensionsLoader.updateGlobalExtensionsObject(url_extensions) url_extensions # Helpers @@ -86,9 +85,9 @@ class NLWExtensionManager @getPrimitiveJSONSrc: (url) -> # Get the base name from the URL and append '.json' - baseName = NLWExtensionManager.getBaseNameFromURL(url) + baseName = NLWExtensionsLoader.getBaseNameFromURL(url) return "#{baseName}.json" # Exports -export default NLWExtensionManager \ No newline at end of file +export default NLWExtensionsLoader \ No newline at end of file diff --git a/app/assets/javascripts/beak/tortoise.coffee b/app/assets/javascripts/beak/tortoise.coffee index 45e89f16e..dd24af59b 100644 --- a/app/assets/javascripts/beak/tortoise.coffee +++ b/app/assets/javascripts/beak/tortoise.coffee @@ -2,7 +2,7 @@ import SessionLite from "./session-lite.js" import { DiskSource, NewSource, UrlSource, ScriptSource } from "./nlogo-source.js" import { toNetLogoWebMarkdown, nlogoToSections, sectionsToNlogo } from "./tortoise-utils.js" import { createNotifier, listenerEvents } from "../notifications/listener-events.js" -import NLWExtensionManager from "./nlw-extensions-manager.js" +import NLWExtensionsLoader from "./nlw-extensions-loader.js" # (String|DomElement, BrowserCompiler, Array[Rewriter], Array[Listener], ModelResult, # Boolean, String, String, NlogoSource, Boolean) => SessionLite @@ -90,7 +90,7 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, getWorkInProgress, callback, rewriters, listeners, extraWidgets = []) -> compiler = new BrowserCompiler() - extensionManager = new NLWExtensionManager(compiler) + extensionsLoader = new NLWExtensionsLoader(compiler) parser = new DOMParser(); xmlDoc = parser.parseFromString(nlogoxSource.nlogo, "text/xml"); @@ -105,7 +105,7 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, else codeText.slice("".length)) - await extensionManager.loadURLExtensions(code) + await extensionsLoader.loadURLExtensions(code) notifyListeners = createNotifier(listenerEvents, listeners) @@ -220,9 +220,9 @@ fromNlogoSync = (nlogoSource, container, locale, isUndoReversion, getWorkInProgress, callback, rewriters, listeners, extraWidgets = []) -> compiler = new BrowserCompiler() - extensionManager = new NLWExtensionManager(compiler) + extensionsLoader = new NLWExtensionsLoader(compiler) - await extensionManager.loadURLExtensions(nlogoSource) + await extensionsLoader.loadURLExtensions(nlogoSource) notifyListeners = createNotifier(listenerEvents, listeners) From 568a080eecce4f95b7020eb8e1ea739699fc8a88 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Thu, 24 Jul 2025 13:58:16 -0500 Subject: [PATCH 05/12] Extensions: Update NLWExtensionsLoader public API and some logic --- .../beak/nlw-extensions-loader.coffee | 57 +++++++++++++++---- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/app/assets/javascripts/beak/nlw-extensions-loader.coffee b/app/assets/javascripts/beak/nlw-extensions-loader.coffee index 13f61c6db..afd56491f 100644 --- a/app/assets/javascripts/beak/nlw-extensions-loader.coffee +++ b/app/assets/javascripts/beak/nlw-extensions-loader.coffee @@ -4,7 +4,7 @@ # This is a singleton class for managing NetLogo Web (NLW) extensions. # There is a few, unfortunately, global objects that we have to depend on: # 1. Extensions. -- Managed by Tortoise Engine -# 2. nlwExtensionsLoader -- Managed by Galapagos +# 2. URLExtensionsRepo -- Managed by Galapagos class NLWExtensionsLoader @instance: null constructor: (compiler) -> @@ -33,10 +33,11 @@ class NLWExtensionsLoader # and fetch the primitives JSON file before we # trigger the recompilation. return Promise.resolve().then(() -> + extensionModule = await NLWExtensionsLoader.getModuleFromURL(url) prims = await NLWExtensionsLoader.fetchPrimitives(primURL) NLWExtensionsLoader.confirmNamesMatch(prims, name) return [baseName, { - getExtension: () -> import(url), + extensionModule, prims }] ) @@ -47,17 +48,53 @@ class NLWExtensionsLoader NLWExtensionsLoader.updateGlobalExtensionsObject(url_extensions) url_extensions + # Public API + getPrimitivesFromURL: (url) -> + # Get the primitives JSON file from the URL, if it exists + url = @removeURLProtocol(url) + baseName = NLWExtensionsLoader.getBaseNameFromURL(url) + if @urlRepo[baseName]? + return @urlRepo[baseName].prims + else + return null + + getExtensionModuleFromURL: (url) -> + # Get the extension module from the URL, if it exists + url = @removeURLProtocol(url) + baseName = NLWExtensionsLoader.getBaseNameFromURL(url) + if @urlRepo[baseName]? + return @urlRepo[baseName].extensionModule + else + return null + + appendURLProtocol: (url) -> + return "url://#{url}" + + removeURLProtocol: (name) -> + # Remove the "url://" protocol from the name + if name.startsWith("url://") + return name.slice("url://".length) + else + return name + + isURL: (name) -> + return name.startsWith("url://") + # Helpers + @getModuleFromURL: (url) -> + # Get the module from the URL, if it exists + extensionImport = await import(url) + extensionKeys = Object.keys(extensionImport) + if extensionKeys.length > 0 + return extensionImport[extensionKeys[0]] + else + throw new Error("Extension module at #{url} does not export anything.") + @updateGlobalExtensionsObject: (url_extensions) -> # Update the global Extensions object with the new URL extensions - if not window.Extensions? - window.Extensions = {} - for _, ext of url_extensions - key = ext.prims.name - if window.Extensions[key]? - console.warn("Extension '#{key}' already exists in global Extensions object.") - window.Extensions[key] = ext - console.log("NLW Extensions updated in global object:", window.Extensions) + if not window.URLExtensionsRepo? + window.URLExtensionsRepo = {} + Object.assign(window.URLExtensionsRepo, url_extensions) @confirmNamesMatch: (primitives, name) -> # Check if the primitives JSON file name matches the extension name From e31da785990c7a84a0278b5162827c1ab30ad369 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Thu, 24 Jul 2025 13:58:43 -0500 Subject: [PATCH 06/12] Extensions: Extract code from NLogo file using a dedicated class for abstraction and integrate loadURLExtensions --- app/assets/javascripts/beak/nlogo-file.coffee | 37 +++++++++++++++++++ app/assets/javascripts/beak/tortoise.coffee | 36 +++++++++--------- 2 files changed, 55 insertions(+), 18 deletions(-) create mode 100644 app/assets/javascripts/beak/nlogo-file.coffee diff --git a/app/assets/javascripts/beak/nlogo-file.coffee b/app/assets/javascripts/beak/nlogo-file.coffee new file mode 100644 index 000000000..fdd096f48 --- /dev/null +++ b/app/assets/javascripts/beak/nlogo-file.coffee @@ -0,0 +1,37 @@ +# +# NlogoFile class is a utility class to handle NetLogo files. +# This is not a full implementation. Expand as necessary. +# +class AbstractNlogoFile + constructor: (source) -> + @source = source + + getSource: -> + return @source + + getCode: -> + return 'dummy' + +class NlogoXFile extends AbstractNlogoFile + constructor: (source) -> + super(source) + parser = new DOMParser(); + @doc = parser.parseFromString(source, "text/xml"); + errorNode = @doc.querySelector("parsererror") + if errorNode + throw new Error("Invalid Nlogo XML: " + errorNode.textContent) + + getSource: -> + return @source + + getCode: -> + codeElement = @doc.querySelector("code") + codeText = codeElement.innerHTML + code = if not codeText.startsWith("".length)) + + return code + +export { NlogoXFile } \ No newline at end of file diff --git a/app/assets/javascripts/beak/tortoise.coffee b/app/assets/javascripts/beak/tortoise.coffee index dd24af59b..a3b67399e 100644 --- a/app/assets/javascripts/beak/tortoise.coffee +++ b/app/assets/javascripts/beak/tortoise.coffee @@ -3,6 +3,7 @@ import { DiskSource, NewSource, UrlSource, ScriptSource } from "./nlogo-source.j import { toNetLogoWebMarkdown, nlogoToSections, sectionsToNlogo } from "./tortoise-utils.js" import { createNotifier, listenerEvents } from "../notifications/listener-events.js" import NLWExtensionsLoader from "./nlw-extensions-loader.js" +import { NlogoXFile } from "./nlogo-file.js" # (String|DomElement, BrowserCompiler, Array[Rewriter], Array[Listener], ModelResult, # Boolean, String, String, NlogoSource, Boolean) => SessionLite @@ -92,21 +93,6 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, compiler = new BrowserCompiler() extensionsLoader = new NLWExtensionsLoader(compiler) - parser = new DOMParser(); - xmlDoc = parser.parseFromString(nlogoxSource.nlogo, "text/xml"); - errorNode = xmlDoc.querySelector("parsererror") - if errorNode - throw new Error("Invalid Nlogo XML: " + errorNode.textContent) - - codeElement = xmlDoc.querySelector("code") - codeText = codeElement.innerHTML - code = if not codeText.startsWith("".length)) - - await extensionsLoader.loadURLExtensions(code) - notifyListeners = createNotifier(listenerEvents, listeners) startingNlogoXML = nlogoxSource.nlogo @@ -124,10 +110,24 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, extrasReducer = (extras, rw) -> if rw.getExtraCommands? then extras.concat(rw.getExtraCommands()) else extras extraCommands = rewriters.reduce(extrasReducer, []) + file = new NlogoXFile(rewrittenNlogoXML) + code = file.getCode() + + compileSuccess = true + errors = [] + try + await extensionsLoader.loadURLExtensions(code) + catch e + compileSuccess = false + errors.push(e.message) + notifyListeners('compile-start', rewrittenNlogoXML, startingNlogoXML) result = compiler.fromNlogoXML(rewrittenNlogoXML, extraCommands, { code: "", widgets: extraWidgets }) + if not result.model.success + compileSuccess = false + errors = [...result.model.result, ...errors] - if result.model.success + if compileSuccess # result.code = if (startingNlogoXML is rewrittenNlogoXML) # result.code @@ -176,7 +176,7 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, type: 'failure' , source: 'compile-recoverable' , session: session - , errors: result.model.result + , errors }) result.commands.forEach( (c) -> if c.success then (new Function(c.result))() ) rewriters.forEach( (rw) -> rw.compileComplete?() ) @@ -186,7 +186,7 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, callback({ type: 'failure' , source: 'compile-fatal' - , errors: result.model.result + , errors }) return From 30ff3d668a28d68ed59e9d0aa15aa81fcabff140 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Thu, 24 Jul 2025 16:30:07 -0500 Subject: [PATCH 07/12] Extensions: Limit allowed extensions to .js files and add validate URL method --- .../beak/nlw-extensions-loader.coffee | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/beak/nlw-extensions-loader.coffee b/app/assets/javascripts/beak/nlw-extensions-loader.coffee index afd56491f..76c599259 100644 --- a/app/assets/javascripts/beak/nlw-extensions-loader.coffee +++ b/app/assets/javascripts/beak/nlw-extensions-loader.coffee @@ -7,6 +7,7 @@ # 2. URLExtensionsRepo -- Managed by Galapagos class NLWExtensionsLoader @instance: null + @allowedExtensions = ["js"] constructor: (compiler) -> if NLWExtensionsLoader.instance? return NLWExtensionsLoader.instance @@ -68,7 +69,10 @@ class NLWExtensionsLoader return null appendURLProtocol: (url) -> - return "url://#{url}" + if not url.startsWith("url://") + return "url://" + url + else + return url removeURLProtocol: (name) -> # Remove the "url://" protocol from the name @@ -80,6 +84,20 @@ class NLWExtensionsLoader isURL: (name) -> return name.startsWith("url://") + validateURL: (url) -> + url = @removeURLProtocol(url) + try + new URL(url) + fileExtension = url.split('.').pop() + if @allowedExtensions.includes(fileExtension) + return true + else + console.error("Invalid file extension: #{fileExtension}. Allowed extensions are: #{@allowedExtensions.join(', ')}") + return false + catch e + console.error("Invalid URL: #{url} - #{e.message}") + return false + # Helpers @getModuleFromURL: (url) -> # Get the module from the URL, if it exists From 626e799199107d4f910f738ef9f8aaaaa25469ab Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Thu, 24 Jul 2025 16:34:26 -0500 Subject: [PATCH 08/12] Extensions/Docs --- .../beak/nlw-extensions-loader.coffee | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/assets/javascripts/beak/nlw-extensions-loader.coffee b/app/assets/javascripts/beak/nlw-extensions-loader.coffee index 76c599259..61b983c65 100644 --- a/app/assets/javascripts/beak/nlw-extensions-loader.coffee +++ b/app/assets/javascripts/beak/nlw-extensions-loader.coffee @@ -5,6 +5,22 @@ # There is a few, unfortunately, global objects that we have to depend on: # 1. Extensions. -- Managed by Tortoise Engine # 2. URLExtensionsRepo -- Managed by Galapagos +# +# I tried to keep this file the main source of truth for NLW extensions as +# much as possible. This works hand-in-hand with the Tortoise Engine, particularly +# the NLWExtensionsManager class, which is responsible for managing the +# extensions in the Tortoise Engine. +# +# A lot of the functionality here is part of an API used by the Tortoise Engine. +# I could've implemented those in Scala, but I chose not to do so because +# I wanted to keep the NLW extensions management logic in JavaScript, where +# it is easier to work with URLs and dynamic imports. +# Also, because we cannot trigger asynchronous work in Scala and support in JavaScript +# easily––at least without the browser complaining about it––the `loadURLExtensions` +# had to be implemented in JavaScript anyways. +# +# - Omar Ibrahim, July 2025 +# class NLWExtensionsLoader @instance: null @allowedExtensions = ["js"] From cc1c244afe69102d452be94188d9c68a711ba802 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Mon, 28 Jul 2025 10:53:38 -0500 Subject: [PATCH 09/12] Extensions: add code section extractor utility --- app/assets/javascripts/beak/nlogo-file.coffee | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/beak/nlogo-file.coffee b/app/assets/javascripts/beak/nlogo-file.coffee index fdd096f48..0275f1a96 100644 --- a/app/assets/javascripts/beak/nlogo-file.coffee +++ b/app/assets/javascripts/beak/nlogo-file.coffee @@ -12,6 +12,22 @@ class AbstractNlogoFile getCode: -> return 'dummy' +class NlogoFile extends AbstractNlogoFile + constructor: (source) -> + super(source) + @delimiter = "@#$#@#$#@" + + getSource: -> + return @source + + getCode: -> + # Extract the code from the source using the delimiter + parts = @source.split(@delimiter) + if parts.length > 1 + return parts[0].trim() + else + return @source.trim() + class NlogoXFile extends AbstractNlogoFile constructor: (source) -> super(source) @@ -34,4 +50,4 @@ class NlogoXFile extends AbstractNlogoFile return code -export { NlogoXFile } \ No newline at end of file +export { NlogoXFile, NlogoFile } \ No newline at end of file From f03623839c2ddc2ca25f8026833a379cbe801889 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Mon, 28 Jul 2025 10:54:59 -0500 Subject: [PATCH 10/12] Extensions: Introduce normalized URLs as repo keys for NLWExtensionsLoader --- .../beak/nlw-extensions-loader.coffee | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/app/assets/javascripts/beak/nlw-extensions-loader.coffee b/app/assets/javascripts/beak/nlw-extensions-loader.coffee index 61b983c65..67ed1a383 100644 --- a/app/assets/javascripts/beak/nlw-extensions-loader.coffee +++ b/app/assets/javascripts/beak/nlw-extensions-loader.coffee @@ -20,7 +20,7 @@ # had to be implemented in JavaScript anyways. # # - Omar Ibrahim, July 2025 -# +# class NLWExtensionsLoader @instance: null @allowedExtensions = ["js"] @@ -38,14 +38,15 @@ class NLWExtensionsLoader urlRepo = @urlRepo extensions = @compiler.listExtensions(source) url_extensions = Object.fromEntries (await Promise.all(extensions - .filter((ext) -> ext.url != null) - .map((ext) -> + .filter((ext) -> ext.url != null) + .map((ext) => {name, url} = ext + url = @normalizeURL(@removeURLProtocol(url)) baseName = NLWExtensionsLoader.getBaseNameFromURL(url) primURL = NLWExtensionsLoader.getPrimitiveJSONSrc(url) - if urlRepo[baseName]? + if urlRepo[url]? # If the extension is already loaded, just return it - return [baseName, urlRepo[baseName]] + return [url, urlRepo[url]] # We want to get a lazy loader for the extension, # and fetch the primitives JSON file before we # trigger the recompilation. @@ -53,7 +54,7 @@ class NLWExtensionsLoader extensionModule = await NLWExtensionsLoader.getModuleFromURL(url) prims = await NLWExtensionsLoader.fetchPrimitives(primURL) NLWExtensionsLoader.confirmNamesMatch(prims, name) - return [baseName, { + return [url, { extensionModule, prims }] @@ -68,19 +69,17 @@ class NLWExtensionsLoader # Public API getPrimitivesFromURL: (url) -> # Get the primitives JSON file from the URL, if it exists - url = @removeURLProtocol(url) - baseName = NLWExtensionsLoader.getBaseNameFromURL(url) - if @urlRepo[baseName]? - return @urlRepo[baseName].prims + url = @normalizeURL(url) + if @urlRepo[url]? + return @urlRepo[url].prims else return null getExtensionModuleFromURL: (url) -> # Get the extension module from the URL, if it exists - url = @removeURLProtocol(url) - baseName = NLWExtensionsLoader.getBaseNameFromURL(url) - if @urlRepo[baseName]? - return @urlRepo[baseName].extensionModule + url = @normalizeURL(url) + if @urlRepo[url]? + return @urlRepo[url].extensionModule else return null @@ -101,11 +100,10 @@ class NLWExtensionsLoader return name.startsWith("url://") validateURL: (url) -> - url = @removeURLProtocol(url) try - new URL(url) - fileExtension = url.split('.').pop() - if @allowedExtensions.includes(fileExtension) + url = @normalizeURL(url) + fileExtension = url.split('.').pop().toLowerCase() + if NLWExtensionsLoader.allowedExtensions.includes(fileExtension) return true else console.error("Invalid file extension: #{fileExtension}. Allowed extensions are: #{@allowedExtensions.join(', ')}") @@ -114,6 +112,18 @@ class NLWExtensionsLoader console.error("Invalid URL: #{url} - #{e.message}") return false + normalizeURL: (url) -> + # Remove Search Params and Hash from the URL + try + url = @removeURLProtocol(url) + uri = new URL(url) + uri.search = '' + uri.hash = '' + return uri.toString() + catch e + console.error("Error normalizing URL: #{url} - #{e.message}") + return url + # Helpers @getModuleFromURL: (url) -> # Get the module from the URL, if it exists From 1cc0ac67385026f258758cb3e8c073414c073890 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Mon, 28 Jul 2025 10:55:33 -0500 Subject: [PATCH 11/12] Extensions: Add support for extension load on re-compilation and with .nlogo files --- .../javascripts/beak/session-lite.coffee | 8 ++- app/assets/javascripts/beak/tortoise.coffee | 56 ++++++++++++++++--- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/app/assets/javascripts/beak/session-lite.coffee b/app/assets/javascripts/beak/session-lite.coffee index 194e9fffd..b3c034a88 100644 --- a/app/assets/javascripts/beak/session-lite.coffee +++ b/app/assets/javascripts/beak/session-lite.coffee @@ -40,7 +40,8 @@ class SessionLite # (Tortoise, Element|String, BrowserCompiler, Array[Rewriter], Array[Listener], Array[Widget], # String, String, Boolean, String, String, NlogoSource, String, Boolean) constructor: (@tortoise, container, @compiler, @rewriters, listeners, widgets, - code, info, isReadOnly, @locale, workInProgressState, @nlogoSource, modelJS, lastCompileFailed) -> + code, info, isReadOnly, @locale, workInProgressState, @nlogoSource, modelJS, lastCompileFailed, + @onBeforeRecompile) -> @hnw = new HNWSession( (() => @widgetController) , ((ps) => @compiler.compilePlots(ps))) @@ -231,7 +232,6 @@ class SessionLite if @widgetController.ractive.get('isEditing') and @hnw.isHNW() parent.postMessage({ type: "recompile" }, "*") else - code = @widgetController.code() oldWidgets = @widgetController.widgets() rewritten = @rewriteCode(code) @@ -251,6 +251,10 @@ class SessionLite @widgetController.ractive.fire('recompile-start', source, rewritten, code) try + # # Execute the onBeforeRecompile callbacks + for callback in @onBeforeRecompile + await callback(source, rewritten, code) + res = @compiler.fromModel(compileParams) if res.model.success diff --git a/app/assets/javascripts/beak/tortoise.coffee b/app/assets/javascripts/beak/tortoise.coffee index a3b67399e..957b49363 100644 --- a/app/assets/javascripts/beak/tortoise.coffee +++ b/app/assets/javascripts/beak/tortoise.coffee @@ -3,12 +3,12 @@ import { DiskSource, NewSource, UrlSource, ScriptSource } from "./nlogo-source.j import { toNetLogoWebMarkdown, nlogoToSections, sectionsToNlogo } from "./tortoise-utils.js" import { createNotifier, listenerEvents } from "../notifications/listener-events.js" import NLWExtensionsLoader from "./nlw-extensions-loader.js" -import { NlogoXFile } from "./nlogo-file.js" +import { NlogoXFile, NlogoFile } from "./nlogo-file.js" # (String|DomElement, BrowserCompiler, Array[Rewriter], Array[Listener], ModelResult, # Boolean, String, String, NlogoSource, Boolean) => SessionLite newSession = (container, compiler, rewriters, listeners, modelResult, - isReadOnly, locale, workInProgressState, nlogoSource, lastCompileFailed) -> + isReadOnly, locale, workInProgressState, nlogoSource, lastCompileFailed, onBeforeRecompile) -> { code, info, model: { result }, widgets: wiggies } = modelResult widgets = globalEval(wiggies) info = toNetLogoWebMarkdown(info) @@ -26,7 +26,8 @@ newSession = (container, compiler, rewriters, listeners, modelResult, , workInProgressState , nlogoSource , result - , lastCompileFailed + , lastCompileFailed, + onBeforeRecompile || [] ) session @@ -113,6 +114,8 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, file = new NlogoXFile(rewrittenNlogoXML) code = file.getCode() + notifyListeners('compile-start', rewrittenNlogoXML, startingNlogoXML) + compileSuccess = true errors = [] try @@ -121,12 +124,22 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, compileSuccess = false errors.push(e.message) - notifyListeners('compile-start', rewrittenNlogoXML, startingNlogoXML) result = compiler.fromNlogoXML(rewrittenNlogoXML, extraCommands, { code: "", widgets: extraWidgets }) if not result.model.success compileSuccess = false errors = [...result.model.result, ...errors] + + onBeforeRecompile = [ + (source, rewritten, code) -> + try + await extensionsLoader.loadURLExtensions(rewritten) + return true + catch e + return false + ] + + if compileSuccess # result.code = if (startingNlogoXML is rewrittenNlogoXML) @@ -145,6 +158,7 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, , workInProgressState , nlogoxSource , false + , onBeforeRecompile ) callback({ @@ -170,6 +184,7 @@ fromNlogoXMLSync = (nlogoxSource, container, locale, isUndoReversion, , workInProgressState , nlogoxSource , true + , onBeforeRecompile ) callback({ @@ -222,7 +237,7 @@ fromNlogoSync = (nlogoSource, container, locale, isUndoReversion, compiler = new BrowserCompiler() extensionsLoader = new NLWExtensionsLoader(compiler) - await extensionsLoader.loadURLExtensions(nlogoSource) + notifyListeners = createNotifier(listenerEvents, listeners) @@ -242,15 +257,40 @@ fromNlogoSync = (nlogoSource, container, locale, isUndoReversion, extraCommands = rewriters.reduce(extrasReducer, []) notifyListeners('compile-start', rewrittenNlogo, startingNlogo) + + file = new NlogoFile(rewrittenNlogo) + code = file.getCode() + + compileSuccess = true + errors = [] + try + await extensionsLoader.loadURLExtensions(code) + catch e + compileSuccess = false + errors.push(e.message) + result = compiler.fromNlogo(rewrittenNlogo, extraCommands, { code: "", widgets: extraWidgets }) + if not result.model.success + compileSuccess = false + errors = [...result.model.result, ...errors] - if result.model.success + onBeforeRecompile = [ + (source, rewritten, code) -> + try + await extensionsLoader.loadURLExtensions(rewritten) + return true + catch e + return false + ] + + if compileSuccess result.code = if (startingNlogo is rewrittenNlogo) result.code else nlogoToSections(startingNlogo)[0].slice(0, -1) - + + session = newSession( container , compiler @@ -262,6 +302,7 @@ fromNlogoSync = (nlogoSource, container, locale, isUndoReversion, , workInProgressState , nlogoSource , false + , onBeforeRecompile ) callback({ @@ -287,6 +328,7 @@ fromNlogoSync = (nlogoSource, container, locale, isUndoReversion, , workInProgressState , nlogoSource , true + , onBeforeRecompile ) callback({ From e5d1dd35e10dd68988309f3666aee4a3d720c095 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim Date: Mon, 28 Jul 2025 11:02:58 -0500 Subject: [PATCH 12/12] Extensions/Lint --- app/assets/javascripts/beak/nlogo-file.coffee | 8 +++--- .../beak/nlw-extensions-loader.coffee | 27 ++++++++++--------- app/assets/javascripts/beak/tortoise.coffee | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/app/assets/javascripts/beak/nlogo-file.coffee b/app/assets/javascripts/beak/nlogo-file.coffee index 0275f1a96..fad7f1748 100644 --- a/app/assets/javascripts/beak/nlogo-file.coffee +++ b/app/assets/javascripts/beak/nlogo-file.coffee @@ -31,14 +31,14 @@ class NlogoFile extends AbstractNlogoFile class NlogoXFile extends AbstractNlogoFile constructor: (source) -> super(source) - parser = new DOMParser(); - @doc = parser.parseFromString(source, "text/xml"); + parser = new DOMParser() + @doc = parser.parseFromString(source, "text/xml") errorNode = @doc.querySelector("parsererror") if errorNode throw new Error("Invalid Nlogo XML: " + errorNode.textContent) getSource: -> - return @source + return @source getCode: -> codeElement = @doc.querySelector("code") @@ -50,4 +50,4 @@ class NlogoXFile extends AbstractNlogoFile return code -export { NlogoXFile, NlogoFile } \ No newline at end of file +export { NlogoXFile, NlogoFile } diff --git a/app/assets/javascripts/beak/nlw-extensions-loader.coffee b/app/assets/javascripts/beak/nlw-extensions-loader.coffee index 67ed1a383..deed2940b 100644 --- a/app/assets/javascripts/beak/nlw-extensions-loader.coffee +++ b/app/assets/javascripts/beak/nlw-extensions-loader.coffee @@ -5,7 +5,7 @@ # There is a few, unfortunately, global objects that we have to depend on: # 1. Extensions. -- Managed by Tortoise Engine # 2. URLExtensionsRepo -- Managed by Galapagos -# +# # I tried to keep this file the main source of truth for NLW extensions as # much as possible. This works hand-in-hand with the Tortoise Engine, particularly # the NLWExtensionsManager class, which is responsible for managing the @@ -20,7 +20,7 @@ # had to be implemented in JavaScript anyways. # # - Omar Ibrahim, July 2025 -# +# class NLWExtensionsLoader @instance: null @allowedExtensions = ["js"] @@ -37,8 +37,8 @@ class NLWExtensionsLoader loadURLExtensions: (source) -> urlRepo = @urlRepo extensions = @compiler.listExtensions(source) - url_extensions = Object.fromEntries (await Promise.all(extensions - .filter((ext) -> ext.url != null) + url_extensions = Object.fromEntries(await Promise.all(extensions + .filter((ext) -> ext.url isnt null) .map((ext) => {name, url} = ext url = @normalizeURL(@removeURLProtocol(url)) @@ -48,8 +48,8 @@ class NLWExtensionsLoader # If the extension is already loaded, just return it return [url, urlRepo[url]] # We want to get a lazy loader for the extension, - # and fetch the primitives JSON file before we - # trigger the recompilation. + # and fetch the primitives JSON file before we + # trigger the recompilation. return Promise.resolve().then(() -> extensionModule = await NLWExtensionsLoader.getModuleFromURL(url) prims = await NLWExtensionsLoader.fetchPrimitives(primURL) @@ -106,8 +106,10 @@ class NLWExtensionsLoader if NLWExtensionsLoader.allowedExtensions.includes(fileExtension) return true else - console.error("Invalid file extension: #{fileExtension}. Allowed extensions are: #{@allowedExtensions.join(', ')}") - return false + console.error("Invalid file extension: #{fileExtension}. " + + "Allowed extensions are: #{@allowedExtensions.join(', ')}" + ) + return false catch e console.error("Invalid URL: #{url} - #{e.message}") return false @@ -133,7 +135,7 @@ class NLWExtensionsLoader return extensionImport[extensionKeys[0]] else throw new Error("Extension module at #{url} does not export anything.") - + @updateGlobalExtensionsObject: (url_extensions) -> # Update the global Extensions object with the new URL extensions if not window.URLExtensionsRepo? @@ -143,7 +145,8 @@ class NLWExtensionsLoader @confirmNamesMatch: (primitives, name) -> # Check if the primitives JSON file name matches the extension name if primitives?.name.toLowerCase() isnt name.toLowerCase() - console.warn("Primitives JSON file name '#{primitives.name.toLowerCase()}' does not match extension import name '#{name.toLowerCase()}'") + console.warn("Primitives JSON file name '#{primitives.name.toLowerCase()}' " + + "does not match extension import name '#{name.toLowerCase()}'") @fetchPrimitives: (primURL) -> try @@ -163,7 +166,7 @@ class NLWExtensionsLoader # base name, with a .json extension. # e.g. "my-extension.js" becomes "my-extension.json" url.split('.').slice(0, -1).join('.') - + @getPrimitiveJSONSrc: (url) -> # Get the base name from the URL and append '.json' baseName = NLWExtensionsLoader.getBaseNameFromURL(url) @@ -171,4 +174,4 @@ class NLWExtensionsLoader # Exports -export default NLWExtensionsLoader \ No newline at end of file +export default NLWExtensionsLoader diff --git a/app/assets/javascripts/beak/tortoise.coffee b/app/assets/javascripts/beak/tortoise.coffee index 957b49363..e165fdb0c 100644 --- a/app/assets/javascripts/beak/tortoise.coffee +++ b/app/assets/javascripts/beak/tortoise.coffee @@ -27,7 +27,7 @@ newSession = (container, compiler, rewriters, listeners, modelResult, , nlogoSource , result , lastCompileFailed, - onBeforeRecompile || [] + onBeforeRecompile or [] ) session