Support 'continue' & safer lua code emission (#1161)

* fix functions with more than 200 locals in lua
* completely replace hashtable usage with lua tables during compilation
* support continue statement

Author: Frotty

.circleci/config.yml

@@ -17,8 +17,7 @@ jobs:
     working_directory: ~/repo/de.peeeq.wurstscript
 
     environment:
-      # Customize the JVM maximum heap limit
-      JVM_OPTS: -Xmx3200m
+      GRADLE_OPTS: -Dorg.gradle.parallel=false -Dorg.gradle.workers.max=2
       TERM: dumb
 
     steps:
@@ -28,20 +27,18 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "build.gradle" }}
+            - v2-gradle-{{ checksum "build.gradle" }}-{{ checksum "gradle.properties" }}-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }}
             # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
-
-      - run: ./gradlew dependencies
+            - v2-gradle-
 
       - save_cache:
           paths:
             - ~/.gradle
-          key: v1-dependencies-{{ checksum "build.gradle" }}
-
-      # run tests
-      - run: ./gradlew test --info
+          key: v2-gradle-{{ checksum "build.gradle" }}-{{ checksum "gradle.properties" }}-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }}
 
-      # report tests results
-      - run: ./gradlew jacocoTestReport coveralls
+      # run tests and coverage in one invocation to avoid duplicate config/startup cost
+      - run:
+          name: Run tests and coverage
+          command: ./gradlew --no-daemon --stacktrace test jacocoTestReport coveralls
+          no_output_timeout: 30m
 

de.peeeq.wurstscript/gradle.properties

@@ -3,4 +3,5 @@ org.gradle.configuration-cache=true
 org.gradle.parallel=true
 org.gradle.daemon=true
 org.gradle.java.installations.auto-download=true
-org.gradle.java.installations.auto-detect=true
+org.gradle.java.installations.auto-detect=true
+org.gradle.jvmargs=-Xmx3g -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8

de.peeeq.wurstscript/parserspec/wurstscript.parseq

@@ -147,6 +147,7 @@ ControlflowStatement =
 	  CompoundStatement
 	| StmtReturn(@ignoreForEquality de.peeeq.wurstscript.parser.WPos source, OptExpr returnedObj)
 	| StmtExitwhen(@ignoreForEquality de.peeeq.wurstscript.parser.WPos source, Expr cond)
+	| StmtContinue(de.peeeq.wurstscript.parser.WPos source)
 
 CompoundStatement =
 	  StmtIf(@ignoreForEquality de.peeeq.wurstscript.parser.WPos source, Expr cond, WStatements thenBlock, WStatements elseBlock, boolean hasElse)
@@ -1073,4 +1074,4 @@ Annotation.getAnnotationType()
 
 Annotation.getAnnotationMessage()
     returns String
-    implemented by de.peeeq.wurstscript.attributes.Annotations.annotationMessage
+    implemented by de.peeeq.wurstscript.attributes.Annotations.annotationMessage

de.peeeq.wurstscript/src/main/antlr/de/peeeq/wurstscript/antlr/Wurst.g4

@@ -260,6 +260,7 @@ statement:
 		 | stmtSet (externalLambda|NL)
 		 | stmtReturn (externalLambda|NL)
 		 | stmtBreak NL
+		 | stmtContinue NL
 		 | stmtSkip NL
 		 | expr (externalLambda|NL)
 		 | stmtIf
@@ -435,6 +436,7 @@ forIteratorLoop:
 
 
 stmtBreak:'break';
+stmtContinue:'continue';
 stmtSkip:'skip';
 
 
@@ -461,6 +463,7 @@ WHILE: 'while';
 FOR: 'for';
 IN: 'in';
 BREAK: 'break';
+CONTINUE: 'continue';
 NEW: 'new';
 NULL: 'null';
 PACKAGE: 'package';

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/WurstCompilerJassImpl.java

@@ -28,6 +28,7 @@
 import de.peeeq.wurstscript.translation.imtojass.ImAttrType;
 import de.peeeq.wurstscript.translation.imtojass.ImToJassTranslator;
 import de.peeeq.wurstscript.translation.imtranslation.*;
+import de.peeeq.wurstscript.translation.lua.translation.RemoveGarbage;
 import de.peeeq.wurstscript.translation.lua.translation.LuaTranslator;
 import de.peeeq.wurstscript.types.TypesHelper;
 import de.peeeq.wurstscript.utils.LineOffsets;
@@ -937,7 +938,12 @@ public LuaCompilationUnit transformProgToLua() {
             printDebugImProg("./test-output/lua/im " + stage++ + "_afteroptimize.im");
             timeTaker.endPhase();
         }
-        beginPhase(13, "translate to lua");
+        beginPhase(13, "lua remove garbage");
+        RemoveGarbage.removeGarbage(imProg);
+        imProg.flatten(imTranslator);
+        timeTaker.endPhase();
+
+        beginPhase(14, "translate to lua");
         LuaTranslator luaTranslator = new LuaTranslator(imProg, imTranslator);
         LuaCompilationUnit luaCode = luaTranslator.translate();
         ImAttrType.setWurstClassType(TypesHelper.imInt());

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/LanguageWorker.java

@@ -213,13 +213,8 @@ private Workitem getNextWorkItem() {
                         FileReconcile fr = (FileReconcile) change;
                         affected = modelManager.syncCompilationUnitContent(fr.getFilename(), fr.getContents());
                     } else if (change instanceof FileSystemUpdated || change instanceof FileDeleted) {
-                        // Dependency roots may have changed (e.g. grill install), refresh and sync incrementally.
-                        modelManager.refreshDependencies();
-                        if (change instanceof FileDeleted) {
-                            affected = modelManager.removeCompilationUnit(change.getFilename());
-                        } else {
-                            affected = modelManager.syncCompilationUnit(change.getFilename());
-                        }
+                        // Dependency roots may have changed (e.g. grill install), sync full dependency state.
+                        affected = modelManager.syncDependencyCompilationUnits();
                     } else {
                         // Editor-triggered updates (save/close) use the normal incremental path.
                         affected = modelManager.syncCompilationUnit(change.getFilename());

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/ModelManager.java

@@ -34,6 +34,12 @@ public interface ModelManager {
      */
     void refreshDependencies();
 
+    /**
+     * refresh and synchronize all dependency compilation units.
+     * This handles dependency delete/replace/move/rename scenarios robustly.
+     */
+    Changes syncDependencyCompilationUnits();
+
     Changes syncCompilationUnit(WFile changedFilePath);
 
     Changes syncCompilationUnitContent(WFile filename, String contents);

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/ModelManagerImpl.java

@@ -19,6 +19,7 @@
 
 import java.io.*;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.*;
 import java.util.function.Consumer;
@@ -184,6 +185,39 @@ public void refreshDependencies() {
         readDependencies();
     }
 
+    @Override
+    public synchronized Changes syncDependencyCompilationUnits() {
+        readDependencies();
+
+        WurstModel model2 = model;
+        if (model2 == null) {
+            return Changes.empty();
+        }
+
+        Set<WFile> expectedDependencyFiles = getDependencyWurstFiles().stream()
+            .map(WFile::create)
+            .collect(Collectors.toSet());
+
+        List<WFile> loadedDependencyFiles = model2.stream()
+            .map(this::wFile)
+            .filter(this::isUnderDependenciesFolder)
+            .collect(Collectors.toList());
+
+        Changes changes = Changes.empty();
+
+        for (WFile loadedFile : loadedDependencyFiles) {
+            if (!expectedDependencyFiles.contains(loadedFile)) {
+                changes = changes.mergeWith(removeCompilationUnit(loadedFile));
+            }
+        }
+
+        for (WFile dependencyFile : expectedDependencyFiles) {
+            changes = changes.mergeWith(syncCompilationUnit(dependencyFile));
+        }
+
+        return changes;
+    }
+
     private String getCanonicalPath(File f) {
         try {
             return f.getCanonicalPath();
@@ -861,6 +895,18 @@ private boolean isAlreadyLoaded(WFile file) {
         return false;
     }
 
+    private boolean isUnderDependenciesFolder(WFile file) {
+        try {
+            Path filePath = file.getPath().toAbsolutePath().normalize();
+            Path dependencyRoot = Paths.get(projectPath.getAbsolutePath(), "_build", "dependencies")
+                .toAbsolutePath()
+                .normalize();
+            return filePath.startsWith(dependencyRoot) && Utils.isWurstFile(filePath.toString());
+        } catch (FileNotFoundException e) {
+            return false;
+        }
+    }
+
 
     public File getProjectPath() {
         return projectPath;

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/HoverInfo.java

@@ -422,6 +422,11 @@ public List<Either<String, MarkedString>> case_StmtExitwhen(StmtExitwhen stmtExi
             return string("exitwhen: exits the current loop when the condition is true.");
         }
 
+        @Override
+        public List<Either<String, MarkedString>> case_StmtContinue(StmtContinue stmtContinue) {
+            return string("continue: skips the rest of the current loop iteration.");
+        }
+
         @Override
         public List<Either<String, MarkedString>> case_ConstructorDef(ConstructorDef constr) {
             return description(constr);

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/MapRequest.java

@@ -27,6 +27,7 @@
 import de.peeeq.wurstscript.jassprinter.JassPrinter;
 import de.peeeq.wurstscript.luaAst.LuaCompilationUnit;
 import de.peeeq.wurstscript.parser.WPos;
+import de.peeeq.wurstscript.translation.lua.translation.LuaTranslator;
 import de.peeeq.wurstscript.utils.LineOffsets;
 import de.peeeq.wurstscript.utils.Utils;
 import net.moonlightflower.wc3libs.bin.app.W3I;
@@ -167,6 +168,7 @@ protected File compileMap(File projectFolder, WurstGui gui, Optional<File> mapCo
                 luaCode.get().print(sb, 0);
 
                 String compiledMapScript = sb.toString();
+                LuaTranslator.assertNoLeakedHashtableNativeCalls(compiledMapScript);
                 File buildDir = getBuildDir();
                 File outFile = new File(buildDir, BUILD_COMPILED_LUA_NAME);
                 Files.write(compiledMapScript.getBytes(Charsets.UTF_8), outFile);
@@ -418,8 +420,12 @@ private String resolveCachedMapFileName() {
         if (!cachedMapFileName.isEmpty()) {
             return cachedMapFileName;
         }
+        return resolveCachedMapFileName(runArgs.isLua());
+    }
+
+    private String resolveCachedMapFileName(boolean luaMode) {
         if (!map.isPresent()) {
-            return "cached_map.w3x";
+            return luaMode ? "cached_map_lua.w3x" : "cached_map_jass.w3x";
         }
         File inputMap = map.get();
         String inputName = inputMap.getName();
@@ -428,7 +434,8 @@ private String resolveCachedMapFileName() {
         // Keep only filesystem-safe characters and avoid collisions for same basename from different folders.
         String safeBase = baseName.replaceAll("[^a-zA-Z0-9._-]", "_");
         String pathHash = Integer.toUnsignedString(inputMap.getAbsolutePath().hashCode(), 36);
-        return safeBase + "_" + pathHash + "_cached.w3x";
+        String modeSuffix = luaMode ? "lua" : "jass";
+        return safeBase + "_" + pathHash + "_" + modeSuffix + "_cached.w3x";
     }
 
     protected File ensureWritableTargetFile(File targetFile, String dialogTitle, String lockMessage,
@@ -498,6 +505,7 @@ private boolean isLocked(File targetMap) {
      */
     protected File ensureCachedMap(WurstGui gui) throws IOException {
         File cachedMap = getCachedMapFile();
+        cleanupOppositeModeCacheAndOutputs();
 
         if (!map.isPresent()) {
             throw new RequestFailedException(MessageType.Error, "No source map provided");
@@ -515,6 +523,29 @@ protected File ensureCachedMap(WurstGui gui) throws IOException {
         return cachedMap;
     }
 
+    private void cleanupOppositeModeCacheAndOutputs() {
+        if (cachedMapFileName.isEmpty()) {
+            File cacheDir = new File(getBuildDir(), "cache");
+            String oppositeModeCacheName = resolveCachedMapFileName(!runArgs.isLua());
+            java.nio.file.Path oppositeModeCache = new File(cacheDir, oppositeModeCacheName).toPath();
+            try {
+                java.nio.file.Files.deleteIfExists(oppositeModeCache);
+            } catch (IOException e) {
+                WLogger.warning("Could not delete opposite-mode cached map: " + oppositeModeCache + " (" + e.getMessage() + ")");
+            }
+        }
+
+        File buildDir = getBuildDir();
+        File oppositeCompiledOutput = runArgs.isLua()
+            ? new File(buildDir, BUILD_COMPILED_JASS_NAME)
+            : new File(buildDir, BUILD_COMPILED_LUA_NAME);
+        try {
+            java.nio.file.Files.deleteIfExists(oppositeCompiledOutput.toPath());
+        } catch (IOException e) {
+            WLogger.warning("Could not delete opposite-mode compiled output: " + oppositeCompiledOutput + " (" + e.getMessage() + ")");
+        }
+    }
+
     protected CompilationResult compileScript(ModelManager modelManager, WurstGui gui, Optional<File> testMap,
                                               WurstProjectConfigData projectConfigData, File buildDir,
                                               boolean isProd) throws Exception {