Merge branch 'develop' into bugfix/LS24003755/ds-to-string-when-ds-is…

…-greater-than-string

docs/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,13 @@
 ## Description
 
-Include a summary of the change.
+Include a summary of the changes.
 
 Related to # (issue)
 
 ## Checklist:
-- [ ] There are tests regarding this feature
-- [ ] The code follows the Kotlin conventions (run `./gradlew ktlintCheck`)
-- [ ] The code passes all tests (run `./gradlew check`)
-- [ ] There is a specific documentation in the `docs` directory
+- [ ] If this feature involves RPGLE fixes or improvements, they are well-described in the summary.
+- [ ] There are tests for this feature.
+- [ ] RPGLE code used for tests is easily understandable and includes comments that clarify the purpose of this feature.
+- [ ] The code follows Kotlin conventions (run `./gradlew ktlintCheck`).
+- [ ] The code passes all tests (run `./gradlew check`).
+- [ ] Relevant documentation is included in the `docs` directory.

kolasu/src/main/kotlin/com/strumenta/kolasu/model/Processing.kt

@@ -191,7 +191,7 @@ fun Node.transformChildren(operation: (Node) -> Node, inPlace: Boolean = false):
         }
     }
     var instanceToTransform = this
-    if (!changes.isEmpty()) {
+    if (changes.isNotEmpty()) {
         val constructor = this.javaClass.kotlin.primaryConstructor!!
         val params = HashMap<KParameter, Any?>()
         constructor.parameters.forEach { param ->
@@ -206,9 +206,8 @@ fun Node.transformChildren(operation: (Node) -> Node, inPlace: Boolean = false):
     return instanceToTransform
 }
 
-fun Node.replace(other: Node) {
-    if (this.parent == null) {
-        throw IllegalStateException("Parent not set")
-    }
-    this.parent!!.transformChildren(inPlace = true, operation = { if (it == this) other else it })
+fun Node.replace(other: Node): Node {
+    return this.parent?.let {
+        it.transformChildren(inPlace = true, operation = { node -> if (node == this) other else node })
+    } ?: throw IllegalStateException("Parent not set")
 }

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/interpreter/DBFileMap.kt

@@ -160,8 +160,10 @@ fun Expression.createKList(fileMetadata: FileMetadata, interpreter: InterpreterC
     return if (type() is KListType) {
         interpreter.toSearchValues(this, fileMetadata)
     } else {
-        val value = interpreter.eval(this)
-        listOf(value.asString(fileMetadata.accessFieldsType[0]))
+        when (val value = interpreter.eval(this)) {
+            is StartValValue, is EndValValue -> throw NotImplementedError("$value constant not yet supported.")
+            else -> listOf(value.asString(fileMetadata.accessFieldsType.first()))
+        }
     }
 }
 

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/interpreter/Evaluator.kt

@@ -53,6 +53,8 @@ interface Evaluator {
     fun eval(expression: ArrayAccessExpr): Value
     fun eval(expression: HiValExpr): HiValValue
     fun eval(expression: LowValExpr): LowValValue
+    fun eval(expression: StartValExpr): StartValValue
+    fun eval(expression: EndValExpr): EndValValue
     fun eval(expression: ZeroExpr): ZeroValue
     fun eval(expression: AllExpr): AllValue
     fun eval(expression: TranslateExpr): Value

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/interpreter/ExpressionEvaluation.kt

@@ -327,6 +327,8 @@ class ExpressionEvaluation(
 
     override fun eval(expression: HiValExpr) = proxyLogging(expression) { HiValValue } as HiValValue
     override fun eval(expression: LowValExpr) = proxyLogging(expression) { LowValValue } as LowValValue
+    override fun eval(expression: StartValExpr) = proxyLogging(expression) { StartValValue } as StartValValue
+    override fun eval(expression: EndValExpr) = proxyLogging(expression) { EndValValue } as EndValValue
     override fun eval(expression: ZeroExpr) = proxyLogging(expression) { ZeroValue } as ZeroValue
     override fun eval(expression: AllExpr) = proxyLogging(expression) {
         AllValue(eval(expression.charsToRepeat).asString().value)

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/interpreter/typesystem.kt

@@ -170,6 +170,24 @@ object LowValType : Type() {
     override fun isNumeric() = true
 }
 
+@Serializable
+object StartValType : Type() {
+    override val size: Int
+        get() = throw IllegalStateException("Has variable size")
+
+    override fun hasVariableSize() = true
+    override fun isNumeric() = false
+}
+
+@Serializable
+object EndValType : Type() {
+    override val size: Int
+        get() = throw IllegalStateException("Has variable size")
+
+    override fun hasVariableSize() = true
+    override fun isNumeric() = false
+}
+
 @Serializable
 object TimeStampType : Type() {
     override val size: Int

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/interpreter/values.kt

@@ -771,6 +771,32 @@ object LowValValue : Value {
     }
 }
 
+object StartValValue : Value {
+    override fun toString() = "StartValValue"
+
+    // FIXME
+    override fun assignableTo(expectedType: Type) = true
+    override fun copy(): StartValValue = this
+
+    override operator fun compareTo(other: Value): Int =
+        if (other is StartValValue) 0 else -1
+
+    override fun asString() = "*START".asValue()
+}
+
+object EndValValue : Value {
+    override fun toString() = "EndValValue"
+
+    // FIXME
+    override fun assignableTo(expectedType: Type) = true
+    override fun copy(): EndValValue = this
+
+    override operator fun compareTo(other: Value): Int =
+        if (other is EndValValue) 0 else 1
+
+    override fun asString() = "*END".asValue()
+}
+
 object ZeroValue : Value {
 
     override fun copy() = this
@@ -954,7 +980,7 @@ fun Type.blank(): Value {
         is KListType -> throw UnsupportedOperationException("Blank value not supported for KList")
         is CharacterType -> CharacterValue(Array(this.nChars) { ' ' })
         is FigurativeType -> BlanksValue
-        is LowValType, is HiValType -> TODO()
+        is LowValType, is HiValType, is StartValType, is EndValType -> TODO()
         is UnlimitedStringType -> UnlimitedStringValue("")
         is RecordFormatType -> BlanksValue
     }

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/ast/expressions.kt

@@ -96,6 +96,16 @@ data class LowValExpr(override val position: Position? = null) : FigurativeConst
     override fun evalWith(evaluator: Evaluator): Value = evaluator.eval(this)
 }
 
+@Serializable
+data class StartValExpr(override val position: Position? = null) : FigurativeConstantRef(position) {
+    override fun evalWith(evaluator: Evaluator): Value = evaluator.eval(this)
+}
+
+@Serializable
+data class EndValExpr(override val position: Position? = null) : FigurativeConstantRef(position) {
+    override fun evalWith(evaluator: Evaluator): Value = evaluator.eval(this)
+}
+
 @Serializable
 data class ZeroExpr(override val position: Position? = null) : FigurativeConstantRef(position) {
     override fun evalWith(evaluator: Evaluator): Value = evaluator.eval(this)
@@ -210,7 +220,7 @@ data class DifferentThanExpr(var left: Expression, var right: Expression, overri
 // / Logical operations
 // /
 @Serializable
-data class NotExpr(val base: Expression, override val position: Position? = null) : Expression(position) {
+data class NotExpr(var base: Expression, override val position: Position? = null) : Expression(position) {
     override fun evalWith(evaluator: Evaluator): Value = evaluator.eval(this)
 }
 
@@ -333,7 +343,7 @@ data class QualifiedAccessExpr(val container: Expression, val field: ReferenceBy
 }
 
 @Serializable
-data class ArrayAccessExpr(val array: Expression, val index: Expression, override val position: Position? = null) :
+data class ArrayAccessExpr(var array: Expression, var index: Expression, override val position: Position? = null) :
     AssignableExpression(position) {
     override fun render(): String {
         return "${this.array.render()}(${index.render()}))"

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/ast/serialization.kt

@@ -16,10 +16,7 @@
 
 package com.smeup.rpgparser.parsing.ast
 
-import com.smeup.rpgparser.interpreter.AbstractDataDefinition
-import com.smeup.rpgparser.interpreter.DataDefinition
-import com.smeup.rpgparser.interpreter.FieldDefinition
-import com.smeup.rpgparser.interpreter.InStatementDataDefinition
+import com.smeup.rpgparser.interpreter.*
 import com.smeup.rpgparser.parsing.parsetreetoast.LogicalCondition
 import com.smeup.rpgparser.serialization.BigDecimalSerializer
 import com.smeup.rpgparser.serialization.LocalDateTimeSerializer
@@ -142,6 +139,7 @@ private val modules = SerializersModule {
         subclass(NegationExpr::class)
         subclass(EditcExpr::class)
         subclass(EditwExpr::class)
+        subclass(EndValExpr::class)
         subclass(EofExpr::class)
         subclass(EqualExpr::class)
         subclass(EqualityExpr::class)
@@ -186,6 +184,7 @@ private val modules = SerializersModule {
         subclass(ReplaceExpr::class)
         subclass(ScanExpr::class)
         subclass(SqrtExpr::class)
+        subclass(StartValExpr::class)
         subclass(StringLiteral::class)
         subclass(SubstExpr::class)
         subclass(SubarrExpr::class)

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/ast/statements.kt

@@ -816,7 +816,7 @@ data class CallStmt(
 
 @Serializable
 data class CallPStmt(
-    val functionCall: FunctionCall,
+    var functionCall: FunctionCall,
     val errorIndicator: IndicatorKey? = null,
     override val position: Position? = null
 ) : Statement(position), StatementThatCanDefineData {

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/parsetreetoast/misc.kt

@@ -728,6 +728,8 @@ internal fun SymbolicConstantsContext.toAst(conf: ToAstConfiguration = ToAstConf
     return when {
         this.SPLAT_HIVAL() != null -> HiValExpr(position)
         this.SPLAT_LOVAL() != null -> LowValExpr(position)
+        this.SPLAT_START() != null -> StartValExpr(position)
+        this.SPLAT_END() != null -> EndValExpr(position)
         this.SPLAT_BLANKS() != null -> BlanksRefExpr(position)
         this.SPLAT_ZEROS() != null -> ZeroExpr(position)
         this.SPLAT_OFF() != null -> OffRefExpr(position)
@@ -2224,6 +2226,7 @@ internal fun getProgramNameToCopyBlocks(): ProgramNameToCopyBlocks {
 }
 
 internal fun <T : AbstractDataDefinition> List<T>.removeDuplicatedDataDefinition(): List<T> {
+    // NOTE: With current logic when type matches on duplications the first definition wins
     val dataDefinitionMap = mutableMapOf<String, AbstractDataDefinition>()
     return removeUnnecessaryRecordFormat().filter {
         val dataDefinition = dataDefinitionMap[it.name]
@@ -2241,14 +2244,17 @@ internal fun <T : AbstractDataDefinition> List<T>.removeDuplicatedDataDefinition
 
 internal fun AbstractDataDefinition.matchType(dataDefinition: AbstractDataDefinition): Boolean {
     fun Type.matchType(other: Any?): Boolean {
-        if (this is NumberType && other is NumberType) {
-            val resultDigits = this.entireDigits == other.entireDigits && this.decimalDigits == other.decimalDigits
-            if (rpgType?.isNotBlank()!! && other.rpgType?.isNotEmpty()!!) {
-                return resultDigits && rpgType == other.rpgType
+        // TODO: Improve logic for StringType/UnlimitedStringType matching
+        return when {
+            this is NumberType && other is NumberType -> {
+                val resultDigits = this.entireDigits == other.entireDigits && this.decimalDigits == other.decimalDigits
+                if (rpgType?.isNotBlank()!! && other.rpgType?.isNotEmpty()!!) {
+                    return resultDigits && rpgType == other.rpgType
+                }
+                resultDigits
             }
-            return resultDigits
-        } else {
-            return this == other
+            this is UnlimitedStringType && other is StringType -> true
+            else -> this == other
         }
     }
 

rpgJavaInterpreter-core/src/main/kotlin/com/smeup/rpgparser/parsing/parsetreetoast/resolution.kt

@@ -33,20 +33,44 @@ import com.strumenta.kolasu.validation.Error
 import com.strumenta.kolasu.validation.ErrorType
 import java.util.*
 
+private fun List<CompositeStatement>.findWrappedInStatementDataDefinitions(): List<StatementThatCanDefineData> {
+    if (this.isEmpty()) return emptyList()
+
+    val candidates = this.flatMap { it.body }
+    val free = candidates.filterIsInstance<StatementThatCanDefineData>()
+    val unwrapped = candidates.filterIsInstance<CompositeStatement>().findWrappedInStatementDataDefinitions()
+    return free + unwrapped
+}
+
+private fun List<StatementThatCanDefineData>.moveDefineStmtsToEnd(): List<StatementThatCanDefineData> {
+    val defineStmts = this.filterIsInstance<DefineStmt>()
+    val otherStmts = this.filter { it !is DefineStmt }
+    return otherStmts + defineStmts
+}
+
 private fun CompilationUnit.findInStatementDataDefinitions() {
-    this.allStatements(preserveCompositeStatement = true)
-        .filterIsInstance<StatementThatCanDefineData>()
-        .forEach { statementThatCanDefineData ->
-            kotlin.runCatching {
-                this.addInStatementDataDefinitions(statementThatCanDefineData.dataDefinition())
-            }.onFailure { error ->
-                if (statementThatCanDefineData is Node) {
-                    kotlin.runCatching {
-                        statementThatCanDefineData.error("Error while creating data definition from statement: $statementThatCanDefineData", error)
-                    }
-                } else throw error
-            }
+    // Filter related statements
+    val candidates = this.allStatements(preserveCompositeStatement = true)
+    val compositeStatements = candidates.filterIsInstance<CompositeStatement>()
+    val freeStatements = candidates.filterIsInstance<StatementThatCanDefineData>()
+
+    // Unwrap StatementThatCanDefineData contained in CompositeStatements
+    val unwrappedCompositeStatements = compositeStatements.findWrappedInStatementDataDefinitions()
+
+    // Move define statements to end as they can be based on other instatement definitions
+    val targetStatements = (freeStatements + unwrappedCompositeStatements).moveDefineStmtsToEnd()
+
+    targetStatements.forEach { statementThatCanDefineData ->
+        kotlin.runCatching {
+            this.addInStatementDataDefinitions(statementThatCanDefineData.dataDefinition())
+        }.onFailure { error ->
+            if (statementThatCanDefineData is Node) {
+                kotlin.runCatching {
+                    statementThatCanDefineData.error("Error while creating data definition from statement: $statementThatCanDefineData", error)
+                }
+            } else throw error
         }
+    }
 }
 
 private fun MutableList<InStatementDataDefinition>.addAllDistinct(list: List<InStatementDataDefinition>): List<InStatementDataDefinition> {
@@ -99,16 +123,35 @@ private fun Node.resolveDataRefs(cu: CompilationUnit) {
 private fun Node.resolveFunctionCalls(cu: CompilationUnit) {
     // replace FunctionCall with ArrayAccessExpr where it makes sense
     this.specificProcess(FunctionCall::class.java) { fc ->
-        if (fc.args.size == 1) {
-            val data = cu.allDataDefinitions.firstOrNull { it.name == fc.function.name }
-            if (data != null) {
-                fc.replace(ArrayAccessExpr(
-                        array = DataRefExpr(ReferenceByName(fc.function.name, referred = data)),
-                        index = fc.args[0],
-                        position = fc.position))
-            }
+        fc.tryReplaceWithArrayAccess(cu)
+    }
+}
+
+private fun FunctionCall.tryReplaceWithArrayAccess(cu: CompilationUnit): Optional<Node> {
+    // Only said FunctionCalls with 1 arg can be ArrayAccessExpr
+    if (this.args.size != 1) return Optional.empty()
+
+    // Replacement can only happen when there is a DataDefinition named like this 'FunctionCall'
+    val data = cu.allDataDefinitions.firstOrNull { it.name == this.function.name }
+    data ?: return Optional.empty()
+
+    // Recursively try to process inner expressions
+    var indexExpr = this.args.first()
+    if (indexExpr is FunctionCall) {
+        indexExpr.tryReplaceWithArrayAccess(cu).ifPresent {
+            // Needed for type-checking
+            if (it is Expression) indexExpr = it
         }
     }
+
+    val arrayAccessExpr = ArrayAccessExpr(
+        array = DataRefExpr(ReferenceByName(this.function.name, referred = data)),
+        index = indexExpr,
+        position = this.position
+    )
+
+    val newExpression = this.replace(arrayAccessExpr).children.first()
+    return Optional.of(newExpression)
 }
 
 fun MuteAnnotation.resolveAndValidate(cu: CompilationUnit) {

rpgJavaInterpreter-core/src/test/kotlin/com/smeup/rpgparser/interpreter/JarikoCallbackTest.kt

@@ -26,6 +26,7 @@ import com.smeup.rpgparser.parsing.facade.SourceReference
 import com.smeup.rpgparser.parsing.facade.SourceReferenceType
 import com.smeup.rpgparser.parsing.parsetreetoast.ToAstConfiguration
 import com.smeup.rpgparser.rpginterop.DirRpgProgramFinder
+import com.smeup.rpgparser.smeup.dbmock.TABDS01LDbMock
 import com.smeup.rpgparser.utils.Format
 import com.smeup.rpgparser.utils.compile
 import org.junit.Assert
@@ -660,6 +661,31 @@ class JarikoCallbackTest : AbstractTest() {
         executeSourceLineTest(pgm = "ERROR35")
     }
 
+    /**
+     * NOTE: This is error is thrown because Reload does not support '*START' and '*END' constants yet.
+     * When this feature gets supported please restore it on Jariko side by following these steps:
+     * - Remove this test or mark it as ignored
+     * - Remove the [@Ignore] decorator from the [MULANGT50FileAccess1Test.executeMUDRNRAPU00248] test
+     * - Remove the runtime error (it should be in [Expression.createKList] if not moved)
+     */
+    @Test
+    fun executeERROR36CallBackTest() {
+        TABDS01LDbMock().usePopulated {
+            executePgmCallBackTest(
+                pgm = "ERROR36",
+                sourceReferenceType = SourceReferenceType.Program,
+                sourceId = "ERROR36",
+                lines = listOf(6),
+                reloadConfig = it.createReloadConfig()
+            )
+        }
+    }
+
+    @Test
+    fun executeERROR36SourceLineTest() {
+        executeSourceLineTest(pgm = "ERROR36")
+    }
+
     @Test
     fun bypassSyntaxErrorTest() {
         val configuration = Configuration().apply {