first preliminary version of obiscript.

Former-commit-id: 0d2c0fc5e33e0873ba5c04aca4cf7dd69aa83c90

pkg/obilua/lua.go

@@ -0,0 +1,140 @@
+package obilua
+
+import (
+	"bytes"
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	log "github.com/sirupsen/logrus"
+	lua "github.com/yuin/gopher-lua"
+	"github.com/yuin/gopher-lua/parse"
+)
+
+func NewInterpreter() *lua.LState {
+	lua := lua.NewState()
+
+	RegisterObilib(lua)
+
+	return lua
+}
+
+func Compile(program []byte, name string) (*lua.FunctionProto, error) {
+
+	reader := bytes.NewReader(program)
+	chunk, err := parse.Parse(reader, name)
+	if err != nil {
+		return nil, err
+	}
+
+	proto, err := lua.Compile(chunk, name)
+	if err != nil {
+		return nil, err
+	}
+	return proto, nil
+}
+
+func CompileScript(filePath string) (*lua.FunctionProto, error) {
+	program, err := os.ReadFile(filePath)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return Compile(program, filePath)
+}
+
+func LuaWorker(proto *lua.FunctionProto) obiseq.SeqWorker {
+	interpreter := NewInterpreter()
+	lfunc := interpreter.NewFunctionFromProto(proto)
+
+	f := func(sequence *obiseq.BioSequence) (obiseq.BioSequenceSlice, error) {
+		interpreter.SetGlobal("sequence", obiseq2Lua(interpreter, sequence))
+		interpreter.Push(lfunc)
+		err := interpreter.PCall(0, lua.MultRet, nil)
+
+		return obiseq.BioSequenceSlice{sequence}, err
+	}
+
+	return f
+}
+
+func LuaProcessor(iterator obiiter.IBioSequence, name, program string, breakOnError bool, nworkers int) obiiter.IBioSequence {
+	newIter := obiiter.MakeIBioSequence()
+
+	if nworkers <= 0 {
+		nworkers = obioptions.CLIParallelWorkers()
+	}
+
+	newIter.Add(nworkers)
+
+	go func() {
+		newIter.WaitAndClose()
+	}()
+
+	bp := []byte(program)
+	proto, err := Compile(bp, name)
+
+	if err != nil {
+		log.Fatalf("Cannot compile script %s : %v", name, err)
+	}
+
+	ff := func(iterator obiiter.IBioSequence) {
+		w := LuaWorker(proto)
+		sw := obiseq.SeqToSliceWorker(w, false)
+
+		// iterator = iterator.SortBatches()
+
+		for iterator.Next() {
+			seqs := iterator.Get()
+			slice := seqs.Slice()
+			ns, err := sw(slice)
+
+			if err != nil {
+				if breakOnError {
+					log.Fatalf("Error during Lua sequence processing : %v", err)
+				} else {
+					log.Warnf("Error during Lua sequence processing : %v", err)
+				}
+			}
+
+			newIter.Push(obiiter.MakeBioSequenceBatch(seqs.Order(), ns))
+			seqs.Recycle(false)
+		}
+
+		newIter.Done()
+	}
+
+	for i := 1; i < nworkers; i++ {
+		go ff(iterator.Split())
+	}
+
+	go ff(iterator)
+
+	if iterator.IsPaired() {
+		newIter.MarkAsPaired()
+	}
+
+	return newIter
+
+}
+
+func LuaPipe(name, program string, breakOnError bool, nworkers int) obiiter.Pipeable {
+
+	f := func(input obiiter.IBioSequence) obiiter.IBioSequence {
+		return LuaProcessor(input, name, program, breakOnError, nworkers)
+	}
+
+	return f
+}
+
+func LuaScriptPipe(filename string, breakOnError bool, nworkers int) obiiter.Pipeable {
+	program, err := os.ReadFile(filename)
+
+	if err != nil {
+		log.Fatalf("Cannot read script file %s", filename)
+	}
+
+	return LuaPipe(filename, string(program), breakOnError, nworkers)
+}

pkg/obilua/lua_push_interface.go

@@ -0,0 +1,193 @@
+package obilua
+
+import (
+	"log"
+
+	lua "github.com/yuin/gopher-lua"
+)
+
+// pushInterfaceToLua converts a Go interface{} value to an equivalent Lua value and pushes it onto the stack.
+//
+// L *lua.LState: the Lua state onto which the value will be pushed.
+// val interface{}: the Go interface value to be converted and pushed. This can be a basic type such as string, bool, int, float64,
+// or slices and maps of these basic types. Custom complex types will be converted to userdata with a predefined metatable.
+//
+// No return values. This function operates directly on the Lua state stack.
+func pushInterfaceToLua(L *lua.LState, val interface{}) {
+	switch v := val.(type) {
+	case string:
+		L.Push(lua.LString(v))
+	case bool:
+		L.Push(lua.LBool(v))
+	case int:
+		L.Push(lua.LNumber(v))
+	case float64:
+		L.Push(lua.LNumber(v))
+	// Add other cases as needed for different types
+	case map[string]int:
+		pushMapStringIntToLua(L, v)
+	case map[string]string:
+		pushMapStringStringToLua(L, v)
+	case map[string]bool:
+		pushMapStringBoolToLua(L, v)
+	case map[string]float64:
+		pushMapStringFloat64ToLua(L, v)
+	case []string:
+		pushSliceStringToLua(L, v)
+	case []int:
+		pushSliceIntToLua(L, v)
+	case []float64:
+		pushSliceFloat64ToLua(L, v)
+	case []bool:
+		pushSliceBoolToLua(L, v)
+	case nil:
+		L.Push(lua.LNil)
+	default:
+		log.Fatalf("Cannot deal with value Mv", val)
+	}
+}
+
+// pushMapStringIntToLua creates a new Lua table and iterates over the Go map to set key-value pairs in the Lua table. It then pushes the Lua table onto the stack.
+//
+// L *lua.LState - the Lua state
+// m map[string]int - the Go map containing string to int key-value pairs
+func pushMapStringIntToLua(L *lua.LState, m map[string]int) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go map and set the key-value pairs in the Lua table
+	for key, value := range m {
+		L.SetField(luaTable, key, lua.LNumber(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushMapStringStringToLua creates a new Lua table and sets key-value pairs from the Go map, then pushes the Lua table onto the stack.
+//
+// L *lua.LState, m map[string]string. No return value.
+func pushMapStringStringToLua(L *lua.LState, m map[string]string) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go map and set the key-value pairs in the Lua table
+	for key, value := range m {
+		L.SetField(luaTable, key, lua.LString(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushMapStringBoolToLua creates a new Lua table, iterates over the Go map, sets the key-value pairs in the Lua table, and then pushes the Lua table onto the stack.
+//
+// Parameters:
+//
+//	L *lua.LState - the Lua state
+//	m map[string]bool - the Go map
+//
+// Return type(s): None
+func pushMapStringBoolToLua(L *lua.LState, m map[string]bool) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go map and set the key-value pairs in the Lua table
+	for key, value := range m {
+		L.SetField(luaTable, key, lua.LBool(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushMapStringFloat64ToLua pushes a map of string-float64 pairs to a Lua table on the stack.
+//
+// L *lua.LState - the Lua state
+// m map[string]float64 - the map to be pushed to Lua
+func pushMapStringFloat64ToLua(L *lua.LState, m map[string]float64) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go map and set the key-value pairs in the Lua table
+	for key, value := range m {
+		// Use lua.LNumber since Lua does not differentiate between float and int
+		L.SetField(luaTable, key, lua.LNumber(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushSliceIntToLua creates a new Lua table and sets the elements of a Go slice in the Lua table. Then, it pushes the Lua table onto the stack.
+//
+// L *lua.LState, slice []int
+// None
+func pushSliceIntToLua(L *lua.LState, slice []int) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go slice and set the elements in the Lua table
+	for _, value := range slice {
+		// Append the value to the Lua table
+		// Lua is 1-indexed, so we use the length of the table + 1 as the next index
+		luaTable.Append(lua.LNumber(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushSliceStringToLua creates a new Lua table and sets the elements in the table from the given Go slice. It then pushes the Lua table onto the stack.
+//
+// L *lua.LState - The Lua state
+// slice []string - The Go slice of strings
+func pushSliceStringToLua(L *lua.LState, slice []string) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go slice and set the elements in the Lua table
+	for _, value := range slice {
+		// Append the value to the Lua table
+		luaTable.Append(lua.LString(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushSliceBoolToLua creates a new Lua table and pushes the boolean values from the given slice onto the Lua stack.
+//
+// L *lua.LState - the Lua state
+// slice []bool - the Go slice containing boolean values
+func pushSliceBoolToLua(L *lua.LState, slice []bool) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go slice and insert each boolean into the Lua table
+	for _, value := range slice {
+		// Lua is 1-indexed, so we use the length of the table + 1 as the next index
+		luaTable.Append(lua.LBool(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}
+
+// pushSliceFloat64ToLua creates a new Lua table and pushes it onto the stack.
+//
+// L *lua.LState - the Lua state
+// slice []float64 - the Go slice to be inserted into the Lua table
+func pushSliceFloat64ToLua(L *lua.LState, slice []float64) {
+	// Create a new Lua table
+	luaTable := L.NewTable()
+
+	// Iterate over the Go slice and insert each float64 into the Lua table
+	for _, value := range slice {
+		// Lua is 1-indexed, so we append the value to the Lua table
+		luaTable.Append(lua.LNumber(value))
+	}
+
+	// Push the Lua table onto the stack
+	L.Push(luaTable)
+}

pkg/obilua/obilib.go

@@ -0,0 +1,7 @@
+package obilua
+
+import lua "github.com/yuin/gopher-lua"
+
+func RegisterObilib(luaState *lua.LState) {
+	RegisterObiSeq(luaState)
+}

pkg/obilua/obiseq.go

@@ -0,0 +1,154 @@
+package obilua
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	lua "github.com/yuin/gopher-lua"
+)
+
+func RegisterObiSeq(luaState *lua.LState) {
+	registerBioSequenceType(luaState)
+}
+
+const luaBioSequenceTypeName = "BioSequence"
+
+func registerBioSequenceType(luaState *lua.LState) {
+	bioSequenceType := luaState.NewTypeMetatable(luaBioSequenceTypeName)
+	luaState.SetGlobal(luaBioSequenceTypeName, bioSequenceType)
+	luaState.SetField(bioSequenceType, "new", luaState.NewFunction(newObiSeq))
+
+	luaState.SetField(bioSequenceType, "__index",
+		luaState.SetFuncs(luaState.NewTable(),
+			bioSequenceMethods))
+}
+
+func obiseq2Lua(interpreter *lua.LState,
+	sequence *obiseq.BioSequence) lua.LValue {
+	ud := interpreter.NewUserData()
+	ud.Value = sequence
+	interpreter.SetMetatable(ud, interpreter.GetTypeMetatable(luaBioSequenceTypeName))
+
+	return ud
+}
+func newObiSeq(luaState *lua.LState) int {
+	seqid := luaState.CheckString(1)
+	seq := []byte(luaState.CheckString(2))
+
+	definition := ""
+	if luaState.GetTop() > 2 {
+		definition = luaState.CheckString(3)
+	}
+
+	sequence := obiseq.NewBioSequence(seqid, seq, definition)
+
+	luaState.Push(obiseq2Lua(luaState, sequence))
+	return 1
+}
+
+var bioSequenceMethods = map[string]lua.LGFunction{
+	"id":         bioSequenceGetSetId,
+	"sequence":   bioSequenceGetSetSequence,
+	"definition": bioSequenceGetSetDefinition,
+	"count":      bioSequenceGetSetCount,
+	"taxid":      bioSequenceGetSetTaxid,
+	"attribute":  bioSequenceGetSetAttribute,
+}
+
+// checkBioSequence checks if the first argument in the Lua stack is a *obiseq.BioSequence.
+//
+// This function accepts a pointer to the Lua state and attempts to retrieve a userdata
+// that holds a pointer to a BioSequence. If the conversion is successful, it returns
+// the *BioSequence. If the conversion fails, it raises a Lua argument error.
+// Returns a pointer to obiseq.BioSequence or nil if the argument is not of the expected type.
+func checkBioSequence(L *lua.LState) *obiseq.BioSequence {
+	ud := L.CheckUserData(1)
+	if v, ok := ud.Value.(*obiseq.BioSequence); ok {
+		return v
+	}
+	L.ArgError(1, "obiseq.BioSequence expected")
+	return nil
+}
+
+// bioSequenceGetSetId gets the ID of a biosequence or sets a new ID if provided.
+//
+// This function expects a *lua.LState pointer as its only parameter.
+// If a second argument is provided, it sets the new ID for the biosequence.
+// It returns 0 if a new ID is set, or 1 after pushing the current ID onto the stack.
+func bioSequenceGetSetId(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	if luaState.GetTop() == 2 {
+		s.SetId(luaState.CheckString(2))
+		return 0
+	}
+	luaState.Push(lua.LString(s.Id()))
+	return 1
+}
+
+func bioSequenceGetSetSequence(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	if luaState.GetTop() == 2 {
+		s.SetSequence([]byte(luaState.CheckString(2)))
+		return 0
+	}
+	luaState.Push(lua.LString(s.String()))
+	return 1
+}
+
+func bioSequenceGetSetDefinition(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	if luaState.GetTop() == 2 {
+		s.SetDefinition(luaState.CheckString(2))
+		return 0
+	}
+	luaState.Push(lua.LString(s.Definition()))
+	return 1
+}
+
+func bioSequenceGetSetCount(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	if luaState.GetTop() == 2 {
+		s.SetCount(luaState.CheckInt(2))
+		return 0
+	}
+	luaState.Push(lua.LNumber(s.Count()))
+	return 1
+}
+
+func bioSequenceGetSetTaxid(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	if luaState.GetTop() == 2 {
+		s.SetTaxid(luaState.CheckInt(2))
+		return 0
+	}
+	luaState.Push(lua.LNumber(s.Taxid()))
+	return 1
+}
+
+func bioSequenceGetSetAttribute(luaState *lua.LState) int {
+	s := checkBioSequence(luaState)
+	attName := luaState.CheckString(2)
+
+	if luaState.GetTop() == 3 {
+		ud := luaState.CheckAny(3)
+
+		log.Infof("ud : %v [%v]", ud, ud.Type())
+		//
+		// Perhaps the code needs some type checking on ud.Value
+		// It's a first test
+		//
+
+		s.SetAttribute(attName, ud)
+		return 0
+	}
+
+	value, ok := s.GetAttribute(attName)
+
+	if !ok {
+		luaState.Push(lua.LNil)
+	} else {
+		pushInterfaceToLua(luaState, value)
+	}
+
+	return 1
+}

pkg/obitools/obiscript/obiscript.go

@@ -0,0 +1,14 @@
+package obiscript
+
+import (
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiiter"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obilua"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obioptions"
+)
+
+func CLIScriptPipeline() obiiter.Pipeable {
+
+	pipe := obilua.LuaScriptPipe(CLIScriptFilename(), true, obioptions.CLIParallelWorkers())
+
+	return pipe
+}

pkg/obitools/obiscript/options.go

@@ -0,0 +1,83 @@
+package obiscript
+
+import (
+	"log"
+	"os"
+
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obiconvert"
+	"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obitools/obigrep"
+	"github.com/DavidGamba/go-getoptions"
+)
+
+var _script = ""
+var _askTemplate = false
+
+func ScriptOptionSet(options *getoptions.GetOpt) {
+
+	options.StringVar(&_script, "script", _script,
+		options.Description("The script to execute."),
+		options.Alias("S"),
+		options.Description("Name of a map attribute."))
+
+	options.BoolVar(&_askTemplate, "template", _askTemplate,
+		options.Description("Print on the standard output a script template."),
+	)
+}
+
+func OptionSet(options *getoptions.GetOpt) {
+	ScriptOptionSet(options)
+	obiconvert.OptionSet(options)
+	obigrep.SequenceSelectionOptionSet(options)
+}
+
+func CLIScriptFilename() string {
+	return _script
+}
+
+func CLIScript() string {
+	file, err := os.ReadFile(_script) // Reads the script
+	if err != nil {
+		log.Fatalf("cannot read the script file : %s", _script)
+	}
+	return string(file)
+}
+
+func CLIAskScriptTemplate() bool {
+	return _askTemplate
+}
+
+func CLIScriptTemplate() string {
+	return `
+	import {
+		"sync"
+		"git.metabarcoding.org/obitools/obitools4/obitools4/pkg/obiseq"
+	}
+	//
+	// Begin function run before the first sequence being processed
+	//
+
+	func Begin(environment *sync.Map) {
+
+	}
+
+	//
+	// Begin function run after the last sequence being processed
+	//
+
+	func End(environment *sync.Map) {
+
+	}
+
+	//
+	// Worker function run for each sequence validating the selection predicat as specified by 
+	// the command line options.
+	//
+	// The function must return the altered sequence.
+	// If the function returns nil, the sequence is discarded from the output
+	func Worker(sequence *obiseq.BioSequence, environment *sync.Map) *obiseq.BioSequence {
+
+
+		return sequence
+	}
+	`
+}