My Agent do stupid things in a simple enviroment

Unity Engine
, ,
1

HI! I’m recently studying ML-Agent and many of its applications.
I recently created a new simple project in which a cube must move within a room with the aim of collecting an object that moves with each collection.
I implemented movement along one direction, rotation and jumping.
When I start the training the cube jumps immediately without ever stopping, it doesn’t jump on the spot but it NEVER moves close to the ground.

This is the code:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Unity.MLAgents;
using Unity.MLAgents.Sensors;
using Unity.MLAgents.Actuators;

public class PinoScript : Agent
{
    // Variabili per i componenti e gli oggetti
    Rigidbody PinoBody;

    [Header("Movement")]
    public float MovementForce;
    public float RotationForce;
    public float MaxSpeedRotation;

    [Header("Jump")]
    [SerializeField] private float JumpForce;

    public Transform Target;
    bool OnGround = true;

    internal Vector3 startPos;
    internal Vector3 startRot;

    // Start is called before the first frame update
    void Start()
    {
        // Assegna il componente Rigidbody a PinoBody
        PinoBody = GetComponent<Rigidbody>();
        startPos = transform.position;
        startRot = transform.eulerAngles;
    }

    public override void OnEpisodeBegin()
    {
        //Resetta Pino
        ResetPino();
        // Sposta il target in una posizione casuale
        MoveTarget();
    }

    public override void CollectObservations(VectorSensor sensor)
    {
        //Temporaneamente Vuoto
    }

    public override void OnActionReceived(ActionBuffers actionBuffers)
    {
        int MovementAction = Mathf.FloorToInt(actionBuffers.DiscreteActions[0]);
        int RotationAction = Mathf.FloorToInt(actionBuffers.DiscreteActions[1]);
        int JumpAction = Mathf.FloorToInt(actionBuffers.DiscreteActions[2]);
        Vector3 RotateDir = transform.forward * MovementAction * RotationForce;
        Vector3 MoveDir = Vector3.zero;

        switch (MovementAction)
        {
            case 0: //Avanti
                PinoBody.AddForce(transform.forward * MovementForce);
                break;
            case 1:
                PinoBody.AddForce(-transform.forward * MovementForce);
                break;
            case 2:
                MoveDir = Vector3.zero;
                break;
        }

        switch (RotationAction)
        {
            case 0: //Avanti
                PinoBody.AddTorque(Vector3.up * RotationForce);
                break;
            case 1:
                PinoBody.AddTorque(-Vector3.up * RotationForce);
                break;
            case 2:
                RotateDir = Vector3.zero;
                break;
        }

        // Limit angular velocity
        if (PinoBody.angularVelocity.magnitude > MaxSpeedRotation)
        {
            PinoBody.angularVelocity = PinoBody.angularVelocity.normalized * MaxSpeedRotation;
        }

        if (JumpAction == 1 && OnGround)
        {
            PinoBody.AddForce(Vector3.up * JumpForce, ForceMode.Impulse);
            OnGround = false;

            // Applica penalità per il salto
            AddReward(-5f);
            Debug.Log("Penalità Salto: " + -5f);
        }

        AddReward(-0.0005f);
        //Debug.Log("Penalità Tempo: " + -0.0005f);

        if (transform.position.y < -1.3f)
        {
            AddReward(-0.5f);
            Debug.Log("Penalità Caduta: " + -0.5f);
            EndEpisode();
        }
    }


    public void OnTriggerEnter(Collider trigger)
    {
        // Se Pino raggiunge il target, assegna una ricompensa e termina l'episodio
        if (trigger.gameObject.tag == "PinoTarget")
        {
            AddReward(5f);
            Debug.Log("Ricompensa Target: " + 5f);
            MoveTarget();
        }
    }

    public void OnCollisionEnter(Collision collision)
    {

        if (collision.gameObject.tag == "Muri")
        {
            AddReward(-1f);
            Debug.Log("Penalità Muri assegnata: " + -1f);
        }

        if (collision.gameObject.tag == "Pavimento")
        {
            OnGround = true;
        }
    }

    // Funzione che resetta Pino alla posizione e rotazione iniziali
    void ResetPino()
    {
        //Reset della posizione e rotazione
        transform.position = startPos;
        transform.eulerAngles = startRot;

        //Reset della velocità
        PinoBody.velocity = Vector3.zero;
        PinoBody.angularVelocity = Vector3.zero;

    }

    // Funzione che sposta il target in una posizione casuale
    void MoveTarget()
    {
        Target.position = new Vector3(Random.value * 18 - 9f,
                                      1.1f,
                                      Random.value * 8 - 4);
    }

    public override void Heuristic(in ActionBuffers actionsOut)
    {
        var discreteActionsOut = actionsOut.DiscreteActions;

        // Azzera le azioni per evitare movimenti residui
        discreteActionsOut.Clear();

        // Movimento avanti/indietro (0: fermo, 1: avanti, 2: indietro)
        if (Input.GetKey(KeyCode.W))
        {
            discreteActionsOut[0] = 0;
        }
        else if (Input.GetKey(KeyCode.S))
        {
            discreteActionsOut[0] = 1;
        }
        else
        {
            discreteActionsOut[0] = 2;
        }

        // Rotazione destra/sinistra (0: fermo, 1: sinistra, 2: destra)
        if (Input.GetKey(KeyCode.A))
        {
            discreteActionsOut[1] = 0;
        }
        else if (Input.GetKey(KeyCode.D))
        {
            discreteActionsOut[1] = 1;
        }
        else
        {
            discreteActionsOut[1] = 2;
        }

        // Salto (0: non saltare, 1: saltare)
        if (Input.GetKey(KeyCode.Space))
        {
            discreteActionsOut[2] = 1;
        }
        else
        {
            discreteActionsOut[2] = 0;
        }
    }
}

Can someone explain to me why? it’s right for him to jump but I would like him to do it logically

2

the code looks like it should work ok as far as only jumping when on the ground etc so do you mean it jumps a lot?
this is normal, when training it will just do random actions so unless you punish it for excess jumping (or reward it for staying on the ground etc) it’s going to jump a lot, i see you are trying to punish it for jumping already, it should help reduce how much it jumps after a bit of training.

other than the above methods another method is to use action masks and a timer to enable the discrete action to let it jump only after it’s been on the ground for a while or if you want to really control the logic give it set areas it can jump in etc

3

I tried to let him train for almost 30 minutes but he kept jumping. he never moved without jumping.
I wouldn’t know what limits to set for him, for now the environment is very simple and without obstacles, I wanted to train him momentarily without obstacles and then add them later after he learned to move decently.
So the limited areas of jumping do not have much use and perhaps not even that of the time because if there are areas with obstacles one behind the other he can not jump continuously