[Help] 4 Players like ball game

Hello! I am trying to create a multiplayer game using Netcode for Gameobjects with a maximum 4 players. The players will be spawned on one of the 4 position on the board like this (and they should be able to move left/right or dash (shortly go forward and return to original position)) :

The spawn point comes from this code in the m_SpawnPoints list:

using System.Collections.Generic;
using Unity.Netcode;
using UnityEngine;

public class NetworkPlayerSpawner : NetworkBehaviour
{

    public static NetworkPlayerSpawner Singleton { get; private set; }

    private void Awake()
    {
        // If there is an instance, and it's not me, delete myself.
        if (Singleton != null && Singleton != this)
        {
            Destroy(this);
        }
        else
        {
            Singleton = this;
        }
    }

    public GameObject playerPrefab;
    public int playerCount = 0;
    // List with available spawn points (occupied in reversed order)
    private List<Vector3> m_SpawnPoints = new List<Vector3>(){
        new Vector3(4, 0, 0),
        new Vector3(-4, 0, 0),
        new Vector3(0, 0, 4),
        new Vector3(0, 0, -4)
    };

    public override void OnNetworkSpawn()
    {
        if (IsHost || IsClient)
        {
            // Avoid instantiating player on server. Only host or client.
            SpawnPlayerRpc();
        }
    }

    [Rpc(SendTo.Server)]
    void SpawnPlayerRpc(RpcParams rpcParams = default)
    {
        // Get the ID of the client that sent the RPC
        ulong senderClientId = rpcParams.Receive.SenderClientId;

        Debug.Log($"SpawnPlayerRpc - clientId: {senderClientId}");
        SpawnPlayerOnClients(senderClientId);
    }

    void SpawnPlayerOnClients(ulong clientId)
    {
        Debug.Log($"SpawnPlayerRpc - clientId: {clientId}");

        // Instantiate player object
        GameObject player = Instantiate(playerPrefab);

        // Get the NetworkObject component
        var playerNetworkObject = player.GetComponent<NetworkObject>();

        // Spawn the player object on all clients
        playerNetworkObject.SpawnAsPlayerObject(clientId);

        // Increment player count
        playerCount++;
    }

    // Method to get the next spawn point called by the player
    public Vector3 GetNextSpawnPoint()
    {
        if (m_SpawnPoints.Count == 0)
        {
            return Vector3.zero;
        }

        Vector3 spawnPoint = m_SpawnPoints[m_SpawnPoints.Count - 1];
        m_SpawnPoints.RemoveAt(m_SpawnPoints.Count - 1);
        return spawnPoint;
    }

}

But the problem is when the third player is spawned (or fourth) they are spawned in the correct position and quickly moved to the center (I suppose this is due to anticipatedMovement, because if I comment it then it stays at the correct position). To fix this i tried setting the rb.position to transform.position, but still it corrects the position to the spawned one but also it can’t move the their left/right position (I assume this is due to rigibody getting the world coordinates):

This is the code for player movement:
using UnityEngine;
using System.Collections;
using Unity.Netcode;
using Unity.Netcode.Components;

public class PlayerControllerNet : NetworkBehaviour
{
    public float hSpeed = 10.0f;
    private float dashSpeed = 5f;
    private float dashDuration = 0.1f;
    private float dashTimeLeft;
    private bool isDashing = false;
    private Vector3 originalPosition;

    private float horizontalInput;
    private float serverInput;
    private Rigidbody rb;
    private RigidbodyConstraints initialPostionConstrains;
    private AnticipatedNetworkTransform anticipatedNetworkTransform;

    public NetworkVariable<Vector3> Position = new NetworkVariable<Vector3>();

    public override void OnNetworkSpawn()
    {
        if (!IsServer) return;


        SetSpawnPosition();

        base.OnNetworkSpawn();
    }

    void SetSpawnPosition()
    {
        // Set the player's initial position based on the next available spawn point
        transform.position = NetworkPlayerSpawner.Singleton.GetNextSpawnPoint();
        // Rotate the player to face the origin middle of gameboard
        transform.LookAt(Vector3.zero);
    }

    // Start is called once before the first execution of Update after the MonoBehaviour is created
    void Start()
    {
        rb = GetComponent<Rigidbody>();
        anticipatedNetworkTransform = GetComponent<AnticipatedNetworkTransform>();

        originalPosition = transform.position;

        // Save the initial constraints of the rigidbody
        initialPostionConstrains = rb.constraints;
    }

    // Update is called once per frame
    void Update()
    {
        if (IsOwner)
        {
            MovePlayer();
        }
    }

    void MovePlayer()
    {
        // Get the horizontal input
        horizontalInput = Input.GetAxis("Horizontal");

        MovePlayerRpc(horizontalInput);

        // Dash when pressing spaces
        if (Input.GetKeyDown(KeyCode.Space) && !isDashing)
        {
            StartCoroutine(Dash());
        }
    }

    private void FixedUpdate()
    {

        if (IsOwner)
        {
            Vector3 anticipatedLocalPosition = transform.right * horizontalInput * hSpeed;
            anticipatedNetworkTransform.AnticipateMove(rb.position + anticipatedLocalPosition * Time.deltaTime);
        }
        if (IsServer)
        {
            // Move the player in local space based on input and transform it to world space
            Vector3 localMove = transform.right * serverInput * hSpeed;
            rb.MovePosition(rb.position + localMove * Time.deltaTime);
        }
    }

    [Rpc(SendTo.Server)]
    void MovePlayerRpc(float hInput)
    {
        // Store the input on the server
        serverInput = hInput;
        Position.Value = rb.position;
    }


    IEnumerator Dash()
    {
        // Disable the z position constraints to allow the player to move in the Z axis
        rb.constraints = RigidbodyConstraints.FreezePositionY | RigidbodyConstraints.FreezeRotation;

        isDashing = true;
        dashTimeLeft = dashDuration;

        while (dashTimeLeft > 0)
        {
            Vector3 dashDir = transform.forward * dashSpeed;
            rb.MovePosition(rb.position + dashDir * Time.deltaTime);
            dashTimeLeft -= Time.deltaTime;
            yield return null;
        }

        // Smoothly return to original Z position
        while (Mathf.Abs(transform.position.z - originalPosition.z) >= 0.1f)
        {
            Vector3 returnDir = new Vector3(0, 0, originalPosition.z - transform.position.z);
            rb.MovePosition(rb.position + returnDir * dashSpeed * Time.deltaTime);
            yield return null;
        }

        isDashing = false;
        // Reset the z position constraints
        rb.constraints = initialPostionConstrains;
    }
}

You are setting the position in Start() which may occur before the object is actually spawned on the network. Be sure to run networked initialization code only in or after OnNetworkSpawn.