[FishNet] Having a weird issue with prediction

using Cinemachine;
using FishNet.Component.Prediction;
using FishNet.Object;
using FishNet.Object.Prediction;
using FishNet.Transporting;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.InputSystem;
using UnityEngine.InputSystem.XR;
using UnityEngine.Windows;

[RequireComponent(typeof(CharacterController))]
public class PlayerMovement : NetworkBehaviour
{
    [Header("Player")]
    [Tooltip("Move speed of the character in m/s")]
    public float MoveSpeed = 2.0f;

    [Tooltip("Acceleration and deceleration")]
    public float SpeedChangeRate = 10.0f;

    [Tooltip("Time required to pass before entering the fall state. Useful for walking down stairs")]
    public float FallTimeout = 0.15f;

    [Tooltip("How fast the character turns to face movement direction")]
    [Range(0.0f, 0.3f)]
    public float RotationSmoothTime = 0.12f;

    [Tooltip("How far in degrees can you move the camera up")]
    public float TopClamp = 70.0f;

    [Tooltip("How far in degrees can you move the camera down")]
    public float BottomClamp = -30.0f;

    [Space(10)]
    [Tooltip("The height the player can jump")]
    public float JumpHeight = 1.2f;

    [Tooltip("The character uses its own gravity value. The engine default is -9.81f")]
    public float Gravity = -15.0f;

    [Space(10)]
    [Tooltip("Time required to pass before being able to jump again. Set to 0f to instantly jump again")]
    public float JumpTimeout = 0.50f;

    [Header("Cinemachine")]
    [Tooltip("The follow target set in the Cinemachine Virtual Camera that the camera will follow")]
    public GameObject CinemachineCameraTarget;

    [Tooltip("Additional degress to override the camera. Useful for fine tuning camera position when locked")]
    public float CameraAngleOverride = 0.0f;

    [Tooltip("How far can you scroll the camera out")]
    public float ScrollTopClamp = 40.0f;

    [Tooltip("How far can you scroll the camera in")]
    public float ScrollBottomClamp = 1.0f;

    [Tooltip("The speed the camera zooms in and out")]
    public float ScrollSpeed;

    [Header("Player Grounded")]
    [Tooltip("If the character is grounded or not. Not part of the CharacterController built in grounded check")]
    public bool Grounded = true;

    [Tooltip("Useful for rough ground")]
    public float GroundedOffset = -0.14f;

    [Tooltip("The radius of the grounded check. Should match the radius of the CharacterController")]
    public float GroundedRadius = 0.28f;

    [Tooltip("What layers the character uses as ground")]
    public LayerMask GroundLayers;

    private CharacterController _controller;
    private GameObject _mainCamera;
    private Cinemachine3rdPersonFollow _thirdPersonFollow;

    private float _speed;
    private float _targetRotation = 0.0f;
    private float _rotationVelocity;
    private float _verticalVelocity;
    private float _cinemachineTargetYaw;
    private float _cinemachineTargetPitch;
    private const float _threshold = 0.01f;
    private float _jumpTimeoutDelta;
    private float _fallTimeoutDelta;
    private float _terminalVelocity = 53.0f;
    private bool _callOnce;
    private bool _jump;

    public struct MoveData : IReplicateData
    {
        public uint SentTick;
        public bool Jump;
        public float Horizontal;
        public float Vertical;
        public MoveData(bool jump, float horizontal, float vertical, uint sentTick)
        {
            Jump = jump;
            Horizontal = horizontal;
            Vertical = vertical;
            SentTick = sentTick;
            _tick = 0;
        }

        private uint _tick;
        public void Dispose() { }
        public uint GetTick() => _tick;
        public void SetTick(uint value) => _tick = value;
    }

    public struct ReconcileData : IReconcileData
    {
        public Vector3 Position;
        public float VerticalVelocity;
        public ReconcileData(Vector3 position, float verticalVelocity)
        {
            Position = position;
            VerticalVelocity = verticalVelocity;
            _tick = 0;
        }

        private uint _tick;
        public void Dispose() { }
        public uint GetTick() => _tick;
        public void SetTick(uint value) => _tick = value;
    }

    public override void OnStartNetwork()
    {
        _controller = GetComponent<CharacterController>();
        base.TimeManager.OnTick += TimeManager_OnTick;
    }

    public override void OnStopNetwork()
    {
        base.TimeManager.OnTick -= TimeManager_OnTick;
    }


    private void TimeManager_OnTick()
    {
        HandleMovement(BuildMoveData());
        _jump = false;
        /* The base.IsServer check is not required but does save a little
        * performance by not building the reconcileData if not server. */
        if (base.IsServerStarted)
        {
            ReconcileData rd = new ReconcileData(transform.position, _verticalVelocity);
            Reconciliation(rd);
        }
    }

    private MoveData BuildMoveData()
    {
        if (!base.IsOwner)
            return default;

        Vector2 moveInput = PlayerInputHandler.Instance.MoveInput;

        float horizontal = moveInput.x;
        float vertical = moveInput.y;

        MoveData md;
        if (horizontal != 0 || vertical != 0)
            md = new MoveData(_jump, horizontal, vertical, TimeManager.LocalTick);
        else
            md = new MoveData(_jump, horizontal, vertical, 0);
        _jump = false;

        return md;
    }

    private void SetUpCamera()
    {
        if (!IsOwner || _callOnce)
        {
            return;
        }
        _mainCamera = Camera.main.gameObject;
        _thirdPersonFollow = CinemachineCameraManager.Instance.CinemachineVirtualCamera.GetCinemachineComponent<Cinemachine3rdPersonFollow>();
        CinemachineCameraManager.Instance.CinemachineVirtualCamera.Follow = CinemachineCameraTarget.transform;
        CinemachineCameraManager.Instance.CinemachineVirtualCamera.LookAt = CinemachineCameraTarget.transform;
        _callOnce = true;
    }

    private void Update()
    {
        if (!IsOwner)
        {
            return;
        }
        SetUpCamera();
        GroundedCheck();

        if (PlayerInputHandler.Instance.JumpInput)
        {
            _jump = true;
        }
    }

    private void LateUpdate()
    {
        if (!IsOwner)
        {
            return;
        }
        CameraRotation();
        CameraScroll();
    }

    [Replicate]
    private void HandleMovement(MoveData data, ReplicateState state = ReplicateState.Invalid, Channel channel = Channel.Unreliable)
    {
        Vector2 moveData = new Vector2(data.Horizontal, data.Vertical);

        Move(moveData);
        JumpAndGravity(data.Jump);
    }

    [Reconcile]
    private void Reconciliation(ReconcileData rd, Channel channel = Channel.Unreliable)
    {
        transform.position = rd.Position;
        _verticalVelocity = rd.VerticalVelocity;
    }

    private void CameraRotation()
    {
        Vector2 lookInput = PlayerInputHandler.Instance.LookInput;
        // if there is an input and camera position is not fixed
        if (lookInput.sqrMagnitude >= _threshold)
        {

            _cinemachineTargetYaw += lookInput.x;
            _cinemachineTargetPitch += lookInput.y;
        }

        // clamp our rotations so our values are limited 360 degrees
        _cinemachineTargetYaw = ClampAngle(_cinemachineTargetYaw, float.MinValue, float.MaxValue);
        _cinemachineTargetPitch = ClampAngle(_cinemachineTargetPitch, BottomClamp, TopClamp);

        // Cinemachine will follow this target
        CinemachineCameraTarget.transform.rotation = Quaternion.Euler(_cinemachineTargetPitch + CameraAngleOverride,
            _cinemachineTargetYaw, 0.0f);
    }

    private void CameraScroll()
    {
        float scrollInput = PlayerInputHandler.Instance.ScrollInput;

        float distance = Mathf.Clamp(_thirdPersonFollow.CameraDistance + scrollInput * ScrollSpeed, ScrollBottomClamp, ScrollTopClamp);

        _thirdPersonFollow.CameraDistance = distance;
    }

    private void Move(Vector2 moveInput)
    {
        float delta = (float)TimeManager.TickDelta;
        // set target speed based on move speed, sprint speed and if sprint is pressed
        float targetSpeed = MoveSpeed;

        Debug.Log($"Time.deltaTime:{Time.deltaTime}, TimeManager.TickDelta: {(float)TimeManager.TickDelta}");

        // a simplistic acceleration and deceleration designed to be easy to remove, replace, or iterate upon

        // note: Vector2's == operator uses approximation so is not floating point error prone, and is cheaper than magnitude
        // if there is no input, set the target speed to 0
        if (moveInput == Vector2.zero) targetSpeed = 0.0f;

        // a reference to the players current horizontal velocity
        float currentHorizontalSpeed = new Vector3(_controller.velocity.x, 0.0f, _controller.velocity.z).magnitude;

        float speedOffset = 0.1f;
        float inputMagnitude = moveInput.magnitude;

        // accelerate or decelerate to target speed
        if (currentHorizontalSpeed < targetSpeed - speedOffset ||
            currentHorizontalSpeed > targetSpeed + speedOffset)
        {
            // creates curved result rather than a linear one giving a more organic speed change
            // note T in Lerp is clamped, so we don't need to clamp our speed
            _speed = Mathf.Lerp(currentHorizontalSpeed, targetSpeed * inputMagnitude,
                delta * SpeedChangeRate);

            // round speed to 3 decimal places
            _speed = Mathf.Round(_speed * 1000f) / 1000f;
        }
        else
        {
            _speed = targetSpeed;
        }

        // normalise input direction
        Vector3 inputDirection = new Vector3(moveInput.x, 0.0f, moveInput.y).normalized;

        // note: Vector2's != operator uses approximation so is not floating point error prone, and is cheaper than magnitude
        // if there is a move input rotate player when the player is moving
        if (moveInput != Vector2.zero)
        {
            _targetRotation = Mathf.Atan2(inputDirection.x, inputDirection.z) * Mathf.Rad2Deg +
                              _mainCamera.transform.eulerAngles.y;
            float rotation = Mathf.SmoothDampAngle(transform.eulerAngles.y, _targetRotation, ref _rotationVelocity,
                RotationSmoothTime);

            // rotate to face input direction relative to camera position
            transform.rotation = Quaternion.Euler(0.0f, rotation, 0.0f);
        }


        Vector3 targetDirection = Quaternion.Euler(0.0f, _targetRotation, 0.0f) * Vector3.forward;

        // move the player
        _controller.Move(targetDirection.normalized * (_speed * delta) +
                         new Vector3(0.0f, _verticalVelocity, 0.0f) * delta);
    }

    private static float ClampAngle(float lfAngle, float lfMin, float lfMax)
    {
        if (lfAngle < -360f) lfAngle += 360f;
        if (lfAngle > 360f) lfAngle -= 360f;
        return Mathf.Clamp(lfAngle, lfMin, lfMax);
    }

    private void GroundedCheck()
    {
        // set sphere position, with offset
        Vector3 spherePosition = new Vector3(transform.position.x, transform.position.y - GroundedOffset,
            transform.position.z);
        Grounded = Physics.CheckSphere(spherePosition, GroundedRadius, GroundLayers,
            QueryTriggerInteraction.Ignore);
    }

    private void JumpAndGravity(bool jump)
    {
        float delta = (float)TimeManager.TickDelta;
        if (Grounded)
        {
            // reset the fall timeout timer
            _fallTimeoutDelta = FallTimeout;

            // stop our velocity dropping infinitely when grounded
            if (_verticalVelocity < 0.0f)
            {
                _verticalVelocity = -2f;
            }

            // Jump
            if (jump && _jumpTimeoutDelta <= 0.0f)
            {
                // the square root of H * -2 * G = how much velocity needed to reach desired height
                _verticalVelocity = Mathf.Sqrt(JumpHeight * -2f * Gravity);
            }

            // jump timeout
            if (_jumpTimeoutDelta >= 0.0f)
            {
                _jumpTimeoutDelta -= delta;
            }
        }
        else
        {
            // reset the jump timeout timer
            _jumpTimeoutDelta = JumpTimeout;

            // fall timeout
            if (_fallTimeoutDelta >= 0.0f)
            {
                _fallTimeoutDelta -= delta;
            }
        }

        // apply gravity over time if under terminal (multiply by delta time twice to linearly speed up over time)
        if (_verticalVelocity < _terminalVelocity)
        {
            _verticalVelocity += Gravity * delta;
        }
    }

    private void OnDrawGizmosSelected()
    {
        Color transparentGreen = new Color(0.0f, 1.0f, 0.0f, 0.35f);
        Color transparentRed = new Color(1.0f, 0.0f, 0.0f, 0.35f);

        if (Grounded) Gizmos.color = transparentGreen;
        else Gizmos.color = transparentRed;

        // when selected, draw a gizmo in the position of, and matching radius of, the grounded collider
        Gizmos.DrawSphere(
            new Vector3(transform.position.x, transform.position.y - GroundedOffset, transform.position.z),
            GroundedRadius);
    }
}

I am trying to get this code to work with prediction, and when I use delta as (float)TickManager.TickDelta, for some reason the player when moving will just start speeding into the direction moved until the game breaks a few seconds later. The only thing I can say I notice is that currentHorizontalSpeed and _speed just become huge numbers incredibly quickly.

Some other odd behaviors I noticed so far is that if I change delta to a fixed number, or even time.DeltaTime it works, but I tried making delta equal to what the TickDelta normally is, around .03 and found the same issue happening, tried .04 and it happened again. I think around .06 or .07 makes it work fine. and .01 also worked fine too. I decided to clamp _speed and found that it was acting like I was pressing a key the entire time until I alt tabbed. Debugging Vector2 moveInput in the method though shows it is working properly.

Okay, so I am still not sure how to fix it but I think I know what the cause is. Based on what I found I think the reconciliation method is causing the issue, when the delta is at a certain range, it will cause some sort of feedback loop where it will keep getting out of sync more and more and teleporting the character, increasing the velocity permanently, but if the delta is out of that range, it will teleport and correct itself before the feedback loop occurs, but is causing the movement to look a bit jittery.

Doesn't seem to be the issue, reconcile never gets called so I am back to the drawing board.

I am sorry to not add anything of value. I am facing the same problem. Clients move way too fast and do not get reset to their position on server, even though the reconciliation method does get called. Also, my player actions (jump et. al) are not performed client side; but if the host-client is jumping, all players jump.
This was on FishNet 4.1.4 as well as on Fishnet 4.1.5 using PredictionV2.
Unfortunately right now I don't have the time needed to try older versions and see if the problem was introduced at some point.