Character Controller Colliding with Box Collider with trigger enabled ?

I have a character controller on my player and it is using advanced moving methods to move around…
IE, I apply my own gravity and all that stuff… The character moves np…

Now I have a enemy transform and It has a character controller on it and it uses basic movement… to chase my player. It has a box collider with the trigger enabled which is used as a line of sight trigger for my enemy to chase my player.

Everything works good, the player enters the sight area and my enemy gives chase and attacks…
Now, if I out run the enemy, I have the on exit say to return to spawn and it also works… but I also notice this…
There are times when the enemy line of sight box collider collides with my player controller and it pushes my player back… it only occurs if I outrun the enemy sight box… then for some odd reason I start to collide with the line of sight box and as the enemy is chasing me it pushes my player backward.

Any one have any idea what is going on ?

I ended up ditching the method of using a box collider for sight and instead used a area…
Here is a basic script I made well trying to work through this issue if any one is interested in a spawn point based attack script… simple but a good starting block for simple based stuff.

using UnityEngine;
using System.Collections;

public enum TargetState{Spawnpoint,Enemy}

public class BasicAiAttack : MonoBehaviour {
   
    public CharacterController controller;
    public Transform Target;
    public float ChaseSpeed = 0f;
    public float AttackRate = 0.5F;
    public float AlertArea = 1f;

    //how often do we search for targets..
    public float SearchRate = 0.5F;


    private Transform origspawnLocation;
    private float animationspeed = 0f;
    private Animator charanimator { get ; set; }
    private bool returning = true;
    private float nextattack = 0.0F;
    private float nextsearch = 0.0F;
    private TargetState targetstate = TargetState.Spawnpoint;
   
    void Awake()
    {
        charanimator = GetComponent<Animator>();
        if (charanimator == null)
            Debug.Log("Cant Find Animator to animate enemey");

        controller = GetComponent("CharacterController") as CharacterController;
        ChaseSpeed = 1f;

        var go = new GameObject();
        //go.tag = "SpawnerLocation";
        go.transform.position = transform.position;
        origspawnLocation = go.transform;
        Target = origspawnLocation;
        targetstate = TargetState.Spawnpoint;
    }

    void AddSpeed(float delta)
    {
        animationspeed = Mathf.Clamp( animationspeed + delta, -1f, 1f);
        charanimator.SetFloat("Speed", animationspeed);
    }
   
    void SetSpeed(float spd)
    {
        animationspeed = spd;
        charanimator.SetFloat("Speed", animationspeed);   
    }

    void MoveTowardTarget()
    {
        var forward = transform.TransformDirection (Vector3.forward);
        controller.SimpleMove (forward * ChaseSpeed);
        transform.LookAt (Target.position);
        AddSpeed (0.1f);
    }

    void ReturnedToSpawnPoint()
    {
        returning = false;
        transform.TransformDirection(Target.position);
        SetSpeed(0f);
        charanimator.SetBool ("Attack",false);
    }

    //Right from the Unity Documentation As is.. reversed
    //you may want to modify this to track how much dmg for each player, etc and attack
    //player who does most dmg.. or attack player of class healer.. etc. in order
    private GameObject FindClosestPlayer()
    {
        GameObject[] gos;
        gos = GameObject.FindGameObjectsWithTag("Player");
        GameObject closest = null;

        float distance = Mathf.Infinity;
        Vector3 position = transform.position;
        foreach (GameObject go in gos) {
            Vector3 diff = go.transform.position - position;
            float curDistance = diff.sqrMagnitude;
            if (curDistance < distance)
            {
                closest = go;
                distance = curDistance;
            }
        }
        return closest;
    }

    // Update is called once per frame
    void Update ()
    {

        float distance = 9999f;

        if (Time.time > nextsearch)
        {
            nextsearch = Time.time + SearchRate;
            Debug.Log("searching");

            //locate nearest player..
            Transform nearestplayer = FindClosestPlayer().transform;
            float nearestplayerdistance = Vector3.Distance(nearestplayer.position, transform.position);

            if (nearestplayerdistance <= AlertArea)
            {
                targetstate = TargetState.Enemy;
                Target = nearestplayer;
                Debug.Log("found player");
            }
            else
            {
                //player has ran away..
                Debug.Log("player not close enough");

                targetstate = TargetState.Spawnpoint;
                Target = origspawnLocation;
                returning = true;
            }
        }

       distance = Vector3.Distance(Target.position, transform.position);

        //if returning if flagged then we return to spawn point and ignore player..
        //good idea would be to also be imune to player attack if returning is set.
        if (returning)
        {
            if (distance >= .5f)
                MoveTowardTarget();
            else
                ReturnedToSpawnPoint();
        }

        if (targetstate == TargetState.Enemy)
        {
            //so we are not returning so we are idle or attacking...
            if (distance <= AlertArea) {
                //player is close enough for us to chase or attack.
                if (distance >= 1.2f) {
                    MoveTowardTarget ();
                }

                if (distance <= 1.2f)
                if (Time.time > nextattack)
                {   //attach logic goes here..
                    Debug.Log ("attacking");
                    nextattack = Time.time + AttackRate;
                    SetSpeed (0f);
                    charanimator.SetBool ("Attack", true);
                }
            }
        }
    }
}