I need to spawn a certain amount of enemies in a wave, but instead an infinite amount of enemies are being spawned

public PlayerHealth playerHealth;
public GameObject enemy;
public float spawnTime = 3f;
public Transform spawnPoints;
int totalEnemies = 17;
int livesEnemies = 0;
float waitToWave = 15;
float waveTime;

void Start ()
{
    InvokeRepeating ("Spawn", spawnTime, spawnTime);

}

void Spawn ()
{
    if(playerHealth.currentHealth <= 0f)
    {
        return;
    }
    int spawnPointIndex = Random.Range(0, spawnPoints.Length);

    if (livesEnemies == 0)
    {

        if (waitToWave <= waveTime)
        {
            waveTime = 0;
            livesEnemies = totalEnemies;
            for (int i = 0; i < totalEnemies; i++)

                Instantiate(enemy, spawnPoints[spawnPointIndex].position, spawnPoints[spawnPointIndex].rotation);
                
        }
        else
        {
            waveTime += Time.deltaTime;
        }
    }

}

I created a new Function called NewWave that will check if there are any enemies are alive before wanting to spawn, and then call this from repeating.

you will need to make the code to keep track of how many enemies are still alive, it looks like you are using liveEnemies to mean number of enemies that are still alive.

now this will still spawn enemies infinitely in waves but will only spawn a new wave when the last wave is fully dead.

I also added a counter to know how many waves have spawned in numberOfWaves, when this reaches the maxWaves cancle.invoke will call to stop the repeating function

public PlayerHealth playerHealth;
     public GameObject enemy;
     public float spawnTime = 3f;
     public Transform[] spawnPoints;
     int totalEnemies = 17;
     int livesEnemies = 0;
     float waitToWave = 15;
     float waveTime;
     int numberOfWaves = 0;
     int maxWaves = 5;
     
       
 void Start ()
 {
     InvokeRepeating ("NewWave", spawnTime, spawnTime);
 }

void NewWave()
{
    if(livesEnemies == 0)
    {
        numberOfWaves++
        Spawn()
     }
     else if(numberOfWaves == maxWaves)
     {
           Cancle.Invoke();
     }
}

 void Spawn ()
 {
     if(playerHealth.currentHealth <= 0f)
     {
         return;
     }
     int spawnPointIndex = Random.Range(0, spawnPoints.Length);
     if (livesEnemies == 0)
     {
         if (waitToWave <= waveTime)
         {
             waveTime = 0;
             livesEnemies = totalEnemies;
             for (int i = 0; i < totalEnemies; i++)
                 Instantiate(enemy, spawnPoints[spawnPointIndex].position, spawnPoints[spawnPointIndex].rotation);
                 
         }
         else
         {
             waveTime += Time.deltaTime;
         }
     }
 }