Im going to look into that.
I found this which is working for me to an extent. I am able to download the json database with no errors which still takes some time but it is almost as if the content header is being changed from version 5 to version 3 when half of the file is created. Now I’m not 100% on how I am setting the content header is correct.
I came across this snippet which allows me to set a download buffer for whatever is coming trough unitywebrequest.
it works. but the file always ends up truncated so the json syntax is never completed.
System.ArgumentException: JSON parse error: The document root must not follow by other values.
is the error that is thrown
using System.Collections;
using System.Collections.Generic;
using System;
using System.IO;
using UnityEngine;
using UnityEngine.Networking;
public class CustomWebRequest : DownloadHandlerScript
{
// Standard scripted download handler - will allocate memory on each ReceiveData callback
public CustomWebRequest()
: base()
{
}
// Pre-allocated scripted download handler
// Will reuse the supplied byte array to deliver data.
// Eliminates memory allocation.
public CustomWebRequest(byte[] buffer)
: base(buffer)
{
Init();
}
// Required by DownloadHandler base class. Called when you address the 'bytes' property.
protected override byte[] GetData() { return null; }
// Called once per frame when data has been received from the network.
protected override bool ReceiveData(byte[] byteFromServer, int dataLength)
{
if (byteFromServer == null || byteFromServer.Length < 1)
{
Debug.Log("CustomWebRequest :: ReceiveData - received a null/empty buffer");
return false;
}
//Write the current data chunk to file
AppendFile(byteFromServer, dataLength);
return true;
}
//Where to save the video file
string vidSavePath;
string jsonSavePath;
string efileName = Application.dataPath + "/Resource/LiveEmoteDetails.json";
//The FileStream to save the file
FileStream fileStream = null;
//Used to determine if there was an error while opening or saving the file
bool success;
void Init()
{
vidSavePath = Path.Combine(Application.dataPath, "Resource");
//Create Directory if it does not exist
if (!Directory.Exists(vidSavePath))
{
Directory.CreateDirectory(Path.GetDirectoryName(vidSavePath.Replace("/", "\\")));
}
Debug.Log(vidSavePath);
FileInfo emotefi = new FileInfo(efileName);
// Check if file dose not exists. If yes, use the else statment.
if (!emotefi.Exists) {
StreamWriter outStream = System.IO.File.CreateText (efileName);
outStream.WriteLine ("{0}", "");
outStream.Close ();
fileStream = new FileStream(efileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
Debug.Log("File Successfully opened at" + efileName .Replace("/", "\\"));
success = true;
} else {
//Open the current file to write to
fileStream = new FileStream(efileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
Debug.Log("File Successfully opened at" + efileName .Replace("/", "\\"));
success = true;
}
}
void AppendFile(byte[] buffer, int length)
{
if (success)
{
try
{
//Write the current data to the file
fileStream.Write(buffer, 0, length);
Debug.Log("Written data chunk to: " + efileName.Replace("/", "\\"));
}
catch (Exception e)
{
success = false;
}
}
}
// Called when all data has been received from the server and delivered via ReceiveData
protected override void CompleteContent()
{
if (success)
Debug.Log("Done! Saved File to: " + efileName.Replace("/", "\\"));
else
Debug.LogError("Failed to Save File to: " + efileName.Replace("/", "\\"));
//Close filestream
fileStream.Close();
}
// Called when a Content-Length header is received from the server.
protected override void ReceiveContentLength(int contentLength)
{
Debug.Log(string.Format("CustomWebRequest :: ReceiveContentLength - length {0}", contentLength));
}
}
This snippet is updated version of the original post for GetERequest()
IEnumerator GetERequest(string uri) {
byte[] bytes = new byte[2000];
var euwr = new UnityWebRequest(uri, "GET");
euwr.SetRequestHeader("ContentType", "application/json");
//euwr.SetRequestHeader("ContentType", "application/vnd.twitchtv.v5+json");
euwr.SetRequestHeader("Accept", "application/vnd.twitchtv.v5+json");
euwr.SetRequestHeader("Client-ID", channel_clientid);
//euwr.uploadHandler = (UploadHandler) new UploadHandlerRaw(jsonToSend);
//euwr.downloadHandler = (DownloadHandler) new DownloadHandlerBuffer();
euwr.downloadHandler = new CustomWebRequest(bytes);
//Debug.Log("sent auth");
// Request and wait for the desired page.
yield
return euwr.SendWebRequest();
string[] pages = uri.Split('/');
int page = pages.Length - 1;
if (euwr.isNetworkError) {
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "Emotes Error: " + euwr.error);
emoteerrorstatus = true;
//Debug.Log("Network error");
}
else {
//Debug.Log(euwr.downloadProgress);
if (euwr.downloadHandler.text.Contains("error")) {
if (euwr.downloadHandler.text.Contains("Not Found")) {
if (euwr.downloadHandler.text.Contains("status")) {
if (euwr.downloadHandler.text.Contains("404")) {
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "Emotes Error: Not found.");
emoteerrorstatus = true;
}
}
}
if (euwr.downloadHandler.text.Contains("Gone")) {
if (euwr.downloadHandler.text.Contains("status")) {
if (euwr.downloadHandler.text.Contains("410")) {
if (euwr.downloadHandler.text.Contains("message")) {
//Debug.Log(euwr.downloadHandler.text);
if (euwr.downloadHandler.text.Contains("v3")) {
//WriteEmoteDetails(euwr.downloadHandler.text);
//Debug.Log("v3 found: " + euwr.downloadHandler.text);
//euwr.SetRequestHeader("Accept", "application/vnd.twitchtv.v5+json");
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "Emotes Error: Requesting wrong version. See Doc http://dev.twitch.tv/docs");
emoteerrorstatus = true;
} else {
//Debug.Log("other v3 error: " + euwr.downloadHandler.text);
//euwr.SetRequestHeader("Accept", "application/vnd.twitchtv.v5+json");
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", euwr.downloadHandler.text);
//WriteEmoteDetails(euwr.downloadHandler.text);
emoteerrorstatus = true;
}
}
}
}
} else {
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", euwr.downloadHandler.text);
emoteerrorstatus = true;
Debug.Log(euwr.downloadHandler.text);
}
if (euwr.downloadHandler.text.Contains("Bad Request")) {
if (euwr.downloadHandler.text.Contains("status")) {
if (euwr.downloadHandler.text.Contains("400")) {
if (euwr.downloadHandler.text.Contains("No client id specified")) {
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
emoteerrorstatus = true;
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "Bad Emotes Request - Error 400 - No client id specified.");
//Debug.Log(pages[page] + ":\nCreated1: " + euwr.downloadHandler.text);
} else {
if (euwr.downloadHandler.text.Contains("Invalid client id specified")) {
emoteerrorstatus = true;
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "Bad Emotes Request - Error 400 - Invalid id specified.");
//Debug.Log(pages[page] + ":\nCreated2: " + euwr.downloadHandler.text);
}
}
}
}
}
}
else {
string efileName = Application.dataPath + "/Resource/LiveEmoteDetails.json";
FileInfo fi = new FileInfo(efileName);
//string UserLogData = "";
string totalEmotes = "";
using(StreamReader sr = File.OpenText(fi.ToString())) {
int lnum = 0;
string s = "";
while ((s = sr.ReadLine()) != null) {
totalEmotes = totalEmotes + s;
}
}
EmoteData edata = JsonUtility.FromJson < EmoteData > (totalEmotes);
int itemnum = 0;
if (edata.emoteItems.Count > 0) {
//WriteEmoteDetails(euwr.downloadHandler.text);
itemnum = 0;
foreach(jSONClass etemp in edata.emoteItems) {
Debug.Log(etemp.regex);
itemnum = itemnum + 1;
}
TwitchChat_Reader tal = TwitchAPIListener.GetComponent < TwitchChat_Reader > ();
string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "Channel emotes recieved.");
Debug.Log(pages[page] + ":\nDone downloading emotes.");
emoteerrorstatus = false;
}
// else
// {
// string client_id = UnityEngine.Random.Range(22222222, 88888888).ToString();
// tal.CreateElementLine("0", "0", "0", "#000000", "Notice", "0", "0", client_id, "0", "0", "0", "0", "0", "0", "0", "No emotes recieved.");
// emoteerrorstatus = true;
// }
}
}
}