Hi,
I have been struggling to solve this. I can read async way with WWW, but I have no clue how do I write large files(something like hundreds of MB) to disk in async way. Please help.
What are you trying to do exactly?
void WriteDataToFile(byte[] data, string filePath, Action callback = null)
{
//implementation
//File.WriteAllBytes(filePath, data);//I can not use this because it is not asynchronous like www or asyncoperation in unity
}
// Update is called once per frame
void Update ()
{
byte[] myDLC_Data_574MB = null;//lets say this is my data
string DLC_save_path = Application.persistentDataPath + "/DLC/" + "rpl.dlc";
float timeDLC = Time.time;
WriteDataToFile(myDLC_Data_574MB, DLC_save_path, () =>
{
Debug.LogWarning("file write has been completed in: " + (Time.time - timeDLC) + " seconds");
});
}
Something like this.
You could try to do this in a seperate thread, be aware though that you neither should neither access Unity’s API (except for a few thread-safe calls like logging) nor access or manipulate data you’re currently writing to disk.
The threading thing is so slow. Can not use it.
Why is it slow?
I/O operations are always dependant on many external factors, such as hardware and its performance (read/write, bandwith and other limitations), concurrent usage/access and even locks on potentially shared resources that you have to wait for.
A program cannot speed up these limitations given by hardware, unless you can split the operations (e.g. writing multiple files - even then hardware limitations such as read/write bandwidth may kick in sooner or later).
1 Like
I have no idea why the last test turned out so slow. But now its ok. In non threaded, it took a single GC spike with 423MB for a file of similar size and cpu usage was something like 1100-1200ms. If I use threads, I encounter multiple spike. One of them had 423MB GC, all others did not have GC allocation. And all of them have something like 120-150ms cpu usage. Better. :). Here is the script I created.
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using UnityEngine;
using UnityEngine.UI;
public class BigFileTransfer : MonoBehaviour
{
public bool useThreads = false;
public string serverFileURLAbsolute = "http://192.168.1.28/data.bin";
// Use this for initialization
void Start () {
}
// Update is called once per frame
void Update () {
if (Input.GetKeyDown(KeyCode.Space))
{
DownBigFileThenSaveIt();
}
}
public void DownBigFileThenSaveIt()
{
StartCoroutine(StartBigDataTransfer());
}
private byte[] tmp_data;
private string fileSavePath;
private Thread thread;
IEnumerator StartBigDataTransfer()
{
WWW www = new WWW(serverFileURLAbsolute);
yield return www;
fileSavePath = Application.persistentDataPath + "/data.bin";
Debug.LogWarning("save path: " + fileSavePath);
tmp_data = www.bytes;
www.Dispose();
if (useThreads)
{
SaveDataThreaded();
}
else
{
SaveDataTaskNonThreaded();
}
hasTaskBeenDone_Threaded = false;
StartCoroutine(NowCheckForResourceUnload());
}
private bool hasTaskBeenDone_Threaded = false;
IEnumerator NowCheckForResourceUnload()
{
yield return null;
while (true)
{
if (hasTaskBeenDone_Threaded == true)
{
break;
}
yield return null;
}
Resources.UnloadUnusedAssets();
}
public void SaveDataThreaded()
{
thread = new Thread(SaveDataTaskThreaded);
thread.Start();
}
private void SaveDataTaskThreaded()
{
File.WriteAllBytes(fileSavePath, tmp_data);
Debug.LogWarning("data saved!- used threading");
tmp_data = null;
hasTaskBeenDone_Threaded = true;
}
private void SaveDataTaskNonThreaded()
{
File.WriteAllBytes(fileSavePath, tmp_data);
Debug.LogWarning("data saved!- did not use any threading");
tmp_data = null;
Resources.UnloadUnusedAssets();
}
}
1 Like
I have made a little wrap up of the script with callback support. In case, any of you need it.
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using UnityEngine;
using UnityEngine.UI;
public class BigFileTransfer : MonoBehaviour
{
public static BigFileTransfer instance;
public delegate void OnWriteComplete(bool success);
void Awake()
{
instance = this;
}
// Update is called once per frame
void Update () {
if (Input.GetKeyDown(KeyCode.Space))
{
DoClientCode();
}
}
void DoClientCode()
{
StartCoroutine(DownloadAndThenWrite());
}
IEnumerator DownloadAndThenWrite()
{
WWW www = new WWW("http://192.168.1.28/data.bin");
yield return www;
string fileSavePath = Application.persistentDataPath + "/data.bin";
Debug.LogWarning("will save data on: " + fileSavePath);
byte[] tmp_data = www.bytes;
www.Dispose();
BigFileTransfer.WriteAllBytesAsync(tmp_data, fileSavePath, success =>
{
if (success)
{
Debug.LogWarning("file has been written successfully in: " + fileSavePath);
}
else
{
Debug.LogWarning("could not write file in: " + fileSavePath);
}
});
}
public static void WriteAllBytesAsync(byte[] data, string savePath, OnWriteComplete callback = null)
{
Debug.LogWarning("save path API start: " + savePath);
if (instance.dataPool.ContainsKey(savePath))
{
instance.dataPool[savePath] = data;
}
else
{
instance.dataPool.Add(savePath, data);
}
instance.SaveDataThreaded(savePath);
if (instance.completedFlagPool.ContainsKey(savePath))
{
instance.completedFlagPool[savePath] = false;
}
else
{
instance.completedFlagPool.Add(savePath, false);
}
if (instance.callbackPool.ContainsKey(savePath))
{
instance.callbackPool[savePath] = callback;
}
else
{
instance.callbackPool.Add(savePath, callback);
}
instance.StartCoroutine(instance.NowCheckForResourceUnload(savePath));
}
IEnumerator NowCheckForResourceUnload(string savePath)
{
yield return null;
while (true)
{
bool flag = false;
if (completedFlagPool.ContainsKey(savePath))
{
flag = completedFlagPool[savePath];
}
else
{
throw new Exception("could not find the flag in the pool!");
}
if (flag)
{
break;
}
yield return null;
}
Resources.UnloadUnusedAssets();
if (completedFlagPool.ContainsKey(savePath))
{
completedFlagPool.Remove(savePath);
}
if (completedFlagPool.ContainsKey(savePath))
{
completedFlagPool.Remove(savePath);
}
}
private Dictionary<string, byte[]> dataPool = new Dictionary<string, byte[]>();
private Dictionary<string, Thread> threadPool = new Dictionary<string, Thread>();
private Dictionary<string, bool> completedFlagPool = new Dictionary<string, bool>();
private Dictionary<string, OnWriteComplete> callbackPool = new Dictionary<string, OnWriteComplete>();
void SaveDataThreaded(string savePath)
{
Thread thread = new Thread(() => SaveDataTaskThreaded(savePath));
if (threadPool.ContainsKey(savePath))
{
threadPool[savePath] = thread;
}
else
{
threadPool.Add(savePath, thread);
}
thread.Start();
}
void SaveDataTaskThreaded(string savePath)
{
byte[] data = null;
if (dataPool.ContainsKey(savePath))
{
data = dataPool[savePath];
}
else
{
CallCB(savePath, false);
throw new Exception("could not find the data to write on dataPool!");
}
File.WriteAllBytes(savePath, data);
Debug.LogWarning("data saved!- used threading for path: " + savePath);
data = null;
CallCB(savePath, true);
if (dataPool.ContainsKey(savePath))
{
dataPool.Remove(savePath);
}
if (threadPool.ContainsKey(savePath))
{
threadPool.Remove(savePath);
}
if (callbackPool.ContainsKey(savePath))
{
callbackPool.Remove(savePath);
}
}
void CallCB(string savePath, bool flag)
{
OnWriteComplete cb = null;
if (callbackPool.ContainsKey(savePath))
{
cb = callbackPool[savePath];
}
if (cb != null)
{
cb(flag);
}
}
}
11 Likes