Hello, I’m currently a senior college student doing my capstone project. Part of my project is creating an AR app that will mainly allow user to scan (point their phone’s camera) a painting and a video will be superimposed to it. I’m using firebase to store images and video/animation of a painting. Basically, I want it to be dynamic. Like a basic Artivive AR implementation. This is my first time using unity or making anything related to AR so it’s really overwhelming and quite new to me (also, I don’t have the time to learn from the beginning). Here are the scripts I’ve made :
FirebaseManager:
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using UnityEngine;
using Firebase;
using Firebase.Firestore;
using Firebase.Extensions;
using UnityEngine.Networking;
public class FirebaseManager : MonoBehaviour
{
private FirebaseFirestore db; // Firebase Firestore database instance
public static event Action<string, string> OnImageVideoPairReady; // Event for new image-video pair
void Start()
{
InitializeFirebase(); // Initialize Firebase
RetrieveAndDownloadPaintings(); // Retrieve and download paintings
}
private void InitializeFirebase()
{
db = FirebaseFirestore.DefaultInstance; // Get the default Firestore instance
UnityEngine.Debug.Log("Initializing Firestore database instance.");
}
private void RetrieveAndDownloadPaintings()
{
UnityEngine.Debug.Log("Retrieving paintings...");
db.Collection("paintings").GetSnapshotAsync().ContinueWithOnMainThread(task =>
{
if (task.IsFaulted || task.IsCanceled)
{
UnityEngine.Debug.LogError("Failed to retrieve paintings from Firestore.");
return;
}
QuerySnapshot querySnapshot = task.Result;
UnityEngine.Debug.Log("Successfully retrieved paintings.");
foreach (var document in querySnapshot.Documents)
{
string id = document.Id;
string title = "";
string imageUrl = "";
string animationUrl = "";
if (document.TryGetValue("title", out title) &&
document.TryGetValue("imageUrl", out imageUrl) &&
document.TryGetValue("animationUrl", out animationUrl))
{
if (string.IsNullOrEmpty(animationUrl))
{
UnityEngine.Debug.LogWarning($"Skipping painting {title} because animationUrl is null or empty.");
continue;
}
UnityEngine.Debug.Log($"Processing painting: {title}");
// Start downloading image and animation if URLs are present
StartCoroutine(DownloadAndSaveFile(imageUrl, "PaintingImageFolder", $"{id}-{title}.jpg", (imagePath) =>
{
StartCoroutine(DownloadAndSaveFile(animationUrl, "PaintingVideoFolder", $"{id}-{title}.mp4", (videoPath) =>
{
OnImageVideoPairReady?.Invoke(imagePath, videoPath);
}));
}));
}
else
{
UnityEngine.Debug.LogError($"Document missing required fields: {id}");
}
}
});
}
private IEnumerator DownloadAndSaveFile(string url, string folderName, string fileName, Action<string> onComplete)
{
UnityEngine.Debug.Log("Starting download for: " + url);
UnityWebRequest request = UnityWebRequest.Get(url);
yield return request.SendWebRequest();
if (request.result == UnityWebRequest.Result.ConnectionError || request.result == UnityWebRequest.Result.ProtocolError)
{
UnityEngine.Debug.LogError("Failed to download file: " + request.error);
}
else
{
UnityEngine.Debug.Log("Download successful for: " + url);
string folderPath = Path.Combine(UnityEngine.Application.persistentDataPath, folderName);
if (!Directory.Exists(folderPath))
{
Directory.CreateDirectory(folderPath);
}
string filePath = Path.Combine(folderPath, fileName);
File.WriteAllBytes(filePath, request.downloadHandler.data);
UnityEngine.Debug.Log("File saved to: " + filePath);
onComplete?.Invoke(filePath); // Invoke callback with the file path
}
}
}
ARImageVideoManager:
using System; // Importing System namespace
using System.Collections; // Importing System.Collections namespace
using System.Collections.Generic; // Importing System.Collections.Generic namespace
using System.Diagnostics;
using System.IO; // Importing System.IO namespace
using UnityEngine; // Importing UnityEngine namespace
using UnityEngine.Networking; // Importing UnityEngine.Networking namespace
using UnityEngine.Video; // Importing UnityEngine.Video namespace
using UnityEngine.XR.ARFoundation; // Importing UnityEngine.XR.ARFoundation namespace
using UnityEngine.XR.ARSubsystems; // Importing UnityEngine.XR.ARSubsystems namespace
public class ARImageVideoManager : MonoBehaviour // Defining ARImageVideoManager class inheriting from MonoBehaviour
{
public ARTrackedImageManager trackedImageManager; // Reference to the ARTrackedImageManager
public GameObject videoPrefab; // Prefab for video playback
private Dictionary<string, string> imageToVideoMap = new Dictionary<string, string>(); // Dictionary to map image paths to video paths
private MutableRuntimeReferenceImageLibrary mutableLibrary; // Mutable runtime reference image library
void Start()
{
StartCoroutine(WaitForARSessionReady());
}
private IEnumerator WaitForARSessionReady()
{
// Wait until AR session is ready
while (ARSession.state == ARSessionState.CheckingAvailability ||
ARSession.state == ARSessionState.None)
{
UnityEngine.Debug.Log("Waiting for AR session to become ready. Current state: " + ARSession.state);
yield return new WaitForSeconds(0.3f); // Check every 0.5 seconds
}
if (ARSession.state == ARSessionState.Ready || ARSession.state == ARSessionState.SessionInitializing)
{
UnityEngine.Debug.Log("AR session is ready. Current state: " + ARSession.state);
InitializeRuntimeLibrary();
}
else
{
UnityEngine.Debug.LogWarning("AR session could not be initialized. Current state: " + ARSession.state);
}
}
private void OnEnable() // Method called when the script is enabled
{
FirebaseManager.OnImageVideoPairReady += OnImageVideoPairReady; // Subscribe to the OnImageVideoPairReady event
trackedImageManager.trackedImagesChanged += OnTrackedImagesChanged; // Subscribe to the trackedImagesChanged event
}
private void OnDisable() // Method called when the script is disabled
{
FirebaseManager.OnImageVideoPairReady -= OnImageVideoPairReady; // Unsubscribe from the OnImageVideoPairReady event
trackedImageManager.trackedImagesChanged -= OnTrackedImagesChanged; // Unsubscribe from the trackedImagesChanged event
}
private void InitializeRuntimeLibrary() // Method to initialize the runtime image library
{
if (trackedImageManager.descriptor.supportsMutableLibrary) // Check if the device supports mutable libraries
{
// Create an empty mutable library if supported
mutableLibrary = trackedImageManager.CreateRuntimeLibrary() as MutableRuntimeReferenceImageLibrary; // Create a mutable runtime library
if (mutableLibrary != null) // Check if the mutable library is created successfully
{
trackedImageManager.referenceLibrary = mutableLibrary; // Set the reference library to the mutable library
UnityEngine.Debug.Log("Mutable runtime library created successfully."); // Log success
}
else
{
UnityEngine.Debug.LogError("Failed to create a mutable runtime library."); // Log error if creation failed
}
}
else
{
UnityEngine.Debug.LogWarning("This device does not support mutable image libraries."); // Log warning if device does not support mutable libraries
}
}
private void OnImageVideoPairReady(string imagePath, string videoPath) // Event handler for OnImageVideoPairReady event
{
if (!imageToVideoMap.ContainsKey(imagePath)) // Check if the image path is not already in the dictionary
{
imageToVideoMap[imagePath] = videoPath; // Add the image and video paths to the dictionary
// Only attempt to add images if the library is mutable
if (mutableLibrary != null) // Check if the mutable library is not null
{
StartCoroutine(LoadAndAddImageToLibrary(imagePath, Path.GetFileNameWithoutExtension(imagePath))); // Start coroutine to load and add image to library
UnityEngine.Debug.Log("Added image to mutable image library"); // Log success
}
}
}
private IEnumerator LoadAndAddImageToLibrary(string filePath, string fileName) // Coroutine to load and add image to library
{
UnityEngine.Debug.Log("Loading image from: " + filePath); // Log the image path
using (UnityWebRequest uwr = UnityWebRequestTexture.GetTexture("file://" + filePath)) // Create a UnityWebRequest to load the image
{
yield return uwr.SendWebRequest(); // Send the web request and wait for it to complete
if (uwr.result == UnityWebRequest.Result.ConnectionError || uwr.result == UnityWebRequest.Result.ProtocolError) // Check for errors
{
UnityEngine.Debug.LogError("Failed to load image: " + uwr.error); // Log error if the image failed to load
}
else
{
Texture2D texture = DownloadHandlerTexture.GetContent(uwr); // Get the texture from the web request
if (texture != null) // Check if the texture is not null
{
UnityEngine.Debug.Log("Image loaded successfully: " + fileName); // Log success
mutableLibrary.ScheduleAddImageWithValidationJob(texture, fileName, 0.2f); // Add the image to the mutable library with a specified size
}
else
{
UnityEngine.Debug.LogError("Failed to convert image to texture: " + fileName); // Log error if the texture conversion failed
}
}
}
if (trackedImageManager.referenceLibrary != null)
{
UnityEngine.Debug.Log("Reference library image count: " + trackedImageManager.referenceLibrary.count);
}
}
private void OnTrackedImagesChanged(ARTrackedImagesChangedEventArgs eventArgs) // Event handler for trackedImagesChanged event
{
UnityEngine.Debug.Log("Current AR Session State during tracking at OnTracked: " + ARSession.state);
UnityEngine.Debug.Log("Tracked images changed."); // Log when tracked images change
foreach (ARTrackedImage trackedImage in eventArgs.updated) // Iterate through updated tracked images
{
UnityEngine.Debug.Log("Processing tracked image: " + trackedImage.referenceImage.name); // Log the name of the tracked image being processed
if (trackedImage.trackingState == TrackingState.Tracking) // Check if the tracking state is Tracking
{
UnityEngine.Debug.Log("Tracking state is Tracking for image: " + trackedImage.referenceImage.name); // Log if the image is being tracked
UpdateTrackedImage(trackedImage); // Update the tracked image
}
else
{
UnityEngine.Debug.Log("Tracking state is not Tracking for image: " + trackedImage.referenceImage.name); // Log if the image is not being tracked
foreach (Transform child in trackedImage.transform) // Iterate through child objects of the tracked image
{
UnityEngine.Debug.Log("Destroying child object of tracked image: " + trackedImage.referenceImage.name); // Log the destruction of the child object
Destroy(child.gameObject); // Destroy video prefab if tracking is lost
}
}
}
}
private void UpdateTrackedImage(ARTrackedImage trackedImage) // Method to update a tracked image
{
string imageName = trackedImage.referenceImage.name; // Get the image name
string imagePath = Path.Combine(UnityEngine.Application.persistentDataPath, "PaintingImageFolder", imageName + ".jpg"); // Get the image path
if (imageToVideoMap.ContainsKey(imagePath)) // Check if the image path exists in the dictionary
{
string videoPath = imageToVideoMap[imagePath]; // Get the corresponding video path
GameObject videoObject = Instantiate(videoPrefab, trackedImage.transform); // Instantiate the video prefab
videoObject.GetComponent<VideoPlayer>().url = videoPath; // Set the video URL
videoObject.GetComponent<VideoPlayer>().Play(); // Play the video
UnityEngine.Debug.Log("Playing video for image: " + imageName); // Log success
}
}
}
Also, if it were you how would you go about making the scene? Like which gameObjects and components to make. If possible, can you give me the step-by-step process.
Because in the scene that I’ve created, based on the logs, everything in the script is working except the tracking part of ARImageVideoManager (where, the no logs are printing).
private void OnTrackedImagesChanged(ARTrackedImagesChangedEventArgs eventArgs) // Event handler for trackedImagesChanged event
{
UnityEngine.Debug.Log("Current AR Session State during tracking at OnTracked: " + ARSession.state);
UnityEngine.Debug.Log("Tracked images changed."); // Log when tracked images change
foreach (ARTrackedImage trackedImage in eventArgs.updated) // Iterate through updated tracked images
{
UnityEngine.Debug.Log("Processing tracked image: " + trackedImage.referenceImage.name); // Log the name of the tracked image being processed
if (trackedImage.trackingState == TrackingState.Tracking) // Check if the tracking state is Tracking
{
UnityEngine.Debug.Log("Tracking state is Tracking for image: " + trackedImage.referenceImage.name); // Log if the image is being tracked
UpdateTrackedImage(trackedImage); // Update the tracked image
}
else
{
UnityEngine.Debug.Log("Tracking state is not Tracking for image: " + trackedImage.referenceImage.name); // Log if the image is not being tracked
foreach (Transform child in trackedImage.transform) // Iterate through child objects of the tracked image
{
UnityEngine.Debug.Log("Destroying child object of tracked image: " + trackedImage.referenceImage.name); // Log the destruction of the child object
Destroy(child.gameObject); // Destroy video prefab if tracking is lost
}
}
}
}
private void UpdateTrackedImage(ARTrackedImage trackedImage) // Method to update a tracked image
{
string imageName = trackedImage.referenceImage.name; // Get the image name
string imagePath = Path.Combine(UnityEngine.Application.persistentDataPath, "PaintingImageFolder", imageName + ".jpg"); // Get the image path
if (imageToVideoMap.ContainsKey(imagePath)) // Check if the image path exists in the dictionary
{
string videoPath = imageToVideoMap[imagePath]; // Get the corresponding video path
GameObject videoObject = Instantiate(videoPrefab, trackedImage.transform); // Instantiate the video prefab
videoObject.GetComponent<VideoPlayer>().url = videoPath; // Set the video URL
videoObject.GetComponent<VideoPlayer>().Play(); // Play the video
UnityEngine.Debug.Log("Playing video for image: " + imageName); // Log success
}
}
I’ve been at it for almost a week now, and I can’t seem to make it work. Tutorials available on Youtube and other sites are mostly for static ones. Would really appreciate your help.