How to draw a pixel on a raw image that corresponds to a 3D object's coordinates?

What I am trying to do is use the SetPixel function and draw pixels on a 2d image within a canvas that correspond to the position of a 3d object in the world. The 3d object is a model of a town attached to a terrain chunk. I can successfully get the image representing the terrain.

I currently have this code, and it works to some extent: the pixels are drawn on the image, but their position is incorrect.

Vector3 pos = obj.TransformPoint(obj.position);

image.SetPixel((int)pos.x, (int)pos.z, Color.red);

I assume it’s not as easy as that, could anyone provide any help? I’d be grateful for any assistance, even some pseudocode would be great.

1 Like

_

Unity3DPaint.cs

This code is an example of how to translate world point into a texture coordinate that’s being projected as a rectangular shape of arbitrary rotation.
It proves it by allowing you to draw with mouse in game view - but the general idea will stay the same whatever you want to do here exactly (to replace Material with RawImage I guess).

Unity3DPaint preview

using UnityEngine;
using UnityEngine.InputSystem;

public class Unity3DPaint : MonoBehaviour
{
	[SerializeField][Min(4)] int _textureWidth = 64;
	[SerializeField][Min(4)] int _textureHeight = 64;
	[SerializeField] Vector2 _regionWorldSize = new Vector2{ x=100 , y=100 };
	Texture2D _texture;
	[SerializeField] Material _dstMaterial = null;

	#if UNITY_EDITOR
	void OnValidate () { if( _texture!=null && ( _texture.width!=_textureWidth || _texture.height!=_textureHeight ) ) { Dispose(); CreateNewTexture(); } }
	void OnDrawGizmos ()
	{
		GetAreaWorldCorners( region:_regionWorldSize , forTransform:transform , TR:out var tr , TL:out var tl , BR:out var br , BL:out var bl );
		Gizmos.color = Color.red;				Gizmos.DrawLine(tl,tr);
		Gizmos.color = new Color(0,1,1,1);		Gizmos.DrawLine(bl,br);
		Gizmos.color = Color.green;				Gizmos.DrawLine(bl,tl);
		Gizmos.color = new Color(1,0,1,1);		Gizmos.DrawLine(br,tr);
	}
	#endif
	
	void OnEnable () => CreateNewTexture();
	void OnDisable () => Dispose();

	void Update ()
	{
		var mouse = Mouse.current;
		if( mouse!=null && mouse.leftButton.isPressed )
		{
			Vector3 origin = transform.position;
			Vector3 normal = transform.forward;
			var camera = Camera.main;
			var plane = new Plane( inNormal:normal , inPoint:origin );
			var ray = camera.ScreenPointToRay(Input.mousePosition);
			if( plane.Raycast(ray,out float hitDist) )
			{
				Vector3 point = ray.origin + ray.direction * hitDist;
				bool isPointInsideRegion = PointRegionProjection( out Vector2 coord , point:point , region:_regionWorldSize , center:origin , horizontal:-transform.right , vertical:transform.up );
				if( isPointInsideRegion )
				{
					Debug.DrawLine( camera.transform.position , point , Color.white , 0.1f );
					_texture.SetPixel( (int)( _texture.width * coord.x ) , (int)( _texture.height * coord.y ) , Color.red );
					_texture.Apply();
				}
				else Debug.DrawLine( camera.transform.position , point , Color.black , 0.1f );
			}
		}
	}

	void CreateNewTexture ()
	{
		_texture = new Texture2D( width:_textureWidth , height:_textureHeight , textureFormat:TextureFormat.ARGB32 , mipCount:3 , linear:true );
		_texture.filterMode = FilterMode.Point;// no smooth pixels
		_texture.SetPixel( 0 , 0 , Color.black );
		_texture.SetPixel( x:_texture.width-1 , y:_texture.height-1 , Color.magenta );
		_texture.Apply();
		if( _dstMaterial!=null ) _dstMaterial.mainTexture = _texture;
	}

	void Dispose () => Dispose( _texture );
	void Dispose ( Object obj )
	{
		#if UNITY_EDITOR
		if( Application.isPlaying ) { UnityEngine.Object.Destroy( obj ); }
		else { UnityEngine.Object.DestroyImmediate( obj ); }
		#else
		UnityEngine.Object.Destroy( thisObject );
		#endif
	}

	/// <summary>Projects world point onto a region of a plane with arbitrary rotation.</summary>
	/// <returns>True when coordinate is inside given region.</returns>
	/// <param name="coordinates">UV-type coordinates.</param>
	/// <param name="point">World point you want to project.</param>
	/// <param name="region">Region absolute world space size.</param>
	/// <param name="center">Region center (world space).</param>
	/// <param name="horizontal">Horizontal axis (world space).</param>
	/// <param name="vertical">Vertical axis (world space).</param>
	public static bool PointRegionProjection ( out Vector2 coordinates , Vector3 point , Vector2 region , Vector3 center , Vector3 horizontal , Vector3 vertical )
	{
		GetAreaWorldCorners( region:region , center:center , horizontal:horizontal , vertical:vertical , TR:out var TR , TL:out var TL , BR:out var BR , BL:out var BL );
		Vector3 pointRelativeToBLCorner = point - BL;
		Vector3 H = Vector3.Project( vector:pointRelativeToBLCorner , onNormal:horizontal );
		Vector3 V = Vector3.Project( vector:pointRelativeToBLCorner , onNormal:vertical );
		Vector2 regionalCoords = new Vector2{
			x = H.magnitude * Mathf.Sign( Vector3.Dot(H,horizontal) ) ,
			y = V.magnitude * Mathf.Sign( Vector3.Dot(V,vertical) )
		};
		coordinates = regionalCoords / region;
		return( coordinates.x>0 && coordinates.y>0 && coordinates.x<1f && coordinates.y<1f );
	}

	/// <param name="region">Region absolute world space size.</param>
	/// <param name="center">Region center (world space).</param>
	/// <param name="horizontal">Horizontal axis (world space).</param>
	/// <param name="vertical">Vertical axis (world space).</param>
	/// <param name="TR">Top-right corner (world space).</param>
	/// <param name="TL">Top-left corner (world space).</param>
	/// <param name="BR">Bottom-right corner (world space).</param>
	/// <param name="BL">Bottom-left corner (world space).</param>
	public static void GetAreaWorldCorners (
		Vector2 region ,
		Vector3 center , Vector3 horizontal , Vector3 vertical ,
		out Vector3 TR , out Vector3 TL , out Vector3 BR , out Vector3 BL
	)
	{
		horizontal = horizontal.normalized;
		vertical = vertical.normalized;
		Vector3 horizontalExtent = horizontal * region.x * 0.5f;
		Vector3 verticalExtent = vertical * region.y * 0.5f;

		TR = center + horizontalExtent + verticalExtent;
		TL = center + -horizontalExtent + verticalExtent;
		BR = center + horizontalExtent + -verticalExtent;
		BL = center + -horizontalExtent + -verticalExtent;
	}
	/// <inheritdoc/>
	/// <param name="forTransform">Transform to source <paramref name="center"/>, <paramref name="horizontal"/> and <paramref name="vertical"/> values from.</param>
	public static void GetAreaWorldCorners (
		Vector2 region ,
		Transform forTransform ,
		out Vector3 TR , out Vector3 TL , out Vector3 BR , out Vector3 BL
	)
	{
		Vector3 center = forTransform.position;
		Vector3 left = -forTransform.right;
		Vector3 up = forTransform.up;
		GetAreaWorldCorners( region:region , center:center , horizontal:left , vertical:up , TR:out TR , TL:out TL , BR:out BR , BL:out BL );
	}

}

_

TransformTexturePosition.cs

Still overcomplicated example but fits a bit more what you’re trying to do here. (it calls methods defined in Unity3DPaint class)

TransformTexturePosition preview

using UnityEngine;
public class TransformTexturePosition : MonoBehaviour
{
	[SerializeField] Vector3 _textureWorldHorizontalAxis = new Vector3( 1 , 0 , 0 );
	[SerializeField] Vector3 _textureWorldVerticalAxis = new Vector3( 0 , 0 , 1 );
	[SerializeField] Vector2 _textureWorldSize = new Vector2{ x=100 , y=100 };
	[SerializeField] Vector3 _textureWorldCenter = Vector3.zero;
	[SerializeField][Min(4)] int _textureWidth = 64;
	[SerializeField][Min(4)] int _textureHeight = 64;
	Texture2D _texture;
	[SerializeField] Material _dstMaterial = null;

	#if UNITY_EDITOR
	void OnValidate () { if( _texture!=null && ( _texture.width!=_textureWidth || _texture.height!=_textureHeight ) ) { Dispose(); CreateNewTexture(); } }
	void OnDrawGizmos ()
	{
		Unity3DPaint.GetAreaWorldCorners(
			region:			_textureWorldSize ,
			center:			_textureWorldCenter ,
			horizontal:		_textureWorldHorizontalAxis ,
			vertical:		_textureWorldVerticalAxis ,
			TR:out var tr , TL:out var tl , BR:out var br , BL:out var bl
		);
		Gizmos.color = Color.red;				Gizmos.DrawLine(tl,tr);
		Gizmos.color = new Color(0,1,1,1);		Gizmos.DrawLine(bl,br);
		Gizmos.color = Color.green;				Gizmos.DrawLine(bl,tl);
		Gizmos.color = new Color(1,0,1,1);		Gizmos.DrawLine(br,tr);
	}
	#endif
	
	void OnEnable () => CreateNewTexture();
	void OnDisable () => Dispose();

	void Update ()
	{
		Vector3 objectWorldPosition = transform.position;
		bool isObjectCoordInBounds = Unity3DPaint.PointRegionProjection(
			coordinates:	out Vector2 coord ,
			point:			objectWorldPosition ,
			region:			_textureWorldSize ,
			center:			_textureWorldCenter ,
			horizontal:		_textureWorldHorizontalAxis ,
			vertical:		_textureWorldVerticalAxis
		);
		if( isObjectCoordInBounds )
		{
			_texture.SetPixel( (int)(_texture.width*coord.x) , (int)(_texture.height*coord.y) , Color.red );
			_texture.Apply();
		}
	}

	void CreateNewTexture ()
	{
		_texture = new Texture2D( width:_textureWidth , height:_textureHeight , textureFormat:TextureFormat.ARGB32 , mipCount:3 , linear:true );
		_texture.filterMode = FilterMode.Point;// no smooth pixels
		_texture.SetPixel( 0 , 0 , Color.black );
		_texture.SetPixel( x:_texture.width-1 , y:_texture.height-1 , Color.magenta );
		_texture.Apply();
		if( _dstMaterial!=null ) _dstMaterial.mainTexture = _texture;
	}

	void Dispose () => Dispose( _texture );
	void Dispose ( Object obj )
	{
		#if UNITY_EDITOR
		if( Application.isPlaying ) { UnityEngine.Object.Destroy( obj ); }
		else { UnityEngine.Object.DestroyImmediate( obj ); }
		#else
		UnityEngine.Object.Destroy( thisObject );
		#endif
	}

}

You need to know
a) The size of the terrain you are trying to map
b) The resolution of the image that you are trying to map
c) The origin location of the town in the world (either its upper left or lower left corner, I don’t remember)
**
Then to calculate the corresponding pixel you need to get the position of the object relative to the terrain, normalize that position relative to the terrain size (in other words, on a scale of 0 to 1 how far to the right side of the terrain is my object), then scale back up by the resolution of your texture to find the specific pixel.

float terrainSizeX;
float terrainSizeZ;
Vector3 terrainCornerPosition;
int texResX;
int texResY;

Vector3 worldPos = obj.TransformPoint(obj.position);
Vector3 posRelativeToTerrain = worldPos - TerrainCornerPosition;
Vector3 normalizedPositionRelativeToTerrain = posRelativeToTerrain / new Vector3(terrainSizeX, 0, terrainSizeZ);
int pixelPosX = Mathf.RoundToInt(normalizedPositionRelativeToTerrain.x) * texResX;
int pixelPosY = Mathf.RoundToInt(normalizedPositionRelativeToTerrain.z)  * texResY;

You also may or may not have to flip the sign of the pixelPosY value