We embed the Unity process into a Windows Forms Panel, which is the child of a WindowsFormsHost, which again is the child of a WPF Grid Control.
The Unity process is started with the -parentHWND and the aforementioned Panel as arguments.
The old Input System worked fine.
But since we switched to the new Input System key press events are not fired for any key on the keyboard. Mouse input is still fine however.
Is this a bug? Is this intentional? Is there a way to make it work?
Help is greatly appreciated. Thank you!
It’s been almost a year and the problem with the “new” input system and an embedded Unity process still persists.
Is there any update / tip / magic trick that could finally get us around this problem?
@ConanB@pohype please file a bug report for this, from top of my head I don’t remember anything in our bug tracker related to UWP embedding breaking keyboard input, it should work so if it doesn’t it’s most likely a bug, it would be awesome to have a repro case for our UWP folks.
I investigated a pretty much identical issue several months ago:
TLDR: when hosted in a WPF application, we’re not receiving WM_INPUT messages (and new input system is based on them). Manually forwarding these messages in the WPF app makes the input system work again.
@Tautvydas-Zilys thanks for your answer! In the case we have no control over the parent context (I embed Unity in an Electron window, Electron has little to no control on the win32 api), is there a way to forward those messages? I was thinking about making a side c++ process that forwards the WM_INPUT messages, but it seems a bit overkill for something as simple as inputs…
@Tautvydas-Zilys thank you! I managed to make it work with the Window form example found in the documentation of -parentHWND.
Although, I’m afraid this still doesn’t solve the issue for non C# integrations, especially for Electron integrations in which you can hardly call RegisterRawInputDevices or any win32 api.
Do you know if there’s a workaround to achieve this without having to call win32 bindings from the parent window (maybe by doing it directly from the Unity child app, or something else ?).
Just for those coming after me, here’s the full code (Form1.cs) of the doc example to update for the new input system to work, you’ll also have to compile to x64 (i think SetWindowLongPtrW doesn’t work on 32 bit systems). It would be nice to update the doc zip file :).
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Windows.Forms;
using System.Diagnostics;
using System.Windows.Forms.VisualStyles;
namespace Container
{
public partial class Form1 : Form
{
[DllImport("User32.dll")]
static extern bool MoveWindow(IntPtr handle, int x, int y, int width, int height, bool redraw);
internal delegate int WindowEnumProc(IntPtr hwnd, IntPtr lparam);
[DllImport("user32.dll")]
internal static extern bool EnumChildWindows(IntPtr hwnd, WindowEnumProc func, IntPtr lParam);
[DllImport("user32.dll")]
static extern int SendMessage(IntPtr hWnd, int msg, IntPtr wParam, IntPtr lParam);
private Process process;
private IntPtr unityHWND = IntPtr.Zero;
private const int WM_ACTIVATE = 0x0006;
private const int WM_INPUT = 0x00FF;
private const int GWLP_WNDPROC = -4;
private readonly IntPtr WA_ACTIVE = new IntPtr(1);
private readonly IntPtr WA_INACTIVE = new IntPtr(0);
[StructLayout(LayoutKind.Sequential)]
public struct RAWINPUTDEVICE
{
public ushort usUsagePage;
public ushort usUsage;
public uint dwFlags;
public IntPtr hwndTarget;
}
const ushort HID_USAGE_PAGE_GENERIC = 0x01;
const ushort HID_USAGE_GENERIC_KEYBOARD = 0x06;
[DllImport("user32.dll", SetLastError = true)]
public static extern bool RegisterRawInputDevices([MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 0)] RAWINPUTDEVICE[] pRawInputDevices, int uiNumDevices, int cbSize);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr SetWindowLongPtrW(IntPtr hWnd, int nIndex, IntPtr dwNewLong);
private new delegate IntPtr WndProc(IntPtr hWnd, int msg, IntPtr wParam, IntPtr lParam);
private MulticastDelegate originalWndProc;
private WndProc myWndProc;
public Form1()
{
InitializeComponent();
try
{
process = new Process();
process.StartInfo.FileName = "Child.exe";
process.StartInfo.Arguments = "-parentHWND " + panel1.Handle.ToInt32() + " " + Environment.CommandLine;
process.StartInfo.UseShellExecute = true;
process.StartInfo.CreateNoWindow = true;
process.Start();
process.WaitForInputIdle();
EnumChildWindows(panel1.Handle, WindowEnum, IntPtr.Zero);
SetupRawInput(panel1.Handle);
unityHWNDLabel.Text = "Unity HWND: 0x" + unityHWND.ToString("X8");
}
catch (Exception ex)
{
MessageBox.Show(ex.Message + ".\nCheck if Container.exe is placed next to Child.exe.");
}
}
private void ActivateUnityWindow()
{
SendMessage(unityHWND, WM_ACTIVATE, WA_ACTIVE, IntPtr.Zero);
}
private void DeactivateUnityWindow()
{
SendMessage(unityHWND, WM_ACTIVATE, WA_INACTIVE, IntPtr.Zero);
}
private int WindowEnum(IntPtr hwnd, IntPtr lparam)
{
unityHWND = hwnd;
ActivateUnityWindow();
return 0;
}
private void panel1_Resize(object sender, EventArgs e)
{
MoveWindow(unityHWND, 0, 0, panel1.Width, panel1.Height, true);
ActivateUnityWindow();
}
// Close Unity application
private void Form1_FormClosed(object sender, FormClosedEventArgs e)
{
try
{
process.CloseMainWindow();
Thread.Sleep(1000);
while (process.HasExited == false)
process.Kill();
}
catch (Exception)
{
}
}
private void Form1_Activated(object sender, EventArgs e)
{
ActivateUnityWindow();
}
private void Form1_Deactivate(object sender, EventArgs e)
{
DeactivateUnityWindow();
}
private IntPtr HookWndProc(IntPtr hwnd, int msg, IntPtr wParam, IntPtr lParam)
{
if (msg == WM_INPUT)
{
SendMessage(unityHWND, msg, wParam, lParam);
return IntPtr.Zero;
}
return (IntPtr)originalWndProc.DynamicInvoke(new object[] { hwnd, msg, wParam, lParam });
}
private void SetupRawInput(IntPtr hostHWND)
{
myWndProc = HookWndProc;
var originalWndProcPtr = SetWindowLongPtrW(hostHWND, GWLP_WNDPROC , Marshal.GetFunctionPointerForDelegate(myWndProc));
if (originalWndProcPtr == null)
{
var errorCode = Marshal.GetLastWin32Error();
throw new Win32Exception(errorCode, "Failed to overwrite the original wndproc");
}
Type lel = typeof(MulticastDelegate);
originalWndProc = (MulticastDelegate)Marshal.GetDelegateForFunctionPointer(originalWndProcPtr, lel);
var rawInputDevices = new[]
{
new RAWINPUTDEVICE()
{
usUsagePage = HID_USAGE_PAGE_GENERIC,
usUsage = HID_USAGE_GENERIC_KEYBOARD,
dwFlags = 0,
hwndTarget = hostHWND
}
};
if (!RegisterRawInputDevices(rawInputDevices, 1, Marshal.SizeOf(typeof(RAWINPUTDEVICE))))
{
var lastError = Marshal.GetLastWin32Error();
throw new Win32Exception(lastError, "Failed to register raw input devices");
}
}
}
}
I believe raw input messages don’t arrive in -parentHwnd case due to Unity window becoming a child window. But if you create another window that’s in invisible in the Unity process and set up raw input events for that, you should be able to receive them.
@Tautvydas-Zilys alright I got this working with a bit of change! Initially, the code in the Windows.Input.ForwardRawInput.html documentation wasn’t working with my electron integration, but by adding the RIDEV_INPUTSINKflag on the RAWINPUTDEVICE it works like a charm! RIDEV_INPUTSINK allows the created window (which is invisible) to receive input even when it’s in the foreground (which happens when you focus on the Unity window).
Essentially, this consists in modifying the example code to add the dwFlags (to make this work you also have to enable unsafe code in the player settings and add the doc example Monobehaviour on a gameobject):
With this technique, inputs will always be detected and forwarded to the unity child window. You have to control focus yourself by allowing or not the Update loop to run (I personally do this when focusing inside/outside of HTML using a custom IPC between Electron and Unity).
@Tautvydas-Zilys do you know if this will be backported to the stable versions of Unity (2020 etc.)?
In any case, this opens the door to native integrations of Unity with several UI toolkits (WPF, Qt, Electron) and brings a lot of value. Thanks a lot to all of those who helped!
No, but on older releases you can just forward WM_INPUT event directly using “SendMessage” function. It doesn’t work as of 2021.2 due to this: https://discussions.unity.com/t/856247
I’m surprised you need RIDEV_INPUTSINK flag. It shouldn’t be needed if the focused window belongs to the same process as the one you register raw input on. Also be careful with that flag. AV software likes to flag it as a keylogger.
@Tautvydas-Zilys good to know for the AV flagging, I’ve changed nothing in the doc example except RIDEV_INPUTSINK because I noticed that the inputs were only working when focusing on the created window (after adding the WS_VISIBLE flag to CreateWindowEx at first, to make the window visible and focusable for tests purposes).
Regarding making this work with Unity 2020, I have tried with SendMessage and it works like a charm (still need RIDEV_INPUTSINK though). The only tricky thing is to gather the unityHWND from Unity itself ( see this post ).
For those coming after me, you’ll have to replace this chunk of code to have it work in Unity 2020:
[DllImport("user32.dll")]
static extern int SendMessage(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam);
static IntPtr unityHWND = IntPtr.Zero;
....
Awake(){
// Here write some code to gather the unity HWND, there are several ways to achieve this
// This is mandatory for this code to work
// Personally I've done it using EnumThreadWindows native method, but I'm not sure this is recommended, therefore I have not included it there
// See https://discussions.unity.com/t/699477
unityHWND = ...;
}
...
// Update the example method with SendMessage
[MonoPInvokeCallback(typeof(WndProcDelegate))]
static IntPtr WndProc(IntPtr window, uint message, IntPtr wParam, IntPtr lParam)
{
try
{
if (message == WM_INPUT)
{
SendMessage(unityHWND, message, wParam, lParam);
// ProcessRawInputMessage(lParam);
}
return DefWindowProcW(window, message, wParam, lParam);
}
catch (Exception e)
{
// Never let exception escape to native code as that will crash the app
Debug.LogException(e);
return IntPtr.Zero;
}
}
Once again @Tautvydas-Zilys thanks for your help, it’s really appreciated
Hi,
I had the same issue of the keyboard inputs not being detected and tried adding this code but I get the following error on line 168 of the attached code below:
System.ArgumentException: 'Object of type 'System.Int32' cannot be converted to type 'Interop+User32+WM'.'
Attached code (unity integrated into WPF app):
using System;
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
using System.Windows;
using System.Windows.Interop;
namespace Telekinesis.UnityApp
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
[DllImport("User32.dll")]
static extern bool MoveWindow(IntPtr handle, int x, int y, int width, int height, bool redraw);
internal delegate int WindowEnumProc(IntPtr hwnd, IntPtr lparam);
[DllImport("user32.dll")]
internal static extern bool EnumChildWindows(IntPtr hwnd, WindowEnumProc func, IntPtr lParam);
[DllImport("user32.dll")]
static extern int SendMessage(IntPtr hWnd, int msg, IntPtr wParam, IntPtr lParam);
private Process _process;
private IntPtr _unityHWND = IntPtr.Zero;
bool initialized = false;
private const int GWLP_WNDPROC = -4;
private const int WM_INPUT = 0x00FF;
private const int WM_ACTIVATE = 0x0006;
private readonly IntPtr WA_ACTIVE = new IntPtr(1);
private readonly IntPtr WA_INACTIVE = new IntPtr(0);
// To capture key presses and send to unity
[StructLayout(LayoutKind.Sequential)]
public struct RAWINPUTDEVICE
{
public ushort usUsagePage;
public ushort usUsage;
public uint dwFlags;
public IntPtr hwndTarget;
}
const ushort HID_USAGE_PAGE_GENERIC = 0x01;
const ushort HID_USAGE_GENERIC_KEYBOARD = 0x06;
[DllImport("user32.dll", SetLastError = true)]
public static extern bool RegisterRawInputDevices([MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 0)] RAWINPUTDEVICE[] pRawInputDevices, int uiNumDevices, int cbSize);
[DllImport("user32.dll", SetLastError = true)]
private static extern IntPtr SetWindowLongPtrW(IntPtr hWnd, int nIndex, IntPtr dwNewLong);
private new delegate IntPtr WndProc(IntPtr hWnd, int msg, IntPtr wParam, IntPtr lParam);
private MulticastDelegate originalWndProc;
private WndProc myWndProc;
public MainWindow()
{
InitializeComponent();
System.Windows.Threading.DispatcherTimer dispatcherTimer = new System.Windows.Threading.DispatcherTimer();
dispatcherTimer.Tick += attemptInit;
dispatcherTimer.Interval = new TimeSpan(0, 0, 1);
dispatcherTimer.Start();
}
void attemptInit(object sender, EventArgs e)
{
if (initialized)
return;
//HwndSource source = (HwndSource)HwndSource.FromVisual(UnityGrid);
//IntPtr hWnd = source.Handle;
IntPtr unityHandle = UnityGrid.Handle;
try
{
_process = new Process();
_process.StartInfo.FileName = "UnityWindow.exe";
_process.StartInfo.Arguments = "-parentHWND " + unityHandle.ToInt32() + " " + Environment.CommandLine;
_process.StartInfo.UseShellExecute = true;
_process.StartInfo.CreateNoWindow = true;
_process.Start();
_process.WaitForInputIdle();
// Doesn't work for some reason ?!
//hWnd = _process.MainWindowHandle;
EnumChildWindows(unityHandle, WindowEnum, IntPtr.Zero);
SetupRawInput(unityHandle);
Debug.WriteLine("Unity HWND: 0x" + _unityHWND.ToString("X8"));
UnityContentResize(this, EventArgs.Empty);
PresentationSource s = PresentationSource.FromVisual(Application.Current.MainWindow);
initialized = true;
}
catch (Exception ex)
{
MessageBox.Show(ex.Message + ".\nCheck if Container.exe is placed next to UnityGame.exe.");
}
}
private void ActivateUnityWindow()
{
SendMessage(_unityHWND, WM_ACTIVATE, WA_ACTIVE, IntPtr.Zero);
}
private void DeactivateUnityWindow()
{
SendMessage(_unityHWND, WM_ACTIVATE, WA_INACTIVE, IntPtr.Zero);
}
private int WindowEnum(IntPtr hwnd, IntPtr lparam)
{
_unityHWND = hwnd;
ActivateUnityWindow();
return 0;
}
private void UnityContentResize(object sender, EventArgs e)
{
MoveWindow(_unityHWND, 0, 0, (int)UnityGrid.Width, (int)UnityGrid.Height, true);
Debug.WriteLine("RESIZED UNITY WINDOW TO: " + (int)UnityGrid.Width + "x" + (int)UnityGrid.Height);
ActivateUnityWindow();
}
// Close Unity application
private void ApplicationExit(object sender, EventArgs e)
{
try
{
_process.CloseMainWindow();
Thread.Sleep(1000);
while (!_process.HasExited)
_process.Kill();
}
catch (Exception)
{
}
}
private void UnityContentActivate(object sender, EventArgs e)
{
ActivateUnityWindow();
}
private void UnityContentDeactivate(object sender, EventArgs e)
{
DeactivateUnityWindow();
}
private IntPtr HookWndProc(IntPtr hwnd, int msg, IntPtr wParam, IntPtr lParam)
{
if (msg == WM_INPUT)
{
SendMessage(_unityHWND, msg, wParam, lParam);
return IntPtr.Zero;
}
return (IntPtr)originalWndProc.DynamicInvoke(new object[] { hwnd, msg, wParam, lParam });
}
private void SetupRawInput(IntPtr hostHWND)
{
myWndProc = HookWndProc;
var originalWndProcPtr = SetWindowLongPtrW(hostHWND, GWLP_WNDPROC, Marshal.GetFunctionPointerForDelegate(myWndProc));
if (originalWndProcPtr == null)
{
var errorCode = Marshal.GetLastWin32Error();
throw new Win32Exception(errorCode, "Failed to overwrite the original wndproc");
}
Type lel = typeof(MulticastDelegate);
originalWndProc = (MulticastDelegate)Marshal.GetDelegateForFunctionPointer(originalWndProcPtr, lel);
var rawInputDevices = new[]
{
new RAWINPUTDEVICE()
{
usUsagePage = HID_USAGE_PAGE_GENERIC,
usUsage = HID_USAGE_GENERIC_KEYBOARD,
dwFlags = 0,
hwndTarget = hostHWND
}
};
if (!RegisterRawInputDevices(rawInputDevices, 1, Marshal.SizeOf(typeof(RAWINPUTDEVICE))))
{
var lastError = Marshal.GetLastWin32Error();
throw new Win32Exception(lastError, "Failed to register raw input devices");
}
}
}
}
@Tautvydas-Zilys Do you have any insights as to what may be going wrong?
I have created a bug report and the steps to reproduce it (ticket 1372661).